diff --git a/client/android/client.go b/client/android/client.go index a21348f27..7eea83dc0 100644 --- a/client/android/client.go +++ b/client/android/client.go @@ -26,6 +26,8 @@ import ( "github.com/netbirdio/netbird/client/internal/routemanager" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/formatter" "github.com/netbirdio/netbird/route" @@ -40,11 +42,6 @@ const ( AnonymizeLevelStrict = nbAnonymize.LevelStrictString ) -// ConnectionListener export internal Listener for mobile -type ConnectionListener interface { - peer.Listener -} - // TunAdapter export internal TunAdapter for mobile type TunAdapter interface { device.TunAdapter @@ -85,6 +82,13 @@ type Client struct { deviceName string uiVersion string networkChangeListener listener.NetworkChangeListener + // netState outlives engine restarts: it mirrors the OS connectivity, not + // the engine lifecycle. Run and RunWithoutLogin inject it into each new + // ConnectClient, which distributes it to every reconnection loop. + netState *netstate.State + + // sweeper also outlives engine restarts; NotifyNetworkChange sweeps it. + sweeper *netsweep.Sweeper stateMu sync.RWMutex connectClient *internal.ConnectClient @@ -148,6 +152,7 @@ func NewClient(androidSDKVersion int, deviceName string, uiVersion string, tunAd execWorkaround(androidSDKVersion) net.SetAndroidProtectSocketFn(tunAdapter.ProtectSocket) + system.SetIFaceDiscover(iFaceDiscover) return &Client{ deviceName: deviceName, uiVersion: uiVersion, @@ -156,6 +161,8 @@ func NewClient(androidSDKVersion int, deviceName string, uiVersion string, tunAd recorder: peer.NewRecorder(""), ctxCancelLock: &sync.Mutex{}, networkChangeListener: networkChangeListener, + netState: netstate.New(), + sweeper: netsweep.New(), } } @@ -196,7 +203,8 @@ func (c *Client) Run(platformFiles PlatformFiles, urlOpener URLOpener, isAndroid } // todo do not throw error in case of cancelled context ctx = internal.CtxInitState(ctx) - connectClient := internal.NewConnectClient(ctx, cfg, c.recorder) + connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, + internal.WithNetworkState(c.netState), internal.WithSweeper(c.sweeper)) c.setState(cfg, cacheDir, cfgFile, connectClient) // This path runs the interactive SSO flow, so reaching here means the peer // is authenticated again — release the latch Status() reports from. Clear @@ -237,7 +245,8 @@ func (c *Client) RunWithoutLogin(platformFiles PlatformFiles, dns *DNSList, dnsR // todo do not throw error in case of cancelled context ctx = internal.CtxInitState(ctx) - connectClient := internal.NewConnectClient(ctx, cfg, c.recorder) + connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, + internal.WithNetworkState(c.netState), internal.WithSweeper(c.sweeper)) c.setState(cfg, cacheDir, cfgFile, connectClient) return connectClient.RunOnAndroid(c.tunAdapter, c.iFaceDiscover, c.networkChangeListener, slices.Clone(dns.items), dnsReadyListener, stateFile, cacheDir) } @@ -285,6 +294,24 @@ func (c *Client) GetTunSettings() (*TunSettings, error) { }, nil } +// SetNetworkAvailable feeds OS-reported network availability into the client. +// While unavailable, the internal reconnect loops suspend their attempts and +// the connection listener reports NoNetwork instead of Connecting; when +// availability returns, the loops resume immediately with a fresh backoff. +func (c *Client) SetNetworkAvailable(available bool) { + c.netState.Set(available) + c.recorder.SetNetworkAvailable(available) +} + +// NotifyNetworkChange marks the management, signal and relay connections +// stale after the OS switched networks and schedules a sweep that cuts +// whatever has not redialed on the new network by then. The engine and the +// TUN device stay untouched. +func (c *Client) NotifyNetworkChange() { + c.sweeper.MarkNetworkChange() + log.Infof("network change: connections marked stale") +} + // DebugBundle generates a debug bundle, uploads it, and returns the upload key. // It works both with and without a running engine. anonymizeLevel is "default" // or "strict"; strict also anonymizes internal IP ranges, peer names, and @@ -525,7 +552,11 @@ func (c *Client) OnUpdatedHostDNS(list *DNSList) error { // SetConnectionListener set the network connection listener func (c *Client) SetConnectionListener(listener ConnectionListener) { - c.recorder.SetConnectionListener(listener) + if listener == nil { + c.recorder.RemoveConnectionListener() + return + } + c.recorder.SetConnectionListener(connectionListenerAdapter{listener}) } // RemoveConnectionListener remove connection listener diff --git a/client/android/connection_listener.go b/client/android/connection_listener.go new file mode 100644 index 000000000..77c47574b --- /dev/null +++ b/client/android/connection_listener.go @@ -0,0 +1,41 @@ +//go:build android + +package android + +import ( + "github.com/netbirdio/netbird/client/internal/peer" +) + +// Client state values delivered via ConnectionListener.OnStateChanged, +// re-exported as basic constants so gomobile emits them into the generated +// Java bindings. They mirror peer.ClientState*: append-only, never reorder. +const ( + ClientStateDisconnected = int(peer.ClientStateDisconnected) + ClientStateConnected = int(peer.ClientStateConnected) + ClientStateConnecting = int(peer.ClientStateConnecting) + ClientStateDisconnecting = int(peer.ClientStateDisconnecting) + ClientStateNoNetwork = int(peer.ClientStateNoNetwork) +) + +// ConnectionListener export internal Listener for mobile. It mirrors +// peer.Listener with OnStateChanged taking a plain int (one of the +// ClientState* constants), because gomobile cannot bind named types. +type ConnectionListener interface { + OnStateChanged(state int) + OnConnected() + OnDisconnected() + OnConnecting() + OnDisconnecting() + OnAddressChanged(string, string) + OnPeersListChanged(int) +} + +// connectionListenerAdapter adapts the gomobile-facing ConnectionListener to +// peer.Listener, converting the typed state to the int the binding carries. +type connectionListenerAdapter struct { + ConnectionListener +} + +func (a connectionListenerAdapter) OnStateChanged(state peer.ClientState) { + a.ConnectionListener.OnStateChanged(int(state)) +} diff --git a/client/android/login.go b/client/android/login.go index 897b1561e..24c911eb5 100644 --- a/client/android/login.go +++ b/client/android/login.go @@ -191,40 +191,49 @@ func (a *Auth) login(urlOpener URLOpener, isAndroidTV bool) error { return nil } -// loginHintSetter is implemented by both concrete flows (PKCE and device code) -// but absent from the OAuthFlow interface, hence the assertion below — the same -// way internal/auth wires it in authenticateWithPKCEFlow. -type loginHintSetter interface { - SetLoginHint(hint string) -} - func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener, isAndroidTV bool) (*auth.TokenInfo, error) { - oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV) + oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV, profileLoginHint(a.cfgPath)) if err != nil { return nil, fmt.Errorf("failed to get OAuth flow: %v", err) } - // An empty hint is deliberate, not a fallback: a fresh profile leaves the - // choice to the IdP. Switching accounts is done by switching or removing - // profiles, not by logging out — logout keeps the email. - if a.cfgPath != "" { - if hint := readProfileEmail(a.cfgPath); hint != "" { - if setter, ok := oAuthFlow.(loginHintSetter); ok { - setter.SetLoginHint(hint) - } - } + return runOAuthFlow(a.ctx, oAuthFlow, urlOpener, nil) +} + +// profileLoginHint returns the stored account email for the profile at cfgPath. +// An empty hint is deliberate, not a fallback: a fresh profile leaves the +// choice to the IdP. Switching accounts is done by switching or removing +// profiles, not by logging out — logout keeps the email. +func profileLoginHint(cfgPath string) string { + if cfgPath == "" { + return "" + } + return readProfileEmail(cfgPath) +} + +// runOAuthFlow drives an already acquired OAuth flow to a token: requests the +// flow info, presents the verification URL through the opener and waits for +// the browser round-trip. Open is called synchronously — it is what marks the +// surface as opened on the client side, and a fast token's OnLoginSuccess is +// a no-op until it has, so the dismissal would be dropped rather than +// delayed. Openers must therefore not block: they post their UI work and +// return. onWaiting, when set, runs after the URL is shown, right before the +// blocking wait. +func runOAuthFlow(ctx context.Context, flow auth.OAuthFlow, urlOpener URLOpener, onWaiting func()) (*auth.TokenInfo, error) { + flowInfo, err := flow.RequestAuthInfo(ctx) + if err != nil { + return nil, fmt.Errorf("request auth info: %w", err) } - flowInfo, err := oAuthFlow.RequestAuthInfo(context.TODO()) - if err != nil { - return nil, fmt.Errorf("getting a request OAuth flow info failed: %v", err) + urlOpener.Open(flowInfo.VerificationURIComplete, flowInfo.UserCode) + + if onWaiting != nil { + onWaiting() } - go urlOpener.Open(flowInfo.VerificationURIComplete, flowInfo.UserCode) - - tokenInfo, err := oAuthFlow.WaitToken(a.ctx, flowInfo) + tokenInfo, err := flow.WaitToken(ctx, flowInfo) if err != nil { - return nil, fmt.Errorf("waiting for browser login failed: %v", err) + return nil, fmt.Errorf("wait for token: %w", err) } return &tokenInfo, nil diff --git a/client/android/profile_prefs.go b/client/android/profile_prefs.go new file mode 100644 index 000000000..9c1fd307b --- /dev/null +++ b/client/android/profile_prefs.go @@ -0,0 +1,38 @@ +//go:build android + +package android + +import ( + "fmt" + + "github.com/netbirdio/netbird/client/internal/profilemanager" +) + +type prefsStore interface { + Get(namespace string, v any) (bool, error) + Put(namespace string, v any) error +} + +type profilePrefs struct { + prefs *profilemanager.Prefs +} + +func newProfilePrefs(configDir, profileID string) (*profilePrefs, error) { + if configDir == "" || profileID == "" { + return nil, fmt.Errorf("profile prefs require a config dir and profile ID") + } + pm := NewProfileManager(configDir) + prefs, err := pm.serviceMgr.ProfilePrefs(profilemanager.ID(profileID), androidUsername) + if err != nil { + return nil, fmt.Errorf("resolve profile prefs: %w", err) + } + return &profilePrefs{prefs: prefs}, nil +} + +func (p *profilePrefs) Get(namespace string, v any) (bool, error) { + return p.prefs.Get(namespace, v) +} + +func (p *profilePrefs) Put(namespace string, v any) error { + return p.prefs.Put(namespace, v) +} diff --git a/client/android/ssh_client.go b/client/android/ssh_client.go new file mode 100644 index 000000000..2822b6539 --- /dev/null +++ b/client/android/ssh_client.go @@ -0,0 +1,649 @@ +//go:build android + +package android + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" + gossh "golang.org/x/crypto/ssh" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/auth" + "github.com/netbirdio/netbird/client/internal/profilemanager" + nbssh "github.com/netbirdio/netbird/client/ssh" + "github.com/netbirdio/netbird/client/ssh/detection" +) + +const ( + sshDialTimeout = 30 * time.Second + sshDetectionTimeout = 5 * time.Second +) + +// PasswordRequiredMarker tells Java to prompt for a password and retry. It is +// a string because gomobile flattens errors to their message, so a sentinel +// value would not survive the binding. +const PasswordRequiredMarker = "netbird-ssh-password-required" + +// HostKeyUnknownMarker tells Java to show the fingerprint and, on confirmation, +// retry with TrustHostKey set. The presented fingerprint is appended after the +// marker so the prompt can display it and the retry can guard against a key +// that changed between the two connects. Only regular (non-NetBird) servers +// reach this: NetBird peers verify against the registry. +const HostKeyUnknownMarker = "netbird-ssh-hostkey-unknown" + +var ( + errPasswordRequired = errors.New(PasswordRequiredMarker) + errClientClosed = errors.New("ssh client closed") +) + +// errHostKeyUnknown carries the presented fingerprint so Connect can build the +// marker message the Java side parses. +type errHostKeyUnknown struct { + fingerprint string +} + +func (e *errHostKeyUnknown) Error() string { + return HostKeyUnknownMarker + ":" + e.fingerprint +} + +// SSHTerminalListener receives SSH session events. It is implemented in Java. +// +// All callbacks are invoked from goroutines and may run concurrently with each +// other; the implementation must be safe to call from any thread. +type SSHTerminalListener interface { + OnConnected() + OnData(data []byte) + OnClose(reason string) + OnError(message string) +} + +// SSHClient is a NetBird-aware SSH client exposed to Java via gomobile. +// +// It dials through the running NetBird tunnel and runs a standard SSH session +// on top with PTY enabled. Host-key verification uses the NetBird-provided +// peer SSH host keys, identical to the desktop client. +type SSHClient struct { + nb *Client + mu sync.Mutex + listener SSHTerminalListener + urlOpener URLOpener + + sshClient *gossh.Client + session *gossh.Session + stdin io.WriteCloser + closed bool + + // gen identifies the current connection attempt. Connect and Close bump it, + // so an in-flight dial or a reader left over from a previous connection + // finds itself stale and stays silent instead of publishing OnConnected or + // OnClose for a connection the caller already abandoned. + gen uint64 + dialCancel context.CancelFunc + + // knownHostsConfigDir and knownHostsProfile locate the TOFU store for + // regular SSH servers in the profile's preferences. Java supplies them, + // since an overlay IP is a different host under a different profile. Empty + // until set: without them a regular server cannot be verified and Connect + // refuses one. + knownHostsConfigDir string + knownHostsProfile string + // trustHostKey carries the fingerprint the user confirmed on a previous + // attempt, so the retry accepts exactly that key and persists it. + trustHostKey string +} + +// NewSSHClient creates a new SSH client bound to the running NetBird Client. +func NewSSHClient(c *Client) *SSHClient { + return &SSHClient{nb: c} +} + +// SetListener registers the Java listener. Must be called before Connect to +// receive any events. +func (s *SSHClient) SetListener(l SSHTerminalListener) { + s.mu.Lock() + s.listener = l + s.mu.Unlock() +} + +// SetURLOpener registers the Java URL opener used to display the device-code +// authorization page in a Custom Tabs window when the target peer requires +// JWT authentication. Must be set before Connect to be effective. +func (s *SSHClient) SetURLOpener(opener URLOpener) { + s.mu.Lock() + s.urlOpener = opener + s.mu.Unlock() +} + +// SetKnownHostsStore points the TOFU host-key store at a profile's preferences. +// Must be set before connecting to a regular SSH server; without it such a +// server cannot be verified and Connect refuses one. +func (s *SSHClient) SetKnownHostsStore(configDir, profileID string) { + s.mu.Lock() + s.knownHostsConfigDir = configDir + s.knownHostsProfile = profileID + s.mu.Unlock() +} + +// TrustHostKey records the fingerprint the user confirmed for a regular server, +// so the next Connect accepts that exact key and adds it to the known-hosts +// store. Passing a fingerprint that no longer matches makes the connect fail +// rather than trust a key that changed since the prompt. +func (s *SSHClient) TrustHostKey(fingerprint string) { + s.mu.Lock() + s.trustHostKey = fingerprint + s.mu.Unlock() +} + +// Connect dials the SSH server through the NetBird tunnel and performs the +// SSH handshake. It auto-detects the server type via SSH banner inspection +// and selects the appropriate authentication path: +// +// - NetBird-SSH server requiring JWT: launches the OAuth 2.0 device-code +// flow, opens the verification URL through the registered URLOpener, and +// uses the resulting token as the SSH password. Host-key verification +// uses the NetBird peer registry. +// - NetBird-SSH server without JWT: authenticates with the NetBird SSH +// private key. Host-key verification uses the NetBird peer registry. +// - Regular SSH server (e.g. OpenSSH): authenticates with the NetBird key +// first (so a user-installed NetBird public key works), then falls back +// to the supplied password if non-empty. Host-key verification is +// trust-on-first-use against the per-profile known-hosts store. +// +// The password parameter is only consulted for regular SSH servers. +func (s *SSHClient) Connect(host string, port int, user, password string) error { + if port < 1 || port > 65535 { + return fmt.Errorf("invalid port: %d", port) + } + + cfg, cfgPath, cc := s.nb.authSnapshot() + if cc == nil { + return errors.New("netbird client not running") + } + if cfg == nil { + return errors.New("netbird config not loaded") + } + engine := cc.Engine() + if engine == nil { + return errors.New("netbird engine not available") + } + + s.mu.Lock() + s.gen++ + gen := s.gen + s.mu.Unlock() + + serverType := detectServerType(host, port) + log.Debugf("SSH server type: %s", serverType) + + authMethods, hostKeyCallback, err := s.buildAuth(cfg, cfgPath, engine, serverType, password) + if err != nil { + return err + } + + clientConfig := &gossh.ClientConfig{ + User: user, + Auth: authMethods, + HostKeyCallback: hostKeyCallback, + Timeout: sshDialTimeout, + } + err = s.dialAndHandshake(gen, host, port, clientConfig) + + // An unknown host key is a prompt, not a failure: return the marker intact + // (rootCause would unwrap it) so Java can show the fingerprint and retry. + var unknownHost *errHostKeyUnknown + if errors.As(err, &unknownHost) { + return errors.New(unknownHost.Error()) + } + + // A regular server may still accept a password, so let the caller ask for + // one instead of failing. NetBird servers never use a password, so a + // failure there is genuine. + if err != nil && serverType != detection.ServerTypeNetBirdJWT && + serverType != detection.ServerTypeNetBirdNoJWT && isAuthFailure(err) && + passwordCouldHelp(err, password != "") { + return errPasswordRequired + } + if err != nil { + return rootCause(err) + } + return nil +} + +// StartSession requests a PTY and starts an interactive shell. Output from +// the session is forwarded to the listener via OnData. +func (s *SSHClient) StartSession(cols, rows int) error { + err := s.startSession(cols, rows) + if err != nil { + log.Infof("SSH: start session failed: %v", err) + return rootCause(err) + } + return nil +} + +// Write sends data to the SSH session stdin. +func (s *SSHClient) Write(data []byte) error { + s.mu.Lock() + stdin := s.stdin + s.mu.Unlock() + if stdin == nil { + return errors.New("ssh session not started") + } + if _, err := stdin.Write(data); err != nil { + return fmt.Errorf("write stdin: %w", err) + } + return nil +} + +// Resize updates the PTY window size. +func (s *SSHClient) Resize(cols, rows int) error { + s.mu.Lock() + session := s.session + s.mu.Unlock() + if session == nil { + return errors.New("ssh session not started") + } + return session.WindowChange(rows, cols) +} + +// Reset makes a closed client usable for another Connect: Close leaves the +// one-shot guard set, and clearing it lets the same client back a reconnect. +func (s *SSHClient) Reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = false +} + +// Close terminates the SSH session and underlying connection. Safe to call +// multiple times. +func (s *SSHClient) Close() error { + s.mu.Lock() + s.gen++ + if s.dialCancel != nil { + s.dialCancel() + s.dialCancel = nil + } + sshClient := s.sshClient + session := s.session + stdin := s.stdin + s.sshClient = nil + s.session = nil + s.stdin = nil + notify := !s.closed + s.closed = true + listener := s.listener + s.mu.Unlock() + + if stdin != nil { + if err := stdin.Close(); err != nil { + log.Debugf("ssh: stdin close: %v", err) + } + } + if session != nil { + if err := session.Close(); err != nil && !errors.Is(err, io.EOF) { + log.Debugf("ssh: session close: %v", err) + } + } + var firstErr error + if sshClient != nil { + if err := sshClient.Close(); err != nil { + firstErr = err + } + } + if notify && listener != nil { + listener.OnClose("closed by client") + } + return firstErr +} + +func (s *SSHClient) startSession(cols, rows int) error { + log.Debugf("SSH: starting session %dx%d", cols, rows) + s.mu.Lock() + sshClient := s.sshClient + gen := s.gen + s.mu.Unlock() + + if sshClient == nil { + return errors.New("ssh client not connected") + } + + pty, err := nbssh.StartPTYSession(sshClient, cols, rows) + if err != nil { + return err + } + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + closeQuiet(pty.Session, "stale session") + return errClientClosed + } + s.session = pty.Session + s.stdin = pty.Stdin + s.mu.Unlock() + + readerDone := make(chan string, 2) + go func() { readerDone <- s.readLoop(pty.Stdout, "stdout") }() + go func() { readerDone <- s.readLoop(pty.Stderr, "stderr") }() + go func() { + reason := <-readerDone + if second := <-readerDone; reason == "" { + reason = second + } + s.notifyClose(gen, reason) + }() + log.Debug("SSH: session started, shell running") + return nil +} + +func (s *SSHClient) buildAuth(cfg *profilemanager.Config, cfgPath string, engine *internal.Engine, + serverType detection.ServerType, password string) ([]gossh.AuthMethod, gossh.HostKeyCallback, error) { + + switch serverType { + case detection.ServerTypeNetBirdJWT: + token, err := s.requestJWTToken(cfg, cfgPath) + if err != nil { + return nil, nil, fmt.Errorf("jwt: %w", err) + } + auths := []gossh.AuthMethod{gossh.Password(token)} + return auths, nbssh.CreateHostKeyCallback(nbssh.PeerKeyLookup(engine.GetPeerSSHKey)), nil + + case detection.ServerTypeNetBirdNoJWT: + if cfg.SSHKey == "" { + return nil, nil, errors.New("no NetBird SSH key available") + } + signer, err := gossh.ParsePrivateKey([]byte(cfg.SSHKey)) + if err != nil { + return nil, nil, fmt.Errorf("parse netbird ssh key: %w", err) + } + auths := []gossh.AuthMethod{gossh.PublicKeys(signer)} + return auths, nbssh.CreateHostKeyCallback(nbssh.PeerKeyLookup(engine.GetPeerSSHKey)), nil + + case detection.ServerTypeRegular: + var auths []gossh.AuthMethod + if cfg.SSHKey != "" { + if signer, err := gossh.ParsePrivateKey([]byte(cfg.SSHKey)); err == nil { + auths = append(auths, gossh.PublicKeys(signer)) + } else { + log.Debugf("ssh: parse netbird key for regular auth: %v", err) + } + } + if password != "" { + pw := password + auths = append(auths, gossh.Password(pw)) + auths = append(auths, gossh.KeyboardInteractive(func(_, _ string, questions []string, _ []bool) ([]string, error) { + answers := make([]string, len(questions)) + for i := range questions { + answers[i] = pw + } + return answers, nil + })) + } + if len(auths) == 0 { + // Nothing to offer at all: ask for a password rather than failing, + // so the caller can retry once the user supplies one. + return nil, nil, errPasswordRequired + } + callback, err := s.tofuHostKeyCallback() + if err != nil { + return nil, nil, err + } + return auths, callback, nil + + default: + return nil, nil, fmt.Errorf("unsupported SSH server type: %v", serverType) + } +} + +// tofuHostKeyCallback verifies a regular server's host key against the +// per-profile known-hosts store. An unknown host returns errHostKeyUnknown so +// Java can show the fingerprint and, once confirmed, retry with the key +// trusted; a changed key is rejected outright, as OpenSSH does. When the user +// has confirmed a fingerprint, the callback accepts exactly that key and +// appends it to the store. +func (s *SSHClient) tofuHostKeyCallback() (gossh.HostKeyCallback, error) { + s.mu.Lock() + configDir := s.knownHostsConfigDir + profileID := s.knownHostsProfile + trusted := s.trustHostKey + s.mu.Unlock() + + if configDir == "" || profileID == "" { + return nil, errors.New("no known-hosts store configured for regular SSH") + } + + store, err := openKnownHostsStore(configDir, profileID) + if err != nil { + return nil, fmt.Errorf("load known-hosts store: %w", err) + } + + return func(hostname string, remote net.Addr, key gossh.PublicKey) error { + verdict, err := store.verify(hostname, remote, key) + if err != nil { + return err + } + if verdict == hostKeyMatched { + return nil + } + if verdict == hostKeyChanged { + return fmt.Errorf("SSH host key changed for %s (possible attack)", hostname) + } + + fingerprint := gossh.FingerprintSHA256(key) + if trusted == "" { + return &errHostKeyUnknown{fingerprint: fingerprint} + } + if trusted != fingerprint { + return fmt.Errorf("SSH host key changed since it was confirmed for %s", hostname) + } + if err := store.append(hostname, remote, key); err != nil { + return fmt.Errorf("persist trusted host key: %w", err) + } + // The confirmation is spent: now that the key is stored, a later + // reconnect must verify against the file, not re-accept this fingerprint. + s.mu.Lock() + s.trustHostKey = "" + s.mu.Unlock() + return nil + }, nil +} + +func (s *SSHClient) requestJWTToken(cfg *profilemanager.Config, cfgPath string) (string, error) { + s.mu.Lock() + urlOpener := s.urlOpener + s.mu.Unlock() + if urlOpener == nil { + return "", errors.New("URL opener not configured for JWT auth") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + flow, err := auth.NewOAuthFlow(ctx, cfg, false, true, profileLoginHint(cfgPath)) + if err != nil { + return "", fmt.Errorf("create oauth flow: %w", err) + } + + // The status callback covers the browser round-trip, which would + // otherwise leave the terminal blank. + tokenInfo, err := runOAuthFlow(ctx, flow, urlOpener, func() { + s.notifyStatus("Waiting for browser authentication...") + }) + if err != nil { + return "", err + } + + token := tokenInfo.GetTokenToUse() + if token == "" { + return "", errors.New("empty token returned by IdP") + } + + // Tells the client the browser round-trip is over so it can dismiss the + // surface it opened, the same way the login and session-extend flows do. + // Without it the Custom Tab stays in front of the terminal even though the + // token has already been collected. + urlOpener.OnLoginSuccess() + + return token, nil +} + +func (s *SSHClient) dialAndHandshake(gen uint64, host string, port int, clientConfig *gossh.ClientConfig) error { + addr := net.JoinHostPort(host, strconv.Itoa(port)) + ctx, cancel := context.WithTimeout(context.Background(), sshDialTimeout) + defer cancel() + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + return errClientClosed + } + s.dialCancel = cancel + s.mu.Unlock() + + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + return fmt.Errorf("dial %s: %w", addr, err) + } + + client, err := nbssh.Handshake(ctx, conn, addr, clientConfig) + if err != nil { + return err + } + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + closeQuiet(client, "stale ssh client") + return errClientClosed + } + s.sshClient = client + listener := s.listener + s.mu.Unlock() + + if listener != nil { + listener.OnConnected() + } + return nil +} + +func (s *SSHClient) readLoop(r io.Reader, name string) string { + buf := make([]byte, 4096) + for { + n, err := r.Read(buf) + if n > 0 { + s.mu.Lock() + listener := s.listener + s.mu.Unlock() + if listener != nil { + chunk := make([]byte, n) + copy(chunk, buf[:n]) + listener.OnData(chunk) + } + } + if err != nil { + // EOF is a normal shell exit, so report it without a reason. + if errors.Is(err, io.EOF) { + return "" + } + log.Debugf("ssh %s read: %v", name, err) + return rootCause(err).Error() + } + } +} + +// notifyStatus writes a progress line to the terminal through the normal +// output path, so long steps are visible while nothing else is arriving. +func (s *SSHClient) notifyStatus(text string) { + s.mu.Lock() + listener := s.listener + s.mu.Unlock() + if listener != nil { + listener.OnData([]byte("\r\n\x1b[33m" + text + "\x1b[0m\r\n")) + } +} + +func (s *SSHClient) notifyClose(gen uint64, reason string) { + s.mu.Lock() + if gen != s.gen || s.closed { + s.mu.Unlock() + return + } + s.closed = true + listener := s.listener + s.mu.Unlock() + if listener != nil { + listener.OnClose(reason) + } +} + +func closeQuiet(c io.Closer, label string) { + if c == nil { + return + } + if err := c.Close(); err != nil && !errors.Is(err, io.EOF) { + log.Debugf("ssh: close %s: %v", label, err) + } +} + +func detectServerType(host string, port int) detection.ServerType { + ctx, cancel := context.WithTimeout(context.Background(), sshDetectionTimeout) + defer cancel() + + dialer := &net.Dialer{} + serverType, err := detection.DetectSSHServerType(ctx, dialer, host, port) + if err != nil { + log.Debugf("ssh: server detection failed: %v (assuming regular SSH)", err) + return detection.ServerTypeRegular + } + return serverType +} + +// rootCause returns the innermost error of a %w chain, so the terminal shows +// "i/o timeout" rather than every layer that added context on the way up. +func rootCause(err error) error { + for { + // A joined error has no single root, so keep it as-is. + if _, ok := err.(interface{ Unwrap() []error }); ok { + return err + } + next := errors.Unwrap(err) + if next == nil { + return err + } + err = next + } +} + +// isAuthFailure distinguishes credential rejection from dial, timeout and +// host-key errors, which retrying with a password would not fix. +func isAuthFailure(err error) bool { + if errors.Is(err, errPasswordRequired) { + return true + } + var partial *gossh.PartialSuccessError + if errors.As(err, &partial) { + return true + } + return strings.Contains(err.Error(), "unable to authenticate") +} + +// passwordCouldHelp reports whether prompting for a password again can change +// the outcome. gossh lists a method under "attempted methods" only when the +// server offered it, so a supplied password that was never attempted means the +// server does not accept passwords and the real error should surface instead. +func passwordCouldHelp(err error, passwordOffered bool) bool { + if !passwordOffered { + return true + } + msg := err.Error() + return strings.Contains(msg, "password") || strings.Contains(msg, "keyboard-interactive") +} diff --git a/client/android/ssh_known_hosts.go b/client/android/ssh_known_hosts.go new file mode 100644 index 000000000..eea90fd32 --- /dev/null +++ b/client/android/ssh_known_hosts.go @@ -0,0 +1,168 @@ +//go:build android + +package android + +import ( + "bytes" + "net" + "strconv" + "strings" + "sync" + + gossh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +const knownHostsNamespace = "ssh" + +const ( + hostKeyUnknown hostKeyVerdict = iota + hostKeyMatched + hostKeyChanged +) + +var knownHostsMu sync.Mutex + +type hostKeyVerdict uint8 + +type knownHostsSection struct { + KnownHosts []string `json:"knownHosts"` +} + +type knownHostsStore struct { + prefs prefsStore +} + +// RemoveKnownHost deletes every known-hosts entry for host:port from the +// profile's store, so a host trusted for a session that is being deleted does +// not linger. Java calls this only once no session targets that host, so a +// shared host stays trusted. A missing entry is not an error: the goal state +// is "absent". +func RemoveKnownHost(configDir, profileID, host string, port int) error { + store, err := openKnownHostsStore(configDir, profileID) + if err != nil { + return err + } + return store.removeHost(host, port) +} + +func openKnownHostsStore(configDir, profileID string) (*knownHostsStore, error) { + prefs, err := newProfilePrefs(configDir, profileID) + if err != nil { + return nil, err + } + return &knownHostsStore{prefs: prefs}, nil +} + +func (st *knownHostsStore) verify(hostname string, remote net.Addr, key gossh.PublicKey) (hostKeyVerdict, error) { + lines, err := st.lines() + if err != nil { + return hostKeyUnknown, err + } + targets := knownHostsTargets(hostname, remote) + + verdict := hostKeyUnknown + for _, line := range lines { + pubKey, ok := knownHostsLineKey(line, targets) + if !ok { + continue + } + if pubKey.Type() == key.Type() && bytes.Equal(pubKey.Marshal(), key.Marshal()) { + return hostKeyMatched, nil + } + verdict = hostKeyChanged + } + return verdict, nil +} + +func (st *knownHostsStore) append(hostname string, remote net.Addr, key gossh.PublicKey) error { + line := knownhosts.Line(knownHostsTargets(hostname, remote), key) + + knownHostsMu.Lock() + defer knownHostsMu.Unlock() + + lines, err := st.lines() + if err != nil { + return err + } + return st.prefs.Put(knownHostsNamespace, knownHostsSection{KnownHosts: append(lines, line)}) +} + +func (st *knownHostsStore) removeHost(host string, port int) error { + target := knownhosts.Normalize(net.JoinHostPort(host, strconv.Itoa(port))) + + knownHostsMu.Lock() + defer knownHostsMu.Unlock() + + lines, err := st.lines() + if err != nil { + return err + } + kept := make([]string, 0, len(lines)) + for _, line := range lines { + if knownHostsLineMatches(line, target) { + continue + } + kept = append(kept, line) + } + if len(kept) == len(lines) { + return nil + } + return st.prefs.Put(knownHostsNamespace, knownHostsSection{KnownHosts: kept}) +} + +func (st *knownHostsStore) lines() ([]string, error) { + var section knownHostsSection + if _, err := st.prefs.Get(knownHostsNamespace, §ion); err != nil { + return nil, err + } + return section.KnownHosts, nil +} + +func knownHostsTargets(hostname string, remote net.Addr) []string { + targets := []string{knownhosts.Normalize(hostname)} + if remote != nil { + if normalized := knownhosts.Normalize(remote.String()); normalized != targets[0] { + targets = append(targets, normalized) + } + } + return targets +} + +func knownHostsLineKey(line string, targets []string) (gossh.PublicKey, bool) { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + return nil, false + } + _, hosts, pubKey, _, _, err := gossh.ParseKnownHosts([]byte(trimmed)) + if err != nil { + return nil, false + } + for _, host := range hosts { + for _, target := range targets { + if host == target { + return pubKey, true + } + } + } + return nil, false +} + +// knownHostsLineMatches reports whether a known-hosts line's address list +// contains the normalized target. Comment and blank lines never match. +func knownHostsLineMatches(line, target string) bool { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + return false + } + fields := strings.Fields(trimmed) + if len(fields) == 0 { + return false + } + for _, addr := range strings.Split(fields[0], ",") { + if addr == target { + return true + } + } + return false +} diff --git a/client/android/ssh_sessions.go b/client/android/ssh_sessions.go new file mode 100644 index 000000000..44b5464e9 --- /dev/null +++ b/client/android/ssh_sessions.go @@ -0,0 +1,104 @@ +//go:build android + +package android + +const ( + sshSessionsNamespace = "ssh-sessions" + maxStoredSSHSessions = 50 +) + +type sshSessionRecord struct { + ID string `json:"id"` + Host string `json:"host"` + Port int `json:"port"` + User string `json:"user"` +} + +type sshSessionsSection struct { + Sessions []sshSessionRecord `json:"sessions"` +} + +// SSHSessionEntry is one stored SSH session, without any credential. +type SSHSessionEntry struct { + ID string + Host string + Port int + User string +} + +// SSHSessionArray wraps stored SSH sessions for gomobile compatibility. +type SSHSessionArray struct { + items []*SSHSessionEntry +} + +// NewSSHSessionArray creates an empty session array to fill via Add. +func NewSSHSessionArray() *SSHSessionArray { + return &SSHSessionArray{} +} + +// Add appends a session entry, oldest first. +func (a *SSHSessionArray) Add(id, host string, port int, user string) { + a.items = append(a.items, &SSHSessionEntry{ID: id, Host: host, Port: port, User: user}) +} + +// Length returns the number of entries. +func (a *SSHSessionArray) Length() int { + return len(a.items) +} + +// Get returns the entry at index i, or nil when out of range. +func (a *SSHSessionArray) Get(i int) *SSHSessionEntry { + if i < 0 || i >= len(a.items) { + return nil + } + return a.items[i] +} + +// SSHSessionStore reads and writes a profile's stored SSH sessions. +type SSHSessionStore struct { + prefs prefsStore +} + +// NewSSHSessionStore opens the session store of the given profile. +func NewSSHSessionStore(configDir, profileID string) (*SSHSessionStore, error) { + prefs, err := newProfilePrefs(configDir, profileID) + if err != nil { + return nil, err + } + return &SSHSessionStore{prefs: prefs}, nil +} + +// Load returns the stored sessions, oldest first. +func (s *SSHSessionStore) Load() (*SSHSessionArray, error) { + var section sshSessionsSection + if _, err := s.prefs.Get(sshSessionsNamespace, §ion); err != nil { + return nil, err + } + + out := NewSSHSessionArray() + for _, record := range section.Sessions { + if record.ID == "" || record.Host == "" { + continue + } + out.Add(record.ID, record.Host, record.Port, record.User) + } + return out, nil +} + +// Save replaces the stored sessions, keeping only the newest entries when the +// list exceeds the storage cap. +func (s *SSHSessionStore) Save(sessions *SSHSessionArray) error { + var items []*SSHSessionEntry + if sessions != nil { + items = sessions.items + } + if len(items) > maxStoredSSHSessions { + items = items[len(items)-maxStoredSSHSessions:] + } + + records := make([]sshSessionRecord, 0, len(items)) + for _, item := range items { + records = append(records, sshSessionRecord{ID: item.ID, Host: item.Host, Port: item.Port, User: item.User}) + } + return s.prefs.Put(sshSessionsNamespace, sshSessionsSection{Sessions: records}) +} diff --git a/client/embed/embed.go b/client/embed/embed.go index 99a6b8229..1b2d84d7e 100644 --- a/client/embed/embed.go +++ b/client/embed/embed.go @@ -21,7 +21,7 @@ import ( "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/profilemanager" - sshcommon "github.com/netbirdio/netbird/client/ssh" + nbssh "github.com/netbirdio/netbird/client/ssh" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/shared/management/domain" mgmProto "github.com/netbirdio/netbird/shared/management/proto" @@ -521,12 +521,7 @@ func (c *Client) VerifySSHHostKey(peerAddress string, key []byte) error { return err } - storedKey, found := engine.GetPeerSSHKey(peerAddress) - if !found { - return sshcommon.ErrPeerNotFound - } - - return sshcommon.VerifyHostKey(storedKey, key, peerAddress) + return nbssh.PeerKeyLookup(engine.GetPeerSSHKey).VerifySSHHostKey(peerAddress, key) } // SetPerformance retunes a running Client. Only PreallocatedBuffersPerPool diff --git a/client/grpc/dialer_generic.go b/client/grpc/dialer_generic.go index 479575996..8a80525e9 100644 --- a/client/grpc/dialer_generic.go +++ b/client/grpc/dialer_generic.go @@ -16,28 +16,47 @@ import ( "google.golang.org/grpc" nbnet "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/netsweep" ) func WithCustomDialer(_ bool, _ string) grpc.DialOption { + return grpc.WithContextDialer(dialContext) +} + +// WithSweeper dials like WithCustomDialer but registers connections and +// dials with the sweeper. Append it after WithCustomDialer: gRPC applies +// dial options in order, so the later context dialer wins. +func WithSweeper(sweeper *netsweep.Sweeper) grpc.DialOption { return grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { - if runtime.GOOS == "linux" { - currentUser, err := user.Current() - if err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "failed to get current user: %v", err) - } + dial := sweeper.StartDial(ctx) + defer dial.Release() - // the custom dialer requires root permissions which are not required for use cases run as non-root - if currentUser.Uid != "0" { - log.Debug("Not running as root, using standard dialer") - dialer := &net.Dialer{} - return dialer.DialContext(ctx, "tcp", addr) - } - } - - conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) + conn, err := dialContext(dial.Ctx(), addr) if err != nil { - return nil, fmt.Errorf("nbnet.NewDialer().DialContext: %w", err) + return nil, err } - return conn, nil + return dial.WrapConn(conn) }) } + +func dialContext(ctx context.Context, addr string) (net.Conn, error) { + if runtime.GOOS == "linux" { + currentUser, err := user.Current() + if err != nil { + return nil, status.Errorf(codes.FailedPrecondition, "failed to get current user: %v", err) + } + + // the custom dialer requires root permissions which are not required for use cases run as non-root + if currentUser.Uid != "0" { + log.Debug("Not running as root, using standard dialer") + dialer := &net.Dialer{} + return dialer.DialContext(ctx, "tcp", addr) + } + } + + conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) + if err != nil { + return nil, fmt.Errorf("nbnet.NewDialer().DialContext: %w", err) + } + return conn, nil +} diff --git a/client/grpc/dialer_js.go b/client/grpc/dialer_js.go index b89ec3c21..8863756d7 100644 --- a/client/grpc/dialer_js.go +++ b/client/grpc/dialer_js.go @@ -3,6 +3,7 @@ package grpc import ( "google.golang.org/grpc" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/util/wsproxy/client" ) @@ -11,3 +12,8 @@ import ( func WithCustomDialer(tlsEnabled bool, component string) grpc.DialOption { return client.WithWebSocketDialer(tlsEnabled, component) } + +// WithSweeper is a no-op on WASM/JS: there is no network change signal. +func WithSweeper(_ *netsweep.Sweeper) grpc.DialOption { + return grpc.EmptyDialOption{} +} diff --git a/client/grpc/retry.go b/client/grpc/retry.go new file mode 100644 index 000000000..754ffa341 --- /dev/null +++ b/client/grpc/retry.go @@ -0,0 +1,49 @@ +package grpc + +import ( + "context" + "errors" + "time" + + "github.com/cenkalti/backoff/v4" + + "github.com/netbirdio/netbird/client/netstate" +) + +// Retry mirrors backoff.Retry, but the sleep between attempts also wakes on +// OS network availability transitions: an operation cut down by a network +// change retries the moment the network settles instead of sleeping through +// the recovery. A nil netState never fires, leaving plain backoff.Retry +// behavior. +func Retry(ctx context.Context, operation backoff.Operation, bo backoff.BackOff, netState *netstate.State) error { + bo.Reset() + for { + err := operation() + if err == nil { + return nil + } + + var permanent *backoff.PermanentError + if errors.As(err, &permanent) { + return permanent.Err + } + + next := bo.NextBackOff() + if next == backoff.Stop { + if cerr := ctx.Err(); cerr != nil { + return cerr + } + return err + } + + timer := time.NewTimer(next) + select { + case <-timer.C: + case <-netState.Changed(): + timer.Stop() + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + } + } +} diff --git a/client/grpc/retry_test.go b/client/grpc/retry_test.go new file mode 100644 index 000000000..4edca47b6 --- /dev/null +++ b/client/grpc/retry_test.go @@ -0,0 +1,91 @@ +package grpc + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/netstate" +) + +func TestRetryWakesOnNetworkChange(t *testing.T) { + ns := netstate.New() + attempts := 0 + operation := func() error { + attempts++ + if attempts == 1 { + return errors.New("cut by network change") + } + return nil + } + + go func() { + time.Sleep(20 * time.Millisecond) + ns.Set(false) + }() + + start := time.Now() + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Minute), ns) + + require.NoError(t, err) + assert.Equal(t, 2, attempts, "network change must cause one immediate retry") + assert.Less(t, time.Since(start), time.Second, "the transition must cut the minute-long sleep short") +} + +func TestRetryPermanentError(t *testing.T) { + sentinel := errors.New("permission denied") + operation := func() error { + return backoff.Permanent(sentinel) + } + + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Millisecond), nil) + assert.ErrorIs(t, err, sentinel, "permanent errors must stop retries") +} + +func TestRetryNilNetState(t *testing.T) { + attempts := 0 + operation := func() error { + attempts++ + if attempts < 3 { + return errors.New("transient") + } + return nil + } + + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Millisecond), nil) + require.NoError(t, err) + assert.Equal(t, 3, attempts, "nil network state must preserve timed retries") +} + +func TestRetryStops(t *testing.T) { + failure := errors.New("still failing") + operation := func() error { + return failure + } + + err := Retry(context.Background(), operation, &backoff.StopBackOff{}, nil) + assert.ErrorIs(t, err, failure, "stop backoff must return the operation error") +} + +func TestRetryCtxCancelDuringSleep(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + operation := func() error { + return errors.New("failing") + } + + go func() { + time.Sleep(20 * time.Millisecond) + cancel() + }() + + start := time.Now() + err := Retry(ctx, operation, backoff.NewConstantBackOff(time.Minute), netstate.New()) + + assert.ErrorIs(t, err, context.Canceled, "context cancellation must stop the retry loop") + assert.Less(t, time.Since(start), time.Second, "context cancellation must interrupt backoff sleep") +} diff --git a/client/internal/auth/auth.go b/client/internal/auth/auth.go index 153727a6c..b3a9e1158 100644 --- a/client/internal/auth/auth.go +++ b/client/internal/auth/auth.go @@ -138,26 +138,37 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) { // GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection // This avoids creating a new connection to the management server -func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool) (OAuthFlow, error) { +func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, hint string) (OAuthFlow, error) { var flow OAuthFlow - var err error - err = a.withRetry(ctx, func(client *mgm.GrpcClient) error { + err := a.withRetry(ctx, func(client *mgm.GrpcClient) error { if forceDeviceAuth { - flow, err = a.getDeviceFlow(client) - return err + deviceFlow, err := a.getDeviceFlow(client) + if err != nil { + return err + } + deviceFlow.SetLoginHint(hint) + flow = deviceFlow + return nil } // Try PKCE flow first - flow, err = a.getPKCEFlow(client) + pkceFlow, err := a.getPKCEFlow(client) if err != nil { // If PKCE not supported, try Device flow if s, ok := status.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) { - flow, err = a.getDeviceFlow(client) - return err + deviceFlow, err := a.getDeviceFlow(client) + if err != nil { + return err + } + deviceFlow.SetLoginHint(hint) + flow = deviceFlow + return nil } return err } + pkceFlow.SetLoginHint(hint) + flow = pkceFlow return nil }) diff --git a/client/internal/auth/oauth.go b/client/internal/auth/oauth.go index a50a2ce6f..91329c98b 100644 --- a/client/internal/auth/oauth.go +++ b/client/internal/auth/oauth.go @@ -97,9 +97,7 @@ func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config return nil, fmt.Errorf("getting pkce authorization flow info failed with error: %v", err) } - if hint != "" { - pkceFlowInfo.SetLoginHint(hint) - } + pkceFlowInfo.SetLoginHint(hint) return pkceFlowInfo, nil } @@ -127,9 +125,7 @@ func authenticateWithDeviceCodeFlow(ctx context.Context, config *profilemanager. } } - if hint != "" { - deviceFlowInfo.SetLoginHint(hint) - } + deviceFlowInfo.SetLoginHint(hint) return deviceFlowInfo, nil } diff --git a/client/internal/connect.go b/client/internal/connect.go index ceb39419e..e45ecca44 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -38,6 +38,8 @@ import ( "github.com/netbirdio/netbird/client/internal/updater" "github.com/netbirdio/netbird/client/internal/updater/installer" nbnet "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" cProto "github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/client/ssh" sshconfig "github.com/netbirdio/netbird/client/ssh/config" @@ -70,18 +72,42 @@ type ConnectClient struct { updateManager *updater.Manager persistSyncResponse bool + + // netState gates every reconnection loop on OS-reported network + // availability. Nil (the default) disables gating; mobile platforms + // inject it via WithNetworkState. + netState *netstate.State + + // sweeper cuts the management, signal and relay connections on network + // change; nil disables it. + sweeper *netsweep.Sweeper +} + +// ConnectClientOption configures optional ConnectClient behavior. +type ConnectClientOption func(*ConnectClient) + +// WithNetworkState injects the OS network availability state that gates every +// reconnection loop; without it gating is disabled. +func WithNetworkState(netState *netstate.State) ConnectClientOption { + return func(c *ConnectClient) { c.netState = netState } +} + +// WithSweeper injects the network change sweeper. +func WithSweeper(sweeper *netsweep.Sweeper) ConnectClientOption { + return func(c *ConnectClient) { c.sweeper = sweeper } } func NewConnectClient( ctx context.Context, config *profilemanager.Config, statusRecorder *peer.Status, + opts ...ConnectClientOption, ) *ConnectClient { // Derive the run context here so Stop owns the cancel that unblocks the run // loop. runCancel is set once at construction, so Stop can call it without // racing the run loop's startup. Callers therefore need not cancel before Stop. runCtx, runCancel := context.WithCancel(ctx) - return &ConnectClient{ + c := &ConnectClient{ ctx: runCtx, runCancel: runCancel, runExited: make(chan struct{}), @@ -89,6 +115,10 @@ func NewConnectClient( statusRecorder: statusRecorder, engineMutex: sync.Mutex{}, } + for _, opt := range opts { + opt(c) + } + return c } func (c *ConnectClient) SetUpdateManager(um *updater.Manager) { @@ -274,6 +304,13 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan return nil } + // suspend connection attempts while the OS reports no usable network + if waited, err := c.netState.Wait(c.ctx); err != nil { + return nil + } else if waited { + backOff.Reset() + } + state.Set(StatusConnecting) engineCtx, cancel := context.WithCancel(c.ctx) @@ -285,7 +322,8 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan }() log.Debugf("connecting to the Management service %s", c.config.ManagementURL.Host) - mgmClient, err := mgm.NewClient(engineCtx, c.config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled) + mgmClient, err := mgm.NewClient(engineCtx, c.config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled, + mgm.WithNetworkState(c.netState), mgm.WithSweeper(c.sweeper)) if err != nil { // On daemon shutdown / Down() the parent context is cancelled // and the dial fails with "context canceled". Wrapping that @@ -360,7 +398,7 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan }() // with the global Netbird config in hand connect (just a connection, no stream yet) Signal - signalClient, err := connectToSignal(engineCtx, loginResp.GetNetbirdConfig(), myPrivateKey) + signalClient, err := connectToSignal(engineCtx, loginResp.GetNetbirdConfig(), myPrivateKey, c.netState, c.sweeper) if err != nil { log.Error(err) return wrapErr(err) @@ -396,7 +434,8 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan engineConfig.StateDir = filepath.Dir(path) } - relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String(), engineConfig.MTU) + relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String(), engineConfig.MTU, + relayClient.WithNetworkState(c.netState), relayClient.WithSweeper(c.sweeper)) c.statusRecorder.SetRelayMgr(relayManager) if len(relayURLs) > 0 { if token != nil { @@ -424,6 +463,7 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan UpdateManager: c.updateManager, ClientMetrics: c.clientMetrics, MetricsCtx: c.ctx, + NetState: c.netState, }, mobileDependency) engine.SetSyncResponsePersistence(c.persistSyncResponse) c.engine = engine @@ -480,6 +520,16 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan // status stream stuck at Connecting. err = backoff.Retry(operation, backoff.WithContext(backOff, c.ctx)) if err != nil { + // Once the client context is cancelled backoff.WithContext surfaces the + // bare context error, and any attempt torn down mid-flight reports the + // same. That cancellation is the caller asking us to stop (Stop, Down or + // an engine restart), so exit cleanly instead of handing back a failure + // the caller would have to distinguish from a real one. + if c.ctx.Err() != nil && errors.Is(err, context.Canceled) { + log.Info("exiting client retry loop, context cancelled") + return nil + } + log.Debugf("exiting client retry loop due to unrecoverable error: %s", err) if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.PermissionDenied) { state.Set(StatusNeedsLogin) @@ -673,7 +723,7 @@ func selectMTU(localMTU uint16, peerMTU int32) uint16 { } // connectToSignal creates Signal Service client and established a connection -func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourPrivateKey wgtypes.Key) (*signal.GrpcClient, error) { +func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourPrivateKey wgtypes.Key, netState *netstate.State, sweeper *netsweep.Sweeper) (*signal.GrpcClient, error) { var sigTLSEnabled bool if wtConfig.Signal.Protocol == mgmProto.HostConfig_HTTPS { sigTLSEnabled = true @@ -681,7 +731,8 @@ func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourP sigTLSEnabled = false } - signalClient, err := signal.NewClient(ctx, wtConfig.Signal.Uri, ourPrivateKey, sigTLSEnabled) + signalClient, err := signal.NewClient(ctx, wtConfig.Signal.Uri, ourPrivateKey, sigTLSEnabled, + signal.WithNetworkState(netState), signal.WithSweeper(sweeper)) if err != nil { log.Errorf("error while connecting to the Signal Exchange Service %s: %s", wtConfig.Signal.Uri, err) return nil, gstatus.Errorf(codes.FailedPrecondition, "failed connecting to Signal Service : %s", err) diff --git a/client/internal/dns/host_windows.go b/client/internal/dns/host_windows.go index d20fdd1d6..2852dddb9 100644 --- a/client/internal/dns/host_windows.go +++ b/client/internal/dns/host_windows.go @@ -35,6 +35,8 @@ var ( // exported so a diagnostic reader reports the same locations that are written. const ( // NRPTKeyPrefix starts the name of every NRPT rule key this client creates. + // Older versions used different layouts under the same prefix: a single + // unsuffixed key, then one key per domain, now one key per batch of domains. NRPTKeyPrefix = "NetBird-Match" // DNSPolicyConfigRoot holds the NRPT rules of the local policy store. @@ -89,7 +91,6 @@ type registryConfigurator struct { guid string routingAll bool gpo bool - nrptEntryCount int origNameservers []netip.Addr } @@ -322,14 +323,9 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager } if len(matchDomains) != 0 { - count, err := r.addDNSMatchPolicy(matchDomains, config.ServerIP) - // Update count even on error to ensure cleanup covers partially created rules - r.nrptEntryCount = count - if err != nil { + if err := r.addDNSMatchPolicy(matchDomains, config.ServerIP); err != nil { return fmt.Errorf("add dns match policy: %w", err) } - } else { - r.nrptEntryCount = 0 } r.updateState(stateManager) @@ -345,9 +341,8 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager func (r *registryConfigurator) updateState(stateManager *statemanager.Manager) { if err := stateManager.UpdateState(&ShutdownState{ - Guid: r.guid, - GPO: r.gpo, - NRPTEntryCount: r.nrptEntryCount, + Guid: r.guid, + GPO: r.gpo, }); err != nil { log.Errorf("failed to update shutdown state: %s", err) } @@ -362,7 +357,7 @@ func (r *registryConfigurator) addDNSSetupForAll(ip netip.Addr) error { return nil } -func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr) (int, error) { +func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr) error { // if the gpo key is present, we need to put our DNS settings there, otherwise our config might be ignored // see https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-gpnrpt/8cc31cb9-20cb-4140-9e85-3e08703b4745 @@ -379,19 +374,17 @@ func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr gpoPath := fmt.Sprintf("%s-%d", gpoDnsPolicyConfigMatchPath, ruleIndex) if err := r.configureDNSPolicy(localPath, batchDomains, ip); err != nil { - return ruleIndex, fmt.Errorf("configure DNS Local policy for rule %d: %w", ruleIndex, err) + return fmt.Errorf("configure DNS Local policy for rule %d: %w", ruleIndex, err) } - // Increment immediately so the caller's cleanup path knows about this rule - ruleIndex++ - if r.gpo { if err := r.configureDNSPolicy(gpoPath, batchDomains, ip); err != nil { - return ruleIndex, fmt.Errorf("configure gpo DNS policy for rule %d: %w", ruleIndex-1, err) + return fmt.Errorf("configure gpo DNS policy for rule %d: %w", ruleIndex, err) } } - log.Debugf("added NRPT rule %d with %d domains", ruleIndex-1, len(batchDomains)) + log.Debugf("added NRPT rule %d with %d domains", ruleIndex, len(batchDomains)) + ruleIndex++ } if r.gpo { @@ -401,7 +394,7 @@ func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr } log.Infof("added %d NRPT rules for %d domains", ruleIndex, len(domains)) - return ruleIndex, nil + return nil } func (r *registryConfigurator) configureDNSPolicy(policyPath string, domains []string, ip netip.Addr) error { @@ -534,28 +527,28 @@ func (r *registryConfigurator) restoreHostDNS() error { return nil } +// removeDNSMatchPolicies deletes every NRPT rule this client may have created, +// from the local and the GPO policy store. The rules are found by enumerating +// the registry, the only authoritative record of what was written. Cleanup must +// not depend on a rule count: the in-memory one is scoped to a single +// registryConfigurator and the persisted one is deleted on every clean +// disconnect, and a rule left behind keeps resolving names over an interface +// that is gone, until reboot discards the volatile key. func (r *registryConfigurator) removeDNSMatchPolicies() error { var merr *multierror.Error - // Try to remove the base entries (for backward compatibility) - if err := removeRegistryKeyFromDNSPolicyConfig(dnsPolicyConfigMatchPath); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove local base entry: %w", err)) - } - - if err := removeRegistryKeyFromDNSPolicyConfig(gpoDnsPolicyConfigMatchPath); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove GPO base entry: %w", err)) - } - - for i := 0; i < r.nrptEntryCount; i++ { - localPath := fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i) - gpoPath := fmt.Sprintf("%s-%d", gpoDnsPolicyConfigMatchPath, i) - - if err := removeRegistryKeyFromDNSPolicyConfig(localPath); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove local entry %d: %w", i, err)) + for _, root := range []string{DNSPolicyConfigRoot, GPODNSPolicyConfigRoot} { + names, err := listNRPTRuleKeys(root) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("list rule keys under %s: %w", root, err)) + continue } - if err := removeRegistryKeyFromDNSPolicyConfig(gpoPath); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove GPO entry %d: %w", i, err)) + for _, name := range names { + path := root + `\` + name + if err := removeRegistryKeyFromDNSPolicyConfig(path); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove entry %s: %w", path, err)) + } } } @@ -570,6 +563,39 @@ func (r *registryConfigurator) restoreUncleanShutdownDNS() error { return r.restoreHostDNS() } +// listNRPTRuleKeys returns the names of our NRPT rule keys under a policy store +// root. An absent root holds nothing to clean up, which is the normal state of +// the GPO store on a machine without DNS Client policy. +func listNRPTRuleKeys(root string) ([]string, error) { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, root, registry.ENUMERATE_SUB_KEYS) + switch { + case errors.Is(err, registry.ErrNotExist), errors.Is(err, syscall.ERROR_PATH_NOT_FOUND): + // the GPO store is absent on a machine without DNS client policy + log.Debugf("HKEY_LOCAL_MACHINE\\%s does not exist", root) + return nil, nil + case err != nil: + // any other failure has to reach the caller: reporting no rules would + // report a successful cleanup while leaving the rules in place + return nil, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", root, err) + } + defer closer(k) + + names, err := k.ReadSubKeyNames(-1) + if err != nil { + return nil, fmt.Errorf("read subkey names: %w", err) + } + + var ruleKeys []string + for _, name := range names { + // registry key names are case insensitive + if strings.HasPrefix(strings.ToLower(name), strings.ToLower(NRPTKeyPrefix)) { + ruleKeys = append(ruleKeys, name) + } + } + + return ruleKeys, nil +} + func removeRegistryKeyFromDNSPolicyConfig(regKeyPath string) error { k, err := registry.OpenKey(registry.LOCAL_MACHINE, regKeyPath, registry.QUERY_VALUE) if err != nil { diff --git a/client/internal/dns/host_windows_test.go b/client/internal/dns/host_windows_test.go index 3cd2b1bd5..861613c95 100644 --- a/client/internal/dns/host_windows_test.go +++ b/client/internal/dns/host_windows_test.go @@ -25,7 +25,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) { // Create a test interface registry key so updateSearchDomains doesn't fail testGUID := "{12345678-1234-1234-1234-123456789ABC}" - interfacePath := `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\` + testGUID + interfacePath := InterfaceConfigPath + `\` + testGUID testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE) require.NoError(t, err, "Should create test interface registry key") testKey.Close() @@ -56,7 +56,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) { require.NoError(t, err) // Verify 3 NRPT rules exist - assert.Equal(t, 3, cfg.nrptEntryCount, "Should create 3 NRPT rules for 125 domains") + assert.Equal(t, 3, countNRPTRuleKeys(t), "Should create 3 NRPT rules for 125 domains") for i := 0; i < 3; i++ { exists, err := registryKeyExists(fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i)) require.NoError(t, err) @@ -81,7 +81,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) { require.NoError(t, err) // Verify first 2 NRPT rules exist - assert.Equal(t, 2, cfg.nrptEntryCount, "Should create 2 NRPT rules for 75 domains") + assert.Equal(t, 2, countNRPTRuleKeys(t), "Should create 2 NRPT rules for 75 domains") for i := 0; i < 2; i++ { exists, err := registryKeyExists(fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i)) require.NoError(t, err) @@ -106,9 +106,65 @@ func registryKeyExists(path string) (bool, error) { return true, nil } +// TestNRPTCleanupWithoutRuleCount verifies that rules written by a previous run +// are removed by a configurator that has no record of how many there are: an +// unclean exit loses the in-memory count and a clean disconnect deletes the +// persisted one, so cleanup cannot depend on either. +func TestNRPTCleanupWithoutRuleCount(t *testing.T) { + if testing.Short() { + t.Skip("skipping registry integration test in short mode") + } + + defer cleanupRegistryKeys(t) + cleanupRegistryKeys(t) + + testIP := netip.MustParseAddr("100.64.0.1") + + // 75 domains produce two indexed rules, as the current layout does + domains := make([]string, 75) + for i := range domains { + domains[i] = fmt.Sprintf(".domain%d.com", i+1) + } + + previousRun := ®istryConfigurator{} + require.NoError(t, previousRun.addDNSMatchPolicy(domains, testIP)) + + // the unsuffixed key an older version would have written + require.NoError(t, previousRun.configureDNSPolicy(dnsPolicyConfigMatchPath, []string{".legacy.example.com"}, testIP)) + + // a policy owned by someone else, which cleanup must not touch + foreignPath := DNSPolicyConfigRoot + `\DnsPolicyConfigTestForeign` + foreignKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, foreignPath, registry.SET_VALUE) + require.NoError(t, err, "Should create foreign policy key") + foreignKey.Close() + defer func() { + _ = registry.DeleteKey(registry.LOCAL_MACHINE, foreignPath) + }() + + require.Equal(t, 3, countNRPTRuleKeys(t), "Should have two indexed rules and the legacy one") + + // a configurator that never applied a DNS config, as one built after a + // restart or from a shutdown state without a count is + freshRun := ®istryConfigurator{} + require.NoError(t, freshRun.removeDNSMatchPolicies()) + + assert.Equal(t, 0, countNRPTRuleKeys(t), "Should remove every rule left by the previous run") + + exists, err := registryKeyExists(foreignPath) + require.NoError(t, err) + assert.True(t, exists, "Should not remove a policy that is not ours") +} + +func countNRPTRuleKeys(t *testing.T) int { + t.Helper() + + names, err := listNRPTRuleKeys(DNSPolicyConfigRoot) + require.NoError(t, err, "Should list NRPT rule keys") + return len(names) +} + func cleanupRegistryKeys(*testing.T) { - // Clean up more entries to account for batching tests with many domains - cfg := ®istryConfigurator{nrptEntryCount: 20} + cfg := ®istryConfigurator{} _ = cfg.removeDNSMatchPolicies() } @@ -125,7 +181,7 @@ func TestNRPTDomainBatching(t *testing.T) { // Create a test interface registry key so updateSearchDomains doesn't fail testGUID := "{12345678-1234-1234-1234-123456789ABC}" - interfacePath := `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\` + testGUID + interfacePath := InterfaceConfigPath + `\` + testGUID testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE) require.NoError(t, err, "Should create test interface registry key") testKey.Close() @@ -193,7 +249,7 @@ func TestNRPTDomainBatching(t *testing.T) { require.NoError(t, err) // Verify that exactly expectedRuleCount rules were created - assert.Equal(t, tc.expectedRuleCount, cfg.nrptEntryCount, + assert.Equal(t, tc.expectedRuleCount, countNRPTRuleKeys(t), "Should create %d NRPT rules for %d domains", tc.expectedRuleCount, tc.domainCount) // Verify all expected rules exist diff --git a/client/internal/dns/unclean_shutdown_windows.go b/client/internal/dns/unclean_shutdown_windows.go index 24a9eca50..ab0b2cc63 100644 --- a/client/internal/dns/unclean_shutdown_windows.go +++ b/client/internal/dns/unclean_shutdown_windows.go @@ -5,9 +5,8 @@ import ( ) type ShutdownState struct { - Guid string - GPO bool - NRPTEntryCount int + Guid string + GPO bool } func (s *ShutdownState) Name() string { @@ -16,9 +15,8 @@ func (s *ShutdownState) Name() string { func (s *ShutdownState) Cleanup() error { manager := ®istryConfigurator{ - guid: s.Guid, - gpo: s.GPO, - nrptEntryCount: s.NRPTEntryCount, + guid: s.Guid, + gpo: s.GPO, } if err := manager.restoreUncleanShutdownDNS(); err != nil { diff --git a/client/internal/ebpf/ebpf/manager_linux.go b/client/internal/ebpf/ebpf/manager_linux.go index 7520a6387..64a3e5b54 100644 --- a/client/internal/ebpf/ebpf/manager_linux.go +++ b/client/internal/ebpf/ebpf/manager_linux.go @@ -2,17 +2,21 @@ package ebpf import ( _ "embed" + "fmt" "net" "sync" "github.com/cilium/ebpf/link" "github.com/cilium/ebpf/rlimit" log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" "github.com/netbirdio/netbird/client/internal/ebpf/manager" ) const ( + xdpProgName = "nb_xdp_prog" + mapKeyFeatures uint32 = 0 featureFlagWGProxy = 0b00000001 @@ -68,21 +72,50 @@ func (tf *GeneralManager) loadXdp() error { return err } - // load pre-compiled programs into the kernel. - err = loadBpfObjects(&tf.bpfObjs, nil) + // lo has no native XDP, so the program runs in generic mode. Unless it + // declares multi-buffer support the kernel must linearize every non-linear + // skb before running it. Loopback packets are up to 64 KB, so that is a + // contiguous GFP_ATOMIC allocation per packet, and when it fails the packet + // is dropped before the program runs, stalling local TCP connections. + // Multi-buffer XDP in generic mode requires kernel 6.3, so fall back to a + // plain attach when the kernel rejects it. + err = tf.attachXdp(iFace.Index, true) + if err == nil { + return nil + } + log.Debugf("failed to attach multi-buffer xdp program, retrying without it: %s", err) + + return tf.attachXdp(iFace.Index, false) +} + +func (tf *GeneralManager) attachXdp(iFaceIndex int, multiBuffer bool) error { + spec, err := loadBpf() if err != nil { - return err + return fmt.Errorf("load bpf spec: %w", err) + } + + if multiBuffer { + prog, ok := spec.Programs[xdpProgName] + if !ok { + return fmt.Errorf("program %s not found in bpf spec", xdpProgName) + } + prog.Flags |= unix.BPF_F_XDP_HAS_FRAGS + } + + if err := spec.LoadAndAssign(&tf.bpfObjs, nil); err != nil { + return fmt.Errorf("load bpf objects: %w", err) } tf.link, err = link.AttachXDP(link.XDPOptions{ Program: tf.bpfObjs.NbXdpProg, - Interface: iFace.Index, + Interface: iFaceIndex, }) - if err != nil { - _ = tf.bpfObjs.Close() + if closeErr := tf.bpfObjs.Close(); closeErr != nil { + log.Debugf("failed to close bpf objects after xdp attach error: %s", closeErr) + } tf.link = nil - return err + return fmt.Errorf("attach xdp: %w", err) } return nil } diff --git a/client/internal/engine.go b/client/internal/engine.go index d92c360f2..5380651a5 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -59,6 +59,7 @@ import ( "github.com/netbirdio/netbird/client/internal/syncstore" "github.com/netbirdio/netbird/client/internal/updater" "github.com/netbirdio/netbird/client/jobexec" + "github.com/netbirdio/netbird/client/netstate" cProto "github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/client/system" nbdns "github.com/netbirdio/netbird/dns" @@ -181,6 +182,9 @@ type EngineServices struct { UpdateManager *updater.Manager ClientMetrics *metrics.ClientMetrics MetricsCtx context.Context + // NetState gates the reconnection loops on OS-reported network + // availability; nil disables gating. + NetState *netstate.State } // Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers. @@ -204,6 +208,10 @@ type Engine struct { config *EngineConfig mobileDep MobileDependency + // netState gates the peer reconnection guards on OS-reported network + // availability; nil disables gating. + netState *netstate.State + // STUNs is a list of STUN servers used by ICE STUNs []*stun.URI // TURNs is a list of STUN servers used by ICE @@ -337,6 +345,7 @@ func NewEngine( syncMsgMux: &sync.Mutex{}, config: config, mobileDep: mobileDep, + netState: services.NetState, STUNs: []*stun.URI{}, TURNs: []*stun.URI{}, networkSerial: 0, @@ -1893,7 +1902,8 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs []netip.Prefix, agentV Addr: e.getRosenpassAddr(), PermissiveMode: e.config.RosenpassPermissive, }, - ICEConfig: e.createICEConfig(), + ICEConfig: e.createICEConfig(), + NetworkState: e.netState, } serviceDependencies := peer.ServiceDependencies{ diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index f3235ec7f..a3c320027 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -26,6 +26,7 @@ import ( "github.com/netbirdio/netbird/client/internal/portforward" "github.com/netbirdio/netbird/client/internal/rosenpass" "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/client/netstate" "github.com/netbirdio/netbird/route" relayClient "github.com/netbirdio/netbird/shared/relay/client" ) @@ -93,6 +94,10 @@ type ConnConfig struct { // ICEConfig ICE protocol configuration ICEConfig icemaker.Config + + // NetworkState gates the reconnection guard on OS-reported network + // availability; nil disables gating. + NetworkState *netstate.State } type Conn struct { @@ -254,7 +259,7 @@ func (conn *Conn) open(engineCtx context.Context, firstPacket []byte) error { conn.handshaker.AddICEListener(conn.workerICE.OnNewOffer) } - conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher) + conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher, conn.config.NetworkState) conn.wg.Add(1) go func() { diff --git a/client/internal/peer/guard/guard.go b/client/internal/peer/guard/guard.go index 6c2e846a9..68d77d318 100644 --- a/client/internal/peer/guard/guard.go +++ b/client/internal/peer/guard/guard.go @@ -6,6 +6,8 @@ import ( "github.com/cenkalti/backoff/v4" log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/netstate" ) // ConnStatus represents the connection state as seen by the guard. @@ -31,20 +33,26 @@ type connStatusFunc func() ConnStatus // - Relayed connection disconnected // - ICE candidate changes type Guard struct { - log *log.Entry - isConnectedOnAllWay connStatusFunc - timeout time.Duration - srWatcher *SRWatcher + log *log.Entry + isConnectedOnAllWay connStatusFunc + timeout time.Duration + srWatcher *SRWatcher + // netState gates reconnect attempts on OS-reported network availability; + // nil disables gating. + netState *netstate.State relayedConnDisconnected chan struct{} iCEConnDisconnected chan struct{} } -func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher) *Guard { +// NewGuard creates a reconnection guard for a peer connection. A nil netState +// disables network availability gating. +func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher, netState *netstate.State) *Guard { return &Guard{ log: log, isConnectedOnAllWay: isConnectedFn, timeout: timeout, srWatcher: srWatcher, + netState: netState, relayedConnDisconnected: make(chan struct{}, 1), iCEConnDisconnected: make(chan struct{}, 1), } @@ -96,9 +104,16 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) { iceState := &iceRetryState{log: g.log} defer iceState.reset() + netChanged := g.netState.Changed() + for { select { case <-tickerChannel: + // skip attempts while the OS reports no usable network; the + // netChanged case below resumes the loop once it returns + if !g.netState.IsOnline() { + continue + } switch g.isConnectedOnAllWay() { case ConnStatusConnected: // all good, nothing to do @@ -135,6 +150,23 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) { tickerChannel = ticker.C iceState.reset() + case <-netChanged: + // Re-arm for the next transition before acting on this one. + netChanged = g.netState.Changed() + if !g.netState.IsOnline() { + continue + } + // Ticks skipped while offline drove the backoff towards its + // maximum without ever attempting, and left the ICE budget + // frozen — possibly in hourly mode. Recover on our own so the + // peer does not depend on a signal or relay event that never + // comes when both stayed up across the outage. + g.log.Debugf("network is back, reset reconnection ticker") + ticker.Stop() + ticker = g.newReconnectTicker(ctx) + tickerChannel = ticker.C + iceState.reset() + case <-ctx.Done(): g.log.Debugf("context is done, stop reconnect loop") return diff --git a/client/internal/peer/guard/guard_leak_test.go b/client/internal/peer/guard/guard_leak_test.go index ded3e4aea..3d82ec591 100644 --- a/client/internal/peer/guard/guard_leak_test.go +++ b/client/internal/peer/guard/guard_leak_test.go @@ -15,7 +15,7 @@ import ( func newTestGuard(status connStatusFunc) *Guard { srw := NewSRWatcher(nil, nil, nil, ice.Config{}) - return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw) + return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw, nil) } // countBackoffTickerGoroutines returns how many goroutines are currently sitting diff --git a/client/internal/peer/guard/guard_netstate_test.go b/client/internal/peer/guard/guard_netstate_test.go new file mode 100644 index 000000000..2ab736428 --- /dev/null +++ b/client/internal/peer/guard/guard_netstate_test.go @@ -0,0 +1,107 @@ +package guard + +import ( + "context" + "sync/atomic" + "testing" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/peer/ice" + "github.com/netbirdio/netbird/client/netstate" +) + +// newTestGuardWithNetState builds a guard with a realistic MaxInterval: the +// backoff must be able to grow well past the outage, as it does in production +// where the timeout is seconds to minutes. +func newTestGuardWithNetState(status connStatusFunc, netState *netstate.State) *Guard { + srw := NewSRWatcher(nil, nil, nil, ice.Config{}) + return NewGuard(log.WithField("test", "guard"), status, 30*time.Second, srw, netState) +} + +// TestGuard_RecoversAfterOfflineToOnline covers a peer that stays disconnected +// across a network outage while neither signal nor relay reports an event — +// both stayed up, as on a short airplane mode toggle over Wi-Fi. +// +// Every tick taken while offline is skipped, but it still advances the +// exponential backoff, so by the time the network returns the next tick can be +// tens of seconds away. Without an explicit reaction to the transition the +// peer waits out that interval for a recovery that could start immediately. +func TestGuard_RecoversAfterOfflineToOnline(t *testing.T) { + netState := netstate.New() + + var attempts atomic.Int32 + g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Start from the reconnect ticker (800ms initial interval), the state a + // peer is in after it loses its connection. + go g.Start(ctx, func() { attempts.Add(1) }) + g.SetRelayedConnDisconnected() + + // Let the backoff climb: 0.8s, 1.6s, 3.2s, 6.4s ... every tick is skipped + // while offline, but each one doubles the wait for the next. + netState.Set(false) + time.Sleep(8 * time.Second) + + offlineAttempts := attempts.Load() + if offlineAttempts != 0 { + t.Fatalf("callback ran %d times while offline, want 0", offlineAttempts) + } + + netState.Set(true) + + // The next organic tick is now several seconds out, so anything within + // this window can only come from reacting to the transition itself. + pollCtx, stopPolling := context.WithTimeout(ctx, 2*time.Second) + defer stopPolling() + + select { + case <-pollCtx.Done(): + t.Fatal("peer was not retried within 2s of the network coming back, " + + "with neither a signal nor a relay event to fall back on") + case <-pollUntil(pollCtx, func() bool { return attempts.Load() > 0 }): + } +} + +// TestGuard_OfflineTransitionDoesNotRetry checks the other direction: going +// offline must not itself trigger an attempt. +func TestGuard_OfflineTransitionDoesNotRetry(t *testing.T) { + netState := netstate.New() + + var attempts atomic.Int32 + g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go g.Start(ctx, func() { attempts.Add(1) }) + + netState.Set(false) + time.Sleep(5 * time.Second) + + if got := attempts.Load(); got != 0 { + t.Fatalf("callback ran %d times after going offline, want 0", got) + } +} + +// pollUntil closes the returned channel once cond holds. It gives up when ctx +// is done, so the polling goroutine never outlives the test that started it. +func pollUntil(ctx context.Context, cond func() bool) <-chan struct{} { + done := make(chan struct{}) + go func() { + for { + if cond() { + close(done) + return + } + select { + case <-ctx.Done(): + return + case <-time.After(10 * time.Millisecond): + } + } + }() + return done +} diff --git a/client/internal/peer/listener.go b/client/internal/peer/listener.go index c601fe534..2bb7fcf32 100644 --- a/client/internal/peer/listener.go +++ b/client/internal/peer/listener.go @@ -1,11 +1,40 @@ package peer +// ClientState identifies the client connection state delivered via +// Listener.OnStateChanged. +type ClientState int + +// Client states. The numeric values cross the gomobile boundary (the mobile +// bindings re-export them as integer constants), so they are a wire format: +// append new states at the end, never reorder or insert. +const ( + ClientStateDisconnected ClientState = iota + ClientStateConnected + ClientStateConnecting + ClientStateDisconnecting + // ClientStateNoNetwork is an overlay state: it is never stored as the + // last notification, only derived from ClientStateConnecting while the + // OS reports no usable network (see notifier.effectiveState). + ClientStateNoNetwork +) + // Listener is a callback type about the NetBird network connection state type Listener interface { + // OnStateChanged reports every client state transition. New states are + // delivered only through this callback; the per-state callbacks below + // are kept for compatibility and will be removed once all consumers + // have migrated. + OnStateChanged(state ClientState) + + // Deprecated: consume OnStateChanged instead. OnConnected() + // Deprecated: consume OnStateChanged instead. OnDisconnected() + // Deprecated: consume OnStateChanged instead. OnConnecting() + // Deprecated: consume OnStateChanged instead. OnDisconnecting() + OnAddressChanged(string, string) OnPeersListChanged(int) } diff --git a/client/internal/peer/notifier.go b/client/internal/peer/notifier.go index 8d1954fe5..1ee1d32ea 100644 --- a/client/internal/peer/notifier.go +++ b/client/internal/peer/notifier.go @@ -4,31 +4,64 @@ import ( "sync" ) -const ( - stateDisconnected = iota - stateConnected - stateConnecting - stateDisconnecting -) - type notifier struct { + // publishLock orders state publication: it is held across computing the + // effective state and handing it to the listener, so a transition cannot + // overtake a newer one and leave the listener on a stale state. + publishLock sync.Mutex serverStateLock sync.Mutex listenersLock sync.Mutex listener Listener currentClientState bool - lastNotification int + lastNotification ClientState lastNumberOfPeers int lastFqdnAddress string lastIPAddress string + networkAvailable bool } func newNotifier() *notifier { - return ¬ifier{} + return ¬ifier{ + networkAvailable: true, + } +} + +// effectiveState maps the computed state to what listeners should see: +// while the OS reports no usable network, "Connecting" would be a lie — +// connection attempts are suspended — so it is reported as NoNetwork. +// Caller must hold serverStateLock. +func (n *notifier) effectiveState(state ClientState) ClientState { + if !n.networkAvailable && state == ClientStateConnecting { + return ClientStateNoNetwork + } + return state +} + +// setNetworkAvailable records the OS network availability and re-notifies +// the listener when the flag flips the effective state (Connecting <-> +// NoNetwork). +func (n *notifier) setNetworkAvailable(available bool) { + n.publishLock.Lock() + defer n.publishLock.Unlock() + + n.serverStateLock.Lock() + if n.networkAvailable == available { + n.serverStateLock.Unlock() + return + } + previous := n.effectiveState(n.lastNotification) + n.networkAvailable = available + current := n.effectiveState(n.lastNotification) + n.serverStateLock.Unlock() + + if previous != current { + n.notify(current) + } } func (n *notifier) setListener(listener Listener) { n.serverStateLock.Lock() - lastNotification := n.lastNotification + lastNotification := n.effectiveState(n.lastNotification) numOfPeers := n.lastNumberOfPeers fqdnAddress := n.lastFqdnAddress address := n.lastIPAddress @@ -52,6 +85,9 @@ func (n *notifier) removeListener() { } func (n *notifier) updateServerStates(mgmState bool, signalState bool) { + n.publishLock.Lock() + defer n.publishLock.Unlock() + n.serverStateLock.Lock() calculatedState := n.calculateState(mgmState, signalState) @@ -61,43 +97,54 @@ func (n *notifier) updateServerStates(mgmState bool, signalState bool) { } n.lastNotification = calculatedState + effective := n.effectiveState(calculatedState) n.serverStateLock.Unlock() - n.notify(calculatedState) + n.notify(effective) } func (n *notifier) clientStart() { + n.publishLock.Lock() + defer n.publishLock.Unlock() + n.serverStateLock.Lock() n.currentClientState = true - n.lastNotification = stateConnecting + n.lastNotification = ClientStateConnecting + effective := n.effectiveState(ClientStateConnecting) n.serverStateLock.Unlock() - n.notify(stateConnecting) + n.notify(effective) } func (n *notifier) clientStop() { + n.publishLock.Lock() + defer n.publishLock.Unlock() + n.serverStateLock.Lock() n.currentClientState = false - n.lastNotification = stateDisconnected + n.lastNotification = ClientStateDisconnected n.serverStateLock.Unlock() - n.notify(stateDisconnected) + n.notify(ClientStateDisconnected) } func (n *notifier) clientTearDown() { + n.publishLock.Lock() + defer n.publishLock.Unlock() + n.serverStateLock.Lock() n.currentClientState = false - n.lastNotification = stateDisconnecting + n.lastNotification = ClientStateDisconnecting n.serverStateLock.Unlock() - n.notify(stateDisconnecting) + n.notify(ClientStateDisconnecting) } -func (n *notifier) isServerStateChanged(newState int) bool { +func (n *notifier) isServerStateChanged(newState ClientState) bool { return n.lastNotification != newState } -func (n *notifier) notify(state int) { +func (n *notifier) notify(state ClientState) { n.listenersLock.Lock() listener := n.listener n.listenersLock.Unlock() @@ -109,20 +156,20 @@ func (n *notifier) notify(state int) { notifyListener(listener, state) } -func (n *notifier) calculateState(managementConn, signalConn bool) int { +func (n *notifier) calculateState(managementConn, signalConn bool) ClientState { if managementConn && signalConn { - return stateConnected + return ClientStateConnected } if !managementConn && !signalConn && !n.currentClientState { - return stateDisconnected + return ClientStateDisconnected } - if n.lastNotification == stateDisconnecting { - return stateDisconnecting + if n.lastNotification == ClientStateDisconnecting { + return ClientStateDisconnecting } - return stateConnecting + return ClientStateConnecting } func (n *notifier) peerListChanged(numOfPeers int) { @@ -159,15 +206,19 @@ func (n *notifier) localAddressChanged(fqdn, address string) { listener.OnAddressChanged(fqdn, address) } -func notifyListener(l Listener, state int) { +func notifyListener(l Listener, state ClientState) { + // legacy per-state callbacks; NoNetwork is delivered only via + // OnStateChanged below switch state { - case stateDisconnected: + case ClientStateDisconnected: l.OnDisconnected() - case stateConnected: + case ClientStateConnected: l.OnConnected() - case stateConnecting: + case ClientStateConnecting: l.OnConnecting() - case stateDisconnecting: + case ClientStateDisconnecting: l.OnDisconnecting() } + + l.OnStateChanged(state) } diff --git a/client/internal/peer/notifier_concurrent_test.go b/client/internal/peer/notifier_concurrent_test.go new file mode 100644 index 000000000..fcaaaad3b --- /dev/null +++ b/client/internal/peer/notifier_concurrent_test.go @@ -0,0 +1,108 @@ +package peer + +import ( + "sync" + "testing" + "time" +) + +type recordingListener struct { + mu sync.Mutex + states []ClientState + onState func(ClientState) +} + +func (l *recordingListener) OnStateChanged(state ClientState) { + l.mu.Lock() + l.states = append(l.states, state) + hook := l.onState + l.mu.Unlock() + + if hook != nil { + hook(state) + } +} + +func (l *recordingListener) last() (ClientState, bool) { + l.mu.Lock() + defer l.mu.Unlock() + if len(l.states) == 0 { + return 0, false + } + return l.states[len(l.states)-1], true +} + +func (l *recordingListener) snapshot() []ClientState { + l.mu.Lock() + defer l.mu.Unlock() + return append([]ClientState(nil), l.states...) +} + +func (l *recordingListener) OnConnected() {} +func (l *recordingListener) OnDisconnected() {} +func (l *recordingListener) OnConnecting() {} +func (l *recordingListener) OnDisconnecting() {} +func (l *recordingListener) OnAddressChanged(string, string) {} +func (l *recordingListener) OnPeersListChanged(int) {} + +// TestNotifier_ConcurrentAvailabilityFlipOrdersPublication holds the first +// transition inside the listener callback and flips availability again from +// another goroutine while it is parked. The second flip must not publish +// ahead of the one in flight, otherwise the listener ends up on a state the +// notifier already superseded. +func TestNotifier_ConcurrentAvailabilityFlipOrdersPublication(t *testing.T) { + n := newNotifier() + n.currentClientState = true + n.lastNotification = ClientStateConnecting + + entered := make(chan struct{}) + release := make(chan struct{}) + + l := &recordingListener{} + l.onState = func(state ClientState) { + if state != ClientStateNoNetwork { + return + } + l.mu.Lock() + l.onState = nil + l.mu.Unlock() + close(entered) + <-release + } + n.listener = l + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + n.setNetworkAvailable(false) + }() + + <-entered + + flipped := make(chan struct{}) + go func() { + defer close(flipped) + n.setNetworkAvailable(true) + }() + + select { + case <-flipped: + t.Fatal("the online transition published while the offline one was " + + "still in flight; publication is not serialized") + case <-time.After(200 * time.Millisecond): + } + + close(release) + <-flipped + wg.Wait() + + got, ok := l.last() + if !ok { + t.Fatal("listener never observed a state") + } + if got != ClientStateConnecting { + t.Fatalf("listener holds %v after the network came back, want Connecting; sequence: %v", + got, l.snapshot()) + } +} diff --git a/client/internal/peer/notifier_test.go b/client/internal/peer/notifier_test.go index 0b7722b0c..a73016b05 100644 --- a/client/internal/peer/notifier_test.go +++ b/client/internal/peer/notifier_test.go @@ -6,29 +6,32 @@ import ( ) type mocListener struct { - lastState int + lastState ClientState wg sync.WaitGroup peersWg sync.WaitGroup peers int } func (l *mocListener) OnConnected() { - l.lastState = stateConnected + l.lastState = ClientStateConnected l.wg.Done() } func (l *mocListener) OnDisconnected() { - l.lastState = stateDisconnected + l.lastState = ClientStateDisconnected l.wg.Done() } func (l *mocListener) OnConnecting() { - l.lastState = stateConnecting + l.lastState = ClientStateConnecting l.wg.Done() } func (l *mocListener) OnDisconnecting() { - l.lastState = stateDisconnecting + l.lastState = ClientStateDisconnecting l.wg.Done() } +func (l *mocListener) OnStateChanged(state ClientState) { + +} func (l *mocListener) OnAddressChanged(host, addr string) { } @@ -57,15 +60,15 @@ func Test_notifier_serverState(t *testing.T) { type scenario struct { name string - expected int + expected ClientState mgmState bool signalState bool } scenarios := []scenario{ - {"connected", stateConnected, true, true}, - {"mgm down", stateConnecting, false, true}, - {"signal down", stateConnecting, true, false}, - {"disconnected", stateDisconnected, false, false}, + {"connected", ClientStateConnected, true, true}, + {"mgm down", ClientStateConnecting, false, true}, + {"signal down", ClientStateConnecting, true, false}, + {"disconnected", ClientStateDisconnected, false, false}, } for _, tt := range scenarios { @@ -85,7 +88,7 @@ func Test_notifier_SetListener(t *testing.T) { listener.setPeersWaiter() n := newNotifier() - n.lastNotification = stateConnecting + n.lastNotification = ClientStateConnecting n.setListener(listener) listener.wait() listener.waitPeers() @@ -99,7 +102,7 @@ func Test_notifier_RemoveListener(t *testing.T) { listener.setWaiter() listener.setPeersWaiter() n := newNotifier() - n.lastNotification = stateConnecting + n.lastNotification = ClientStateConnecting n.setListener(listener) // setListener replays cached state on a goroutine; wait for both the state // and peers callbacks to finish so we don't race on listener.peers. diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 423ce9b23..24e3e7fac 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -1211,6 +1211,12 @@ func (d *Status) ClientTeardown() { d.notifyStateChange() } +// SetNetworkAvailable records the OS-reported network availability; while +// unavailable, listeners see NoNetwork instead of Connecting. +func (d *Status) SetNetworkAvailable(available bool) { + d.notifier.setNetworkAvailable(available) +} + // SetConnectionListener set a listener to the notifier func (d *Status) SetConnectionListener(listener Listener) { d.notifier.setListener(listener) diff --git a/client/internal/profilemanager/prefs.go b/client/internal/profilemanager/prefs.go new file mode 100644 index 000000000..5613b0be3 --- /dev/null +++ b/client/internal/profilemanager/prefs.go @@ -0,0 +1,130 @@ +package profilemanager + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/netbirdio/netbird/util" +) + +const prefsFileSuffix = ".prefs.json" + +var prefsMu sync.Mutex + +// Prefs is a namespaced per-profile preference store backed by a single JSON +// file next to the profile config; it is deleted together with the profile. +type Prefs struct { + path string +} + +// ProfilePrefs returns the preference store of the profile identified by id. +func (s *ServiceManager) ProfilePrefs(id ID, username string) (*Prefs, error) { + if !IsValidProfileFilenameStem(id) { + return nil, fmt.Errorf("invalid profile ID: %q", id) + } + if id == defaultProfileName { + return &Prefs{path: filepath.Join(filepath.Dir(DefaultConfigPath), id.String()+prefsFileSuffix)}, nil + } + configDir, err := s.getConfigDir(username) + if err != nil { + return nil, fmt.Errorf("get config directory for user %s: %w", username, err) + } + return &Prefs{path: filepath.Join(configDir, id.String()+prefsFileSuffix)}, nil +} + +// Get unmarshals the namespace section into v and reports whether it exists. +func (p *Prefs) Get(namespace string, v any) (bool, error) { + if namespace == "" { + return false, fmt.Errorf("empty prefs namespace") + } + + prefsMu.Lock() + defer prefsMu.Unlock() + + sections, err := readPrefsFile(p.path) + if err != nil { + return false, err + } + raw, ok := sections[namespace] + if !ok { + return false, nil + } + if err := json.Unmarshal(raw, v); err != nil { + return false, fmt.Errorf("decode prefs namespace %q: %w", namespace, err) + } + return true, nil +} + +// Put stores v as the namespace section, replacing any previous value. +func (p *Prefs) Put(namespace string, v any) error { + if namespace == "" { + return fmt.Errorf("empty prefs namespace") + } + raw, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("encode prefs namespace %q: %w", namespace, err) + } + + prefsMu.Lock() + defer prefsMu.Unlock() + + sections, err := readPrefsFile(p.path) + if err != nil { + return err + } + sections[namespace] = raw + return writePrefsFile(p.path, sections) +} + +// Remove deletes the namespace section; a missing one is not an error. +func (p *Prefs) Remove(namespace string) error { + if namespace == "" { + return fmt.Errorf("empty prefs namespace") + } + + prefsMu.Lock() + defer prefsMu.Unlock() + + sections, err := readPrefsFile(p.path) + if err != nil { + return err + } + if _, ok := sections[namespace]; !ok { + return nil + } + delete(sections, namespace) + return writePrefsFile(p.path, sections) +} + +func removePrefsFile(path string) error { + prefsMu.Lock() + defer prefsMu.Unlock() + return os.Remove(path) +} + +func readPrefsFile(path string) (map[string]json.RawMessage, error) { + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return map[string]json.RawMessage{}, nil + } + if err != nil { + return nil, fmt.Errorf("read prefs: %w", err) + } + + sections := map[string]json.RawMessage{} + if err := json.Unmarshal(data, §ions); err != nil { + return nil, fmt.Errorf("decode prefs: %w", err) + } + return sections, nil +} + +func writePrefsFile(path string, sections map[string]json.RawMessage) error { + if err := util.WriteJsonWithRestrictedPermission(context.Background(), path, sections); err != nil { + return fmt.Errorf("write prefs: %w", err) + } + return nil +} diff --git a/client/internal/profilemanager/prefs_test.go b/client/internal/profilemanager/prefs_test.go new file mode 100644 index 000000000..692ade70f --- /dev/null +++ b/client/internal/profilemanager/prefs_test.go @@ -0,0 +1,138 @@ +package profilemanager + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testPrefsSection struct { + Mode uint8 `json:"mode"` + Dest string `json:"dest"` +} + +func TestProfilePrefs_RoundTrip(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + + require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 2, Dest: "/tmp/x"})) + require.NoError(t, prefs.Put("other", map[string]int{"n": 1})) + + var got testPrefsSection + found, err := prefs.Get("filedrop", &got) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, testPrefsSection{Mode: 2, Dest: "/tmp/x"}, got) + + var other map[string]int + found, err = prefs.Get("other", &other) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, map[string]int{"n": 1}, other) + }) +} + +func TestProfilePrefs_GetMissingNamespace(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + + var got testPrefsSection + found, err := prefs.Get("filedrop", &got) + require.NoError(t, err) + assert.False(t, found) + }) +} + +func TestProfilePrefs_RemoveNamespace(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + + require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 1})) + require.NoError(t, prefs.Put("other", map[string]int{"n": 1})) + require.NoError(t, prefs.Remove("filedrop")) + require.NoError(t, prefs.Remove("missing")) + + var got testPrefsSection + found, err := prefs.Get("filedrop", &got) + require.NoError(t, err) + assert.False(t, found) + + var other map[string]int + found, err = prefs.Get("other", &other) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, map[string]int{"n": 1}, other) + }) +} + +func TestProfilePrefs_RejectsInvalidID(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + _, err := sm.ProfilePrefs("../escape", username) + assert.Error(t, err) + }) +} + +func TestProfilePrefs_RejectsEmptyNamespace(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + + _, err = prefs.Get("", &testPrefsSection{}) + assert.Error(t, err) + assert.Error(t, prefs.Put("", testPrefsSection{})) + assert.Error(t, prefs.Remove("")) + }) +} + +func TestProfilePrefs_DefaultProfile(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + prefs, err := sm.ProfilePrefs(defaultProfileName, username) + require.NoError(t, err) + + require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 1})) + + expected := filepath.Join(filepath.Dir(DefaultConfigPath), "default"+prefsFileSuffix) + _, err = os.Stat(expected) + require.NoError(t, err) + }) +} + +func TestRemoveProfile_DeletesPrefsFile(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 2})) + + configDir, err := sm.getConfigDir(username) + require.NoError(t, err) + prefsPath := filepath.Join(configDir, created.ID.String()+prefsFileSuffix) + _, err = os.Stat(prefsPath) + require.NoError(t, err) + + require.NoError(t, sm.RemoveProfile(created.ID, username)) + _, err = os.Stat(prefsPath) + assert.True(t, errors.Is(err, os.ErrNotExist), "prefs file should be removed") + }) +} diff --git a/client/internal/profilemanager/service.go b/client/internal/profilemanager/service.go index 696a60310..ec287f01a 100644 --- a/client/internal/profilemanager/service.go +++ b/client/internal/profilemanager/service.go @@ -420,6 +420,11 @@ func (s *ServiceManager) RemoveProfile(id ID, username string) error { log.Warnf("failed to remove profile state file %s: %v", stateFile, err) } + prefsFile := filepath.Join(filepath.Dir(target.Path), id.String()+prefsFileSuffix) + if err := removePrefsFile(prefsFile); err != nil && !os.IsNotExist(err) { + log.Warnf("failed to remove profile prefs file %s: %v", prefsFile, err) + } + return nil } diff --git a/client/internal/updater/installer/doc.go b/client/internal/updater/installer/doc.go index 0a60454bb..11b0512ac 100644 --- a/client/internal/updater/installer/doc.go +++ b/client/internal/updater/installer/doc.go @@ -37,23 +37,32 @@ // Updater Process (Setup): // // 1. Receives parameters from service via command-line arguments -// 2. Runs installer with appropriate silent/quiet flags: +// 2. Terminates the UI so the installer does not have to replace a locked image +// file, which would otherwise leave the install needing a reboot +// 3. Runs installer with appropriate silent/quiet flags: // - Windows EXE: installer.exe /S -// - Windows MSI: msiexec.exe /i installer.msi /quiet /qn /l*v msi.log +// - Windows MSI: msiexec.exe /i installer.msi /qn /norestart REBOOT=ReallySuppress /l*v msi.log // - macOS PKG: installer -pkg installer.pkg -target / // - macOS Homebrew: brew upgrade netbirdio/tap/netbird -// 3. Installer terminates daemon and UI processes -// 4. Installer replaces binaries with new version -// 5. Updater waits for installer to complete -// 6. Updater restarts daemon: +// 4. Installer terminates the daemon +// 5. Installer replaces binaries with new version +// 6. Updater waits for installer to complete. On Windows, MSI exit codes 3010 +// (ERROR_SUCCESS_REBOOT_REQUIRED) and 1641 (ERROR_SUCCESS_REBOOT_INITIATED) +// are a pending-reboot outcome, not a failure: the install succeeded, but +// some files are only replaced on the next restart (the reboot itself is +// suppressed via /norestart and REBOOT=ReallySuppress), and the flow +// continues as on success +// 7. Updater restarts daemon: // - Windows: netbird.exe service start // - macOS/Linux: netbird service start -// 7. Updater restarts UI: -// - Windows: Launches netbird-ui.exe as active console user using CreateProcessAsUser +// 8. Updater restarts UI: +// - Windows: Launches netbird-ui.exe using CreateProcessAsUser in every +// session it was terminated in, falling back to the active console session // - macOS: Uses launchctl asuser to launch NetBird.app for console user // - Linux: Not implemented (UI typically auto-starts) -// 8. Updater writes result.json with success/error status -// 9. Updater process exits +// 9. Updater writes result.json with success/error status (a pending reboot is +// recorded as success) +// 10. Updater process exits // // # Result Communication // diff --git a/client/internal/updater/installer/installer_common.go b/client/internal/updater/installer/installer_common.go index 8e44bee82..17566f7de 100644 --- a/client/internal/updater/installer/installer_common.go +++ b/client/internal/updater/installer/installer_common.go @@ -42,6 +42,9 @@ func NewWithDir(tempDir string) *Installer { // This will run by the original service process func (u *Installer) RunInstallation(ctx context.Context, targetVersion string) (err error) { resultHandler := NewResultHandler(u.tempDir) + if err := resultHandler.ClearStaleResult(); err != nil { + log.Warnf("clear stale installer result: %v", err) + } defer func() { if err != nil { diff --git a/client/internal/updater/installer/installer_run_windows.go b/client/internal/updater/installer/installer_run_windows.go index 70c7e32cf..b2ecf3299 100644 --- a/client/internal/updater/installer/installer_run_windows.go +++ b/client/internal/updater/installer/installer_run_windows.go @@ -2,6 +2,7 @@ package installer import ( "context" + "errors" "fmt" "os" "os/exec" @@ -22,6 +23,12 @@ const ( msiLogFile = "msi.log" + // ERROR_SUCCESS_REBOOT_REQUIRED and ERROR_SUCCESS_REBOOT_INITIATED + msiRebootRequired = 3010 + msiRebootInitiated = 1641 + + processExitWait = 10 * time.Second + msiDownloadURL = "https://github.com/netbirdio/netbird/releases/download/v%version/netbird_installer_%version_windows_%arch.msi" exeDownloadURL = "https://github.com/netbirdio/netbird/releases/download/v%version/netbird_installer_%version_windows_%arch.exe" ) @@ -38,6 +45,8 @@ var ( func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string, daemonFolder string) (resultErr error) { resultHandler := NewResultHandler(u.tempDir) + var uiSessions []uint32 + // Always ensure daemon and UI are restarted after setup defer func() { log.Infof("starting daemon back") @@ -46,7 +55,7 @@ func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string } log.Infof("starting UI back") - if err := u.startUIAsUser(daemonFolder); err != nil { + if err := u.startUI(daemonFolder, uiSessions); err != nil { log.Errorf("failed to start UI: %v", err) } @@ -75,6 +84,14 @@ func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string return } + // The UI holds an open handle on its own image. Left running, Restart Manager + // cannot shut it down (msiexec runs as LocalSystem here, the UI as the + // interactive user), so the MSI falls back to replacing the file on reboot and + // marks the install as restart-required. The deferred close-application action + // in the package runs too late to prevent that, it happens after + // InstallValidate has already registered the file as in use. + uiSessions = killUI() + var cmd *exec.Cmd switch installerType { case TypeExe: @@ -84,7 +101,9 @@ func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string installerDir := filepath.Dir(installerFile) logPath := filepath.Join(installerDir, msiLogFile) log.Infof("run msi installer: %s", installerFile) - cmd = exec.CommandContext(ctx, "msiexec.exe", "/i", filepath.Base(installerFile), "/quiet", "/qn", "/l*v", logPath) + // REBOOT=ReallySuppress: a silent install has no way to ask, so without it + // msiexec reboots the machine on its own if it decides one is needed. + cmd = exec.CommandContext(ctx, "msiexec.exe", "/i", filepath.Base(installerFile), "/qn", "/norestart", "REBOOT=ReallySuppress", "/l*v", logPath) } cmd.Dir = filepath.Dir(installerFile) @@ -95,9 +114,13 @@ func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string } log.Infof("installer started with PID %d", cmd.Process.Pid) - if resultErr = cmd.Wait(); resultErr != nil { - log.Errorf("installer process finished with error: %v", resultErr) - return + if err := cmd.Wait(); err != nil { + if !isRebootPending(err) { + resultErr = err + log.Errorf("installer process finished with error: %v", err) + return + } + log.Warnf("installer completed but reported a pending reboot, some files will be replaced on the next restart") } return nil @@ -117,16 +140,142 @@ func (u *Installer) startDaemon(daemonFolder string) error { return nil } -func (u *Installer) startUIAsUser(daemonFolder string) error { +func (u *Installer) startUI(daemonFolder string, sessionIDs []uint32) error { uiPath := filepath.Join(daemonFolder, uiName) log.Infof("starting netbird-ui: %s", uiPath) - // Get the active console session ID - sessionID := windows.WTSGetActiveConsoleSessionId() - if sessionID == 0xFFFFFFFF { - return fmt.Errorf("no active user session found") + if len(sessionIDs) == 0 { + sessionID := windows.WTSGetActiveConsoleSessionId() + if sessionID == 0xFFFFFFFF { + return fmt.Errorf("no active user session found") + } + sessionIDs = []uint32{sessionID} } + var errs []error + for _, sessionID := range sessionIDs { + if err := startUIInSession(uiPath, sessionID); err != nil { + errs = append(errs, fmt.Errorf("session %d: %w", sessionID, err)) + continue + } + log.Infof("netbird-ui started successfully in session %d", sessionID) + } + return errors.Join(errs...) +} + +// isRebootPending reports whether the installer exit code means it succeeded but +// left work for the next restart. The reboot itself is suppressed, so this is not +// a failure. +func isRebootPending(err error) bool { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + return false + } + + switch exitErr.ExitCode() { + case msiRebootRequired, msiRebootInitiated: + return true + default: + return false + } +} + +// killUI terminates any running netbird-ui process and returns the IDs of the +// interactive sessions the terminated processes belonged to. Setup starts the +// UI again in those sessions once the installer is done. +func killUI() []uint32 { + pids, err := processIDsByName(uiName) + if err != nil { + log.Warnf("failed to look up %s processes: %v", uiName, err) + return nil + } + + sessions := make(map[uint32]struct{}) + for _, pid := range pids { + var sessionID uint32 + if err := windows.ProcessIdToSessionId(pid, &sessionID); err != nil { + log.Warnf("failed to look up session of %s (PID %d): %v", uiName, pid, err) + } + + if err := terminateProcess(pid); err != nil { + log.Warnf("failed to terminate %s (PID %d): %v", uiName, pid, err) + continue + } + log.Infof("terminated %s (PID %d) in session %d", uiName, pid, sessionID) + + if sessionID != 0 { + sessions[sessionID] = struct{}{} + } + } + + sessionIDs := make([]uint32, 0, len(sessions)) + for sessionID := range sessions { + sessionIDs = append(sessionIDs, sessionID) + } + return sessionIDs +} + +func processIDsByName(name string) ([]uint32, error) { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0) + if err != nil { + return nil, fmt.Errorf("create process snapshot: %w", err) + } + defer func() { + if err := windows.CloseHandle(snapshot); err != nil { + log.Warnf("failed to close process snapshot: %v", err) + } + }() + + var entry windows.ProcessEntry32 + entry.Size = uint32(unsafe.Sizeof(entry)) + + var pids []uint32 + for err = windows.Process32First(snapshot, &entry); err == nil; err = windows.Process32Next(snapshot, &entry) { + if strings.EqualFold(windows.UTF16ToString(entry.ExeFile[:]), name) { + pids = append(pids, entry.ProcessID) + } + } + if !errors.Is(err, windows.ERROR_NO_MORE_FILES) { + return nil, fmt.Errorf("enumerate processes: %w", err) + } + + return pids, nil +} + +func terminateProcess(pid uint32) error { + handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE|windows.SYNCHRONIZE, false, pid) + if err != nil { + // The process may have exited between enumeration and now. + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + return nil + } + return fmt.Errorf("open process: %w", err) + } + defer func() { + if err := windows.CloseHandle(handle); err != nil { + log.Warnf("failed to close process handle: %v", err) + } + }() + + if err := windows.TerminateProcess(handle, 0); err != nil { + return fmt.Errorf("terminate process: %w", err) + } + + // Wait for the handle to signal so the image file is released before the + // installer tries to overwrite it. A timeout is reported through the returned + // event, not through err, which stays nil unless the wait itself failed. + event, err := windows.WaitForSingleObject(handle, uint32(processExitWait.Milliseconds())) + if err != nil { + return fmt.Errorf("wait for process exit: %w", err) + } + if event != windows.WAIT_OBJECT_0 { + return fmt.Errorf("wait for process exit: unexpected wait result %#x", event) + } + + return nil +} + +func startUIInSession(uiPath string, sessionID uint32) error { // Get the user token for that session var userToken windows.Token err := windows.WTSQueryUserToken(sessionID, &userToken) @@ -158,6 +307,16 @@ func (u *Installer) startUIAsUser(daemonFolder string) error { } }() + var env *uint16 + if err := windows.CreateEnvironmentBlock(&env, primaryToken, false); err != nil { + return fmt.Errorf("create environment block: %w", err) + } + defer func() { + if err := windows.DestroyEnvironmentBlock(env); err != nil { + log.Warnf("failed to destroy environment block: %v", err) + } + }() + // Prepare startup info var si windows.StartupInfo si.Cb = uint32(unsafe.Sizeof(si)) @@ -180,7 +339,7 @@ func (u *Installer) startUIAsUser(daemonFolder string) error { nil, false, creationFlags, - nil, + env, nil, &si, &pi, @@ -197,7 +356,6 @@ func (u *Installer) startUIAsUser(daemonFolder string) error { log.Warnf("failed to close thread handle: %v", err) } - log.Infof("netbird-ui started successfully in session %d", sessionID) return nil } diff --git a/client/internal/updater/installer/installer_run_windows_test.go b/client/internal/updater/installer/installer_run_windows_test.go new file mode 100644 index 000000000..6a4540610 --- /dev/null +++ b/client/internal/updater/installer/installer_run_windows_test.go @@ -0,0 +1,108 @@ +package installer + +import ( + "errors" + "os/exec" + "slices" + "strconv" + "testing" +) + +// exitErrorWithCode returns a real *exec.ExitError carrying the given exit code. +func exitErrorWithCode(t *testing.T, code int) error { + t.Helper() + + err := exec.Command("cmd.exe", "/c", "exit "+strconv.Itoa(code)).Run() + if err == nil { + t.Fatalf("expected a non-zero exit for code %d", code) + } + return err +} + +func TestIsRebootPending(t *testing.T) { + tests := []struct { + name string + code int + want bool + }{ + {name: "reboot required", code: msiRebootRequired, want: true}, + {name: "reboot initiated", code: msiRebootInitiated, want: true}, + {name: "generic failure", code: 1603, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isRebootPending(exitErrorWithCode(t, tt.code)); got != tt.want { + t.Errorf("isRebootPending(exit %d) = %v, want %v", tt.code, got, tt.want) + } + }) + } +} + +// TestProcessIDsByNameAndTerminate spawns a long-running system process, finds it +// by name and terminates it, covering the path the updater uses to release the UI +// image file before the installer replaces it. +func TestProcessIDsByNameAndTerminate(t *testing.T) { + cmd := exec.Command("ping.exe", "-n", "60", "127.0.0.1") + if err := cmd.Start(); err != nil { + t.Fatalf("start ping: %v", err) + } + + pid := uint32(cmd.Process.Pid) + killed := false + t.Cleanup(func() { + if !killed { + _ = cmd.Process.Kill() + } + _ = cmd.Wait() + }) + + // Name matching must be case-insensitive: the snapshot reports PING.EXE. + pids, err := processIDsByName("ping.exe") + if err != nil { + t.Fatalf("processIDsByName: %v", err) + } + + if !slices.Contains(pids, pid) { + t.Fatalf("PID %d not among the ping.exe processes found: %v", pid, pids) + } + + if err := terminateProcess(pid); err != nil { + t.Fatalf("terminateProcess: %v", err) + } + killed = true + + // terminateProcess only returns once the handle has signalled, so the process + // is already gone and Wait must not block. It exits with the code passed to + // TerminateProcess, which is 0, so Wait reports no error. + if err := cmd.Wait(); err != nil { + t.Fatalf("wait for terminated ping: %v", err) + } + if !cmd.ProcessState.Exited() { + t.Error("process did not exit after terminateProcess") + } + + remaining, err := processIDsByName("ping.exe") + if err != nil { + t.Fatalf("processIDsByName after terminate: %v", err) + } + if slices.Contains(remaining, pid) { + t.Errorf("PID %d still listed after terminateProcess", pid) + } +} + +func TestProcessIDsByNameNoMatch(t *testing.T) { + pids, err := processIDsByName("netbird-nonexistent-process.exe") + if err != nil { + t.Fatalf("processIDsByName: %v", err) + } + if len(pids) != 0 { + t.Errorf("expected no matches, got %v", pids) + } +} + +func TestIsRebootPendingNonExitError(t *testing.T) { + if isRebootPending(errors.New("start installer: file not found")) { + t.Error("a non-exit error must not be treated as a pending reboot") + } +} diff --git a/client/internal/updater/installer/result.go b/client/internal/updater/installer/result.go index 526c3eb53..55a0d8ac8 100644 --- a/client/internal/updater/installer/result.go +++ b/client/internal/updater/installer/result.go @@ -54,6 +54,12 @@ func (rh *ResultHandler) GetErrorResultReason() string { return "" } +// ClearStaleResult removes a result file left over from a previous installation +// attempt so result watchers cannot read an outdated outcome for the current attempt. +func (rh *ResultHandler) ClearStaleResult() error { + return rh.cleanup() +} + func (rh *ResultHandler) WriteSuccess() error { result := Result{ Success: true, diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index 5df6f92f1..f92f085ab 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -22,6 +22,8 @@ import ( "github.com/netbirdio/netbird/client/internal/listener" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/formatter" "github.com/netbirdio/netbird/route" @@ -36,11 +38,6 @@ const ( AnonymizeLevelStrict = nbAnonymize.LevelStrictString ) -// ConnectionListener export internal Listener for mobile -type ConnectionListener interface { - peer.Listener -} - // RouteListener export internal RouteListener for mobile type NetworkChangeListener interface { listener.NetworkChangeListener @@ -87,6 +84,12 @@ type Client struct { onHostDnsFn func([]string) dnsManager dns.IosDnsManager loginComplete bool + // netState outlives engine restarts: it mirrors the OS connectivity, not + // the engine lifecycle. Run injects it into each new ConnectClient, which + // distributes it to every reconnection loop. + netState *netstate.State + // sweeper also outlives engine restarts; NotifyNetworkChange sweeps it. + sweeper *netsweep.Sweeper // preloadedConfig holds config loaded from JSON (used on tvOS where file writes are blocked) preloadedConfig *profilemanager.Config @@ -109,6 +112,8 @@ func NewClient(cfgFile, stateFile, cacheDir, logFilePath, deviceName string, osV ctxCancelLock: &sync.Mutex{}, networkChangeListener: networkChangeListener, dnsManager: dnsManager, + netState: netstate.New(), + sweeper: netsweep.New(), } } @@ -184,7 +189,8 @@ func (c *Client) Run(fd int32, interfaceName string, envList *EnvList) error { c.onHostDnsFn = func([]string) {} cfg.WgIface = interfaceName - connectClient := internal.NewConnectClient(ctx, cfg, c.recorder) + connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, + internal.WithNetworkState(c.netState), internal.WithSweeper(c.sweeper)) c.setState(cfg, connectClient) // Persist the latest sync response so DebugBundle can include the network // map. On iOS this is backed by disk to keep it out of the constrained @@ -193,6 +199,25 @@ func (c *Client) Run(fd int32, interfaceName string, envList *EnvList) error { return connectClient.RunOniOS(fd, c.networkChangeListener, c.dnsManager, c.stateFile, c.cacheDir, c.logFilePath) } +// SetNetworkAvailable feeds OS-reported network availability into the client +// (e.g. from NWPathMonitor). While unavailable, the internal reconnect loops +// suspend their attempts and the connection listener reports NoNetwork +// instead of Connecting; when availability returns, the loops resume +// immediately with a fresh backoff. +func (c *Client) SetNetworkAvailable(available bool) { + c.netState.Set(available) + c.recorder.SetNetworkAvailable(available) +} + +// NotifyNetworkChange marks the management, signal and relay connections +// stale after the OS switched networks and schedules a sweep that cuts +// whatever has not redialed on the new network by then. The engine and the +// TUN device stay untouched. +func (c *Client) NotifyNetworkChange() { + c.sweeper.MarkNetworkChange() + log.Infof("network change: connections marked stale") +} + // Stop the internal client and free the resources func (c *Client) Stop() { c.ctxCancelLock.Lock() @@ -331,7 +356,11 @@ func (c *Client) GetStatusDetails() *StatusDetails { // SetConnectionListener set the network connection listener func (c *Client) SetConnectionListener(listener ConnectionListener) { - c.recorder.SetConnectionListener(listener) + if listener == nil { + c.recorder.RemoveConnectionListener() + return + } + c.recorder.SetConnectionListener(connectionListenerAdapter{listener}) } // RemoveConnectionListener remove connection listener diff --git a/client/ios/NetBirdSDK/connection_listener.go b/client/ios/NetBirdSDK/connection_listener.go new file mode 100644 index 000000000..d792537ba --- /dev/null +++ b/client/ios/NetBirdSDK/connection_listener.go @@ -0,0 +1,43 @@ +//go:build ios + +package NetBirdSDK + +import ( + "github.com/netbirdio/netbird/client/internal/peer" +) + +// Client state values, re-exported as basic constants so gomobile emits them +// into the generated bindings. They mirror peer.ClientState*: append-only, +// never reorder. +const ( + ClientStateDisconnected = int(peer.ClientStateDisconnected) + ClientStateConnected = int(peer.ClientStateConnected) + ClientStateConnecting = int(peer.ClientStateConnecting) + ClientStateDisconnecting = int(peer.ClientStateDisconnecting) + ClientStateNoNetwork = int(peer.ClientStateNoNetwork) +) + +// ConnectionListener export internal Listener for mobile. +// +// It intentionally lacks OnStateChanged for now: adding a method to a gomobile +// interface breaks every Swift implementation, so the iOS app keeps building +// against the legacy per-state callbacks. A follow-up will extend it together +// with the app. +type ConnectionListener interface { + OnConnected() + OnDisconnected() + OnConnecting() + OnDisconnecting() + OnAddressChanged(string, string) + OnPeersListChanged(int) +} + +// connectionListenerAdapter adapts the gomobile-facing ConnectionListener to +// peer.Listener. +type connectionListenerAdapter struct { + ConnectionListener +} + +// OnStateChanged is dropped on iOS until the app adopts the state callback; +// the legacy per-state callbacks continue to fire. +func (a connectionListenerAdapter) OnStateChanged(peer.ClientState) {} diff --git a/client/ios/NetBirdSDK/login.go b/client/ios/NetBirdSDK/login.go index 6cba0c411..42a575359 100644 --- a/client/ios/NetBirdSDK/login.go +++ b/client/ios/NetBirdSDK/login.go @@ -323,7 +323,7 @@ func (a *Auth) login(urlOpener URLOpener, forceDeviceAuth bool, deviceName strin const authInfoRequestTimeout = 30 * time.Second func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener, forceDeviceAuth bool) (*auth.TokenInfo, error) { - oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, forceDeviceAuth) + oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, forceDeviceAuth, "") if err != nil { return nil, fmt.Errorf("failed to get OAuth flow: %v", err) } diff --git a/client/netstate/netstate.go b/client/netstate/netstate.go new file mode 100644 index 000000000..0d7a1268b --- /dev/null +++ b/client/netstate/netstate.go @@ -0,0 +1,110 @@ +// Package netstate tracks OS-reported network availability for the client. +// +// A State instance is owned by the platform integration (e.g. the Android or +// iOS bindings, fed from ConnectivityManager callbacks or NWPathMonitor) and +// is injected into the connection retry loops (management, signal, relay, +// peer guards and the top-level connect loop), which consult it to avoid +// burning CPU and battery on reconnect attempts while the device has no +// network at all (e.g. airplane mode), and to reset their backoff as soon as +// the network returns. +// +// Consumers hold a *State that may be nil — every non-mobile platform leaves +// it unset. The read methods are safe on a nil receiver: they report online +// and never block, so consumers behave as if this package did not exist. +package netstate + +import ( + "context" + "sync" + + log "github.com/sirupsen/logrus" +) + +// State holds the OS-reported network availability. The zero value is not +// usable; create instances with New. +type State struct { + mu sync.Mutex + online bool + changed chan struct{} +} + +// New creates a State that starts online. Platforms without network tracking +// pass a nil *State instead: the read methods treat nil as always online and +// never block, so consumers need no nil guards. +func New() *State { + return &State{ + online: true, + changed: make(chan struct{}), + } +} + +// Set records whether the OS reports any usable network. Transitions wake up +// all Wait callers immediately. Unlike the read methods, Set is not nil-safe: +// it is only for the platform owner that created the State with New. +func (s *State) Set(online bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.online == online { + return + } + s.online = online + close(s.changed) + s.changed = make(chan struct{}) + log.Infof("OS network availability changed: online=%t", online) +} + +// IsOnline reports whether the OS reports at least one usable network. On a +// nil receiver — no State injected — it reports online. +func (s *State) IsOnline() bool { + if s == nil { + return true + } + s.mu.Lock() + defer s.mu.Unlock() + return s.online +} + +// Changed returns a channel closed on the next availability transition, for +// callers that already own a select loop and cannot block in Wait. Re-read it +// after every fire: each transition installs a fresh channel. On a nil +// receiver — no State injected — it returns nil, which blocks forever in a +// select, so the caller simply never observes a transition. +func (s *State) Changed() <-chan struct{} { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + return s.changed +} + +// Wait blocks while the network is offline. It reports whether it had to +// wait, so callers can reset their backoff after an outage. It returns early +// with the context error when ctx is done. On a nil receiver — no State +// injected — it returns immediately. +func (s *State) Wait(ctx context.Context) (bool, error) { + if s == nil { + return false, nil + } + waited := false + for { + s.mu.Lock() + if s.online { + s.mu.Unlock() + return waited, nil + } + ch := s.changed + s.mu.Unlock() + + if !waited { + waited = true + log.Debugf("network is offline, pausing connection attempts") + } + + select { + case <-ctx.Done(): + return waited, ctx.Err() + case <-ch: + } + } +} diff --git a/client/netstate/netstate_test.go b/client/netstate/netstate_test.go new file mode 100644 index 000000000..ea7015761 --- /dev/null +++ b/client/netstate/netstate_test.go @@ -0,0 +1,170 @@ +package netstate + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewStateIsOnline(t *testing.T) { + assert.True(t, New().IsOnline(), "a fresh State should start online") +} + +func TestSetTogglesOnlineState(t *testing.T) { + s := New() + + s.Set(false) + assert.False(t, s.IsOnline(), "state should be offline after Set(false)") + + s.Set(true) + assert.True(t, s.IsOnline(), "state should be online after Set(true)") +} + +func TestWaitReturnsImmediatelyWhenOnline(t *testing.T) { + s := New() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + waited, err := s.Wait(ctx) + require.NoError(t, err) + assert.False(t, waited, "Wait should not block when the network is online") +} + +func TestWaitBlocksUntilOnline(t *testing.T) { + s := New() + s.Set(false) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + result := make(chan bool, 1) + go func() { + waited, err := s.Wait(ctx) + if err != nil { + result <- false + return + } + result <- waited + }() + + // Verify Wait is actually blocking while offline + select { + case <-result: + t.Fatal("Wait should block while the network is offline") + case <-time.After(100 * time.Millisecond): + } + + s.Set(true) + + select { + case waited := <-result: + assert.True(t, waited, "Wait should report that it had to wait for the network") + case <-time.After(2 * time.Second): + t.Fatal("Wait should return promptly after the network becomes available") + } +} + +func TestWaitReturnsOnContextCancel(t *testing.T) { + s := New() + s.Set(false) + + ctx, cancel := context.WithCancel(context.Background()) + + result := make(chan error, 1) + go func() { + _, err := s.Wait(ctx) + result <- err + }() + + cancel() + + select { + case err := <-result: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("Wait should return promptly after context cancellation") + } +} + +func TestWaitWakesAllWaiters(t *testing.T) { + s := New() + s.Set(false) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + const waiters = 10 + var wg sync.WaitGroup + results := make(chan bool, waiters) + for i := 0; i < waiters; i++ { + wg.Add(1) + go func() { + defer wg.Done() + waited, err := s.Wait(ctx) + if err != nil { + results <- false + return + } + results <- waited + }() + } + + time.Sleep(100 * time.Millisecond) + s.Set(true) + wg.Wait() + + close(results) + count := 0 + for waited := range results { + assert.True(t, waited, "every waiter should report that it waited") + count++ + } + assert.Equal(t, waiters, count, "all waiters should have returned") +} + +func TestNilStateReadsAreNoops(t *testing.T) { + var s *State + + assert.True(t, s.IsOnline(), "nil State should report online") + + waited, err := s.Wait(context.Background()) + require.NoError(t, err) + assert.False(t, waited, "nil State's Wait should not block") +} + +func TestConcurrentSetAndWait(t *testing.T) { + s := New() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + s.Set(j%2 == 0) + s.IsOnline() + } + }() + } + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + if _, err := s.Wait(ctx); err != nil { + return + } + } + }() + } + + wg.Wait() +} diff --git a/client/netsweep/netsweep.go b/client/netsweep/netsweep.go new file mode 100644 index 000000000..46bc0a709 --- /dev/null +++ b/client/netsweep/netsweep.go @@ -0,0 +1,267 @@ +// Package netsweep cuts network-bound activity when the OS switches networks: +// a sweep closes the registered connections and aborts the in-flight dials, so +// their owners redial immediately instead of waiting for the old sockets to +// time out. +// +// A nil *Sweeper disables everything: all methods are nil-safe no-ops. +package netsweep + +import ( + "context" + "errors" + "net" + "sync" + "time" + + "github.com/cenkalti/backoff/v4" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/netstate" +) + +// DefaultSweepDelay absorbs network flapping while the OS settles on a +// default network before the stale registrations are cut. +const DefaultSweepDelay = 500 * time.Millisecond + +const recentMarkWindow = 3 * time.Second + +// Config customizes a Sweeper. The zero value applies the defaults. +type Config struct { + // SweepDelay overrides DefaultSweepDelay when positive. + SweepDelay time.Duration +} + +// ErrSwept reports that a dial finished after a network change swept its +// registration. The connection is already closed; the caller must treat it +// as a failed dial and redial on the new network. +var ErrSwept = errors.New("netsweep: connection swept by network change") + +// sweepID identifies one registration in a sweeper. Connections and dials +// draw from the same counter, so an id is unique across both registries. +type sweepID uint64 + +type connEntry struct { + conn net.Conn + gen uint64 +} + +// Dial tracks one dial from start to connection registration. It hands the +// dialed connection to the sweeper atomically, so a sweep can never fall +// between the dial finishing and the connection being registered. +type Dial struct { + sweeper *Sweeper + ctx context.Context + cancel context.CancelFunc + id sweepID + done bool // set by a sweep, WrapConn or Release; guarded by sweeper.mu + gen uint64 +} + +// Ctx returns the dial's context. A sweep cancels it, so a dial started on the +// old network aborts instead of waiting out its handshake timeout. +func (d *Dial) Ctx() context.Context { + return d.ctx +} + +// Release ends the dial's registration and cancels its context. It is +// idempotent and safe after WrapConn, so callers can defer it. +func (d *Dial) Release() { + s := d.sweeper + if s == nil { + return + } + + s.mu.Lock() + d.done = true + delete(s.dials, d.id) + s.mu.Unlock() + + d.cancel() +} + +// sweptConn deregisters itself from the sweeper when closed. +type sweptConn struct { + net.Conn + sweeper *Sweeper + id sweepID +} + +func (c *sweptConn) Close() error { + c.sweeper.deregister(c.id) + return c.Conn.Close() +} + +// Sweeper registers live connections and in-flight dials so the +// network-change sweep can cut everything registered before the change. +type Sweeper struct { + mu sync.Mutex + conns map[sweepID]connEntry + dials map[sweepID]*Dial + nextID sweepID + gen uint64 + timer *time.Timer + sweepDelay time.Duration + lastMark time.Time +} + +// New creates an empty sweeper with the default configuration. +func New() *Sweeper { + return NewWithConfig(Config{}) +} + +// NewWithConfig creates an empty sweeper customized by cfg. +func NewWithConfig(cfg Config) *Sweeper { + delay := cfg.SweepDelay + if delay <= 0 { + delay = DefaultSweepDelay + } + return &Sweeper{ + conns: make(map[sweepID]connEntry), + dials: make(map[sweepID]*Dial), + sweepDelay: delay, + } +} + +// StartDial registers an in-flight dial. Dial with Ctx, hand the result to +// WrapConn, and Release the dial when the attempt is over, typically deferred. +func (s *Sweeper) StartDial(ctx context.Context) *Dial { + if s == nil { + return &Dial{ctx: ctx} + } + + ctx, cancel := context.WithCancel(ctx) + d := &Dial{sweeper: s, ctx: ctx, cancel: cancel} + + s.mu.Lock() + d.id = s.nextID + s.nextID++ + d.gen = s.gen + s.dials[d.id] = d + s.mu.Unlock() + + return d +} + +// WrapConn hands conn over to the sweeper. If a sweep ran since StartDial, +// the connection belongs to the old network: it is closed and ErrSwept is +// returned. Otherwise conn is registered against the next sweep and returned +// wrapped, deregistering itself on Close. Call it once, before Release. +func (d *Dial) WrapConn(conn net.Conn) (net.Conn, error) { + s := d.sweeper + if s == nil { + return conn, nil + } + + s.mu.Lock() + if d.done { + s.mu.Unlock() + if err := conn.Close(); err != nil { + log.Debugf("swept dial close error: %v", err) + } + return nil, ErrSwept + } + d.done = true + delete(s.dials, d.id) + id := s.nextID + s.nextID++ + // The conn inherits the dial's generation: the socket was bound to the + // network that was default when the dial started, not when it finished. + s.conns[id] = connEntry{conn: conn, gen: d.gen} + s.mu.Unlock() + + return &sweptConn{Conn: conn, sweeper: s, id: id}, nil +} + +// MarkNetworkChange records that the OS switched networks: everything +// registered so far becomes stale, and a sweep is (re)scheduled after the +// configured delay to cut whatever is still stale by then. Owners that +// redialed in the meantime hold fresh-generation registrations and survive, +// so no cancellation is needed around the sweep. +func (s *Sweeper) MarkNetworkChange() { + if s == nil { + return + } + + s.mu.Lock() + s.gen++ + cutoff := s.gen + s.lastMark = time.Now() + if s.timer != nil { + s.timer.Stop() + } + s.timer = time.AfterFunc(s.sweepDelay, func() { + n := s.sweep(cutoff) + log.Infof("network change sweep: closed %d stale connections", n) + }) + s.mu.Unlock() +} + +// QuickRetryBackoff wraps bo so that after each Reset the first retry comes +// quickly when the disconnect followed a recent network change and the +// network is online. Any other failure keeps bo's spread, so the clients of +// a restarted server still scatter their reconnects. A nil sweeper returns +// bo unchanged. +func (s *Sweeper) QuickRetryBackoff(ctx context.Context, bo backoff.BackOff, netState *netstate.State) backoff.BackOff { + if s == nil { + return bo + } + return backoff.WithContext(newQuickRetryBackoff(bo, s, netState), ctx) +} + +func (s *Sweeper) markedRecently() bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + return !s.lastMark.IsZero() && time.Since(s.lastMark) < recentMarkWindow +} + +// sweep closes the registered connections and aborts the in-flight dials +// older than cutoff, and returns how many connections it closed. A dial +// whose connection was not yet handed to WrapConn is marked, so the late +// WrapConn closes it instead of registering it. +func (s *Sweeper) sweep(cutoff uint64) int { + if s == nil { + return 0 + } + + s.mu.Lock() + var conns []net.Conn + for id, e := range s.conns { + if e.gen < cutoff { + delete(s.conns, id) + conns = append(conns, e.conn) + } + } + var dials []*Dial + for id, d := range s.dials { + if d.gen < cutoff { + d.done = true + delete(s.dials, id) + dials = append(dials, d) + } + } + s.mu.Unlock() + + if len(dials) > 0 { + log.Debugf("aborting %d in-flight dials", len(dials)) + for _, d := range dials { + d.cancel() + } + } + + for _, conn := range conns { + log.Debugf("sweeping connection %s -> %s", conn.LocalAddr(), conn.RemoteAddr()) + if err := conn.Close(); err != nil { + log.Debugf("swept connection close error: %v", err) + } + } + return len(conns) +} + +func (s *Sweeper) deregister(id sweepID) { + s.mu.Lock() + delete(s.conns, id) + s.mu.Unlock() +} diff --git a/client/netsweep/netsweep_test.go b/client/netsweep/netsweep_test.go new file mode 100644 index 000000000..88d660c2d --- /dev/null +++ b/client/netsweep/netsweep_test.go @@ -0,0 +1,241 @@ +package netsweep + +import ( + "context" + "math" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSweepClosesRegisteredConns(t *testing.T) { + sweeper := New() + + c1 := wrap(t, sweeper, connPair(t)) + c2 := wrap(t, sweeper, connPair(t)) + + assert.Equal(t, 2, sweeper.sweepAll(), "both live connections should be closed") + + // The wrappers must report closed now. + buf := make([]byte, 1) + _, err := c1.Read(buf) + assert.Error(t, err, "first connection should be unusable after the sweep") + _, err = c2.Read(buf) + assert.Error(t, err, "second connection should be unusable after the sweep") + + assert.Equal(t, 0, sweeper.sweepAll(), "second sweep should find nothing") +} + +func TestCloseDeregisters(t *testing.T) { + sweeper := New() + + conn := wrap(t, sweeper, connPair(t)) + require.NoError(t, conn.Close()) + + assert.Equal(t, 0, sweeper.sweepAll(), "closed connection must leave the registry") +} + +func TestCloseIsIdempotent(t *testing.T) { + sweeper := New() + + conn := wrap(t, sweeper, connPair(t)) + require.NoError(t, conn.Close()) + assert.Error(t, conn.Close(), "double close surfaces the underlying error but must not panic") +} + +func TestSweepOnlyAffectsOlderConns(t *testing.T) { + sweeper := New() + + _ = wrap(t, sweeper, connPair(t)) + assert.Equal(t, 1, sweeper.sweepAll()) + + // A connection dialed after the sweep must survive until the next one. + _ = wrap(t, sweeper, connPair(t)) + assert.Equal(t, 1, sweeper.sweepAll(), "post-sweep connection belongs to the next sweep") +} + +func TestSweepAbortsInFlightDials(t *testing.T) { + sweeper := New() + + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + + sweeper.sweepAll() + + assert.ErrorIs(t, dial.Ctx().Err(), context.Canceled, "sweep must cancel the in-flight dial context") +} + +func TestReleasedDialIsNotAborted(t *testing.T) { + sweeper := New() + + // Simulate a dial that finished before the sweep. + released := sweeper.StartDial(context.Background()) + released.Release() + + // A dial still in flight during the sweep. + pending := sweeper.StartDial(context.Background()) + defer pending.Release() + + sweeper.sweepAll() + assert.ErrorIs(t, pending.Ctx().Err(), context.Canceled, "pending dial must be aborted") +} + +func TestSweepBetweenDialAndHandoffClosesConn(t *testing.T) { + sweeper := New() + + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + + // The dial succeeds on the old network, then the sweep lands before the + // connection is handed over. + conn := connPair(t) + assert.Equal(t, 0, sweeper.sweepAll(), "the connection is not registered yet") + + wrapped, err := dial.WrapConn(conn) + require.ErrorIs(t, err, ErrSwept) + require.Nil(t, wrapped) + + buf := make([]byte, 1) + _, err = conn.Read(buf) + assert.Error(t, err, "the old-network connection must be closed, not leaked") + + assert.Equal(t, 0, sweeper.sweepAll(), "nothing may leak into the next sweep") +} + +func TestMarkNetworkChangeSparesFreshConns(t *testing.T) { + sweeper := NewWithConfig(Config{SweepDelay: 10 * time.Millisecond}) + + stale := wrap(t, sweeper, connPair(t)) + sweeper.MarkNetworkChange() + _ = wrap(t, sweeper, connPair(t)) + + _ = stale.SetReadDeadline(time.Now().Add(time.Second)) + buf := make([]byte, 1) + _, err := stale.Read(buf) + require.ErrorIs(t, err, net.ErrClosed, "stale connection must be closed by the delayed sweep") + + assert.Equal(t, 1, sweeper.sweepAll(), "the fresh connection must survive the stale sweep") +} + +func TestMarkNetworkChangeAbortsStaleDials(t *testing.T) { + sweeper := NewWithConfig(Config{SweepDelay: 10 * time.Millisecond}) + + stale := sweeper.StartDial(context.Background()) + defer stale.Release() + sweeper.MarkNetworkChange() + fresh := sweeper.StartDial(context.Background()) + defer fresh.Release() + + assert.Eventually(t, func() bool { + return stale.Ctx().Err() != nil + }, time.Second, 5*time.Millisecond, "stale dial must be aborted by the delayed sweep") + assert.NoError(t, fresh.Ctx().Err(), "post-mark dial must not be aborted") +} + +func TestConnInheritsDialGeneration(t *testing.T) { + sweeper := NewWithConfig(Config{SweepDelay: 20 * time.Millisecond}) + + // The dial starts before the network change but completes after it: the + // socket is bound to the old network, so the sweep must still cut it. + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + sweeper.MarkNetworkChange() + + wrapped, err := dial.WrapConn(connPair(t)) + require.NoError(t, err) + + _ = wrapped.SetReadDeadline(time.Now().Add(time.Second)) + buf := make([]byte, 1) + _, err = wrapped.Read(buf) + require.ErrorIs(t, err, net.ErrClosed, "old-generation connection must be swept") +} + +func TestRepeatedMarksCoalesce(t *testing.T) { + sweeper := NewWithConfig(Config{SweepDelay: 20 * time.Millisecond}) + + first := wrap(t, sweeper, connPair(t)) + sweeper.MarkNetworkChange() + second := wrap(t, sweeper, connPair(t)) + sweeper.MarkNetworkChange() + _ = wrap(t, sweeper, connPair(t)) + + buf := make([]byte, 1) + for _, conn := range []net.Conn{first, second} { + _ = conn.SetReadDeadline(time.Now().Add(time.Second)) + _, err := conn.Read(buf) + require.ErrorIs(t, err, net.ErrClosed, "every pre-mark connection must be swept by the rescheduled sweep") + } + assert.Equal(t, 1, sweeper.sweepAll(), "only the newest-generation connection may remain") +} + +func TestNilSweeperIsNoop(t *testing.T) { + var sweeper *Sweeper + + conn := connPair(t) + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + + wrapped, err := dial.WrapConn(conn) + require.NoError(t, err) + assert.Equal(t, conn, wrapped, "nil sweeper must return the conn unchanged") + assert.NoError(t, dial.Ctx().Err(), "nil sweeper must not cancel the dial context") + assert.Equal(t, 0, sweeper.sweepAll(), "nil sweeper closes nothing") +} + +// wrap registers conn with the sweeper through a completed dial. +func wrap(t *testing.T, sweeper *Sweeper, conn net.Conn) net.Conn { + t.Helper() + + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + + wrapped, err := dial.WrapConn(conn) + require.NoError(t, err) + return wrapped +} + +// connPair dials a loopback TCP connection and keeps the accepted peer open +// until the test ends: a peer that closed early would make the connection +// unreadable on its own, so a read error after the sweep would prove nothing. +func connPair(t *testing.T) net.Conn { + t.Helper() + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { + if err := l.Close(); err != nil { + t.Logf("listener close error: %v", err) + } + }) + + accepted := make(chan net.Conn, 1) + go func() { + conn, err := l.Accept() + if err != nil { + close(accepted) + return + } + accepted <- conn + }() + + conn, err := net.Dial("tcp", l.Addr().String()) + require.NoError(t, err) + + peer, ok := <-accepted + require.True(t, ok, "listener must accept the dialed connection") + t.Cleanup(func() { + if err := peer.Close(); err != nil { + t.Logf("peer close error: %v", err) + } + }) + + return conn +} + +// sweepAll cuts every registration regardless of generation. +func (s *Sweeper) sweepAll() int { + return s.sweep(math.MaxUint64) +} diff --git a/client/netsweep/quick_retry.go b/client/netsweep/quick_retry.go new file mode 100644 index 000000000..524a5c50c --- /dev/null +++ b/client/netsweep/quick_retry.go @@ -0,0 +1,39 @@ +package netsweep + +import ( + "time" + + "github.com/cenkalti/backoff/v4" + + "github.com/netbirdio/netbird/client/netstate" +) + +const quickRetryDelay = 200 * time.Millisecond + +type quickRetryBackoff struct { + backoff.BackOff + sweeper *Sweeper + netState *netstate.State + used bool +} + +func newQuickRetryBackoff(bo backoff.BackOff, sweeper *Sweeper, netState *netstate.State) *quickRetryBackoff { + return &quickRetryBackoff{ + BackOff: bo, + sweeper: sweeper, + netState: netState, + } +} + +func (b *quickRetryBackoff) NextBackOff() time.Duration { + if !b.used && b.sweeper.markedRecently() && b.netState.IsOnline() { + b.used = true + return quickRetryDelay + } + return b.BackOff.NextBackOff() +} + +func (b *quickRetryBackoff) Reset() { + b.used = false + b.BackOff.Reset() +} diff --git a/client/netsweep/quick_retry_test.go b/client/netsweep/quick_retry_test.go new file mode 100644 index 000000000..5505862c5 --- /dev/null +++ b/client/netsweep/quick_retry_test.go @@ -0,0 +1,58 @@ +package netsweep + +import ( + "context" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/stretchr/testify/assert" +) + +func TestQuickRetryAfterRecentMark(t *testing.T) { + sweeper := New() + sweeper.MarkNetworkChange() + + slow := backoff.NewConstantBackOff(5 * time.Second) + bo := sweeper.QuickRetryBackoff(context.Background(), slow, nil) + + assert.Equal(t, quickRetryDelay, bo.NextBackOff(), "first retry after a mark must be quick") + assert.Equal(t, 5*time.Second, bo.NextBackOff(), "second retry must fall back to the wrapped backoff") + + bo.Reset() + assert.Equal(t, quickRetryDelay, bo.NextBackOff(), "reset must re-arm the quick retry") +} + +func TestQuickRetryWithoutMarkKeepsSpread(t *testing.T) { + sweeper := New() + + slow := backoff.NewConstantBackOff(5 * time.Second) + bo := sweeper.QuickRetryBackoff(context.Background(), slow, nil) + + assert.Equal(t, 5*time.Second, bo.NextBackOff(), "without a mark the wrapped backoff decides") + + sweeper.mu.Lock() + sweeper.lastMark = time.Now().Add(-recentMarkWindow) + sweeper.mu.Unlock() + assert.Equal(t, 5*time.Second, bo.NextBackOff(), "a stale mark must not trigger the quick retry") +} + +func TestQuickRetryNilSweeperPassthrough(t *testing.T) { + var sweeper *Sweeper + + slow := backoff.NewConstantBackOff(5 * time.Second) + bo := sweeper.QuickRetryBackoff(context.Background(), slow, nil) + + assert.Equal(t, backoff.BackOff(slow), bo, "nil sweeper must return the backoff unchanged") +} + +func TestQuickRetryHonorsContext(t *testing.T) { + sweeper := New() + sweeper.MarkNetworkChange() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + bo := sweeper.QuickRetryBackoff(ctx, backoff.NewConstantBackOff(time.Millisecond), nil) + + assert.Equal(t, backoff.Stop, bo.NextBackOff(), "cancelled context must stop the retry loop") +} diff --git a/client/ssh/client/client.go b/client/ssh/client/client.go index 4180849cd..31143a4f4 100644 --- a/client/ssh/client/client.go +++ b/client/ssh/client/client.go @@ -313,21 +313,23 @@ func Dial(ctx context.Context, addr, user string, opts DialOptions) (*Client, er // dialSSH establishes an SSH connection without JWT authentication func dialSSH(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*Client, error) { + if config.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, config.Timeout) + defer cancel() + } + dialer := &net.Dialer{} conn, err := dialer.DialContext(ctx, network, addr) if err != nil { return nil, fmt.Errorf("dial %s: %w", addr, err) } - clientConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) + client, err := nbssh.Handshake(ctx, conn, addr, config) if err != nil { - if closeErr := conn.Close(); closeErr != nil { - log.Debugf("connection close after handshake failure: %v", closeErr) - } - return nil, fmt.Errorf("ssh handshake: %w", err) + return nil, err } - client := ssh.NewClient(clientConn, chans, reqs) return &Client{ client: client, }, nil diff --git a/client/ssh/client/terminal_unix.go b/client/ssh/client/terminal_unix.go index aaa3418f9..a963dc8be 100644 --- a/client/ssh/client/terminal_unix.go +++ b/client/ssh/client/terminal_unix.go @@ -12,6 +12,8 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" "golang.org/x/term" + + nbssh "github.com/netbirdio/netbird/client/ssh" ) func (c *Client) setupTerminalMode(ctx context.Context, session *ssh.Session) error { @@ -82,37 +84,7 @@ func (c *Client) setupTerminal(session *ssh.Session, fd int) error { return fmt.Errorf("get terminal size: %w", err) } - modes := ssh.TerminalModes{ - ssh.ECHO: 1, - ssh.TTY_OP_ISPEED: 14400, - ssh.TTY_OP_OSPEED: 14400, - // Ctrl+C - ssh.VINTR: 3, - // Ctrl+\ - ssh.VQUIT: 28, - // Backspace - ssh.VERASE: 127, - // Ctrl+U - ssh.VKILL: 21, - // Ctrl+D - ssh.VEOF: 4, - ssh.VEOL: 0, - ssh.VEOL2: 0, - // Ctrl+Q - ssh.VSTART: 17, - // Ctrl+S - ssh.VSTOP: 19, - // Ctrl+Z - ssh.VSUSP: 26, - // Ctrl+O - ssh.VDISCARD: 15, - // Ctrl+R - ssh.VREPRINT: 18, - // Ctrl+W - ssh.VWERASE: 23, - // Ctrl+V - ssh.VLNEXT: 22, - } + modes := nbssh.DefaultTerminalModes terminal := os.Getenv("TERM") if terminal == "" { diff --git a/client/ssh/client/terminal_windows.go b/client/ssh/client/terminal_windows.go index 462438317..c6156fc26 100644 --- a/client/ssh/client/terminal_windows.go +++ b/client/ssh/client/terminal_windows.go @@ -10,6 +10,8 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" + + nbssh "github.com/netbirdio/netbird/client/ssh" ) const ( @@ -80,28 +82,14 @@ func (c *Client) setupTerminalMode(_ context.Context, session *ssh.Session) erro w, h := c.getWindowsConsoleSize() modes := ssh.TerminalModes{ - ssh.ECHO: 1, - ssh.TTY_OP_ISPEED: 14400, - ssh.TTY_OP_OSPEED: 14400, - ssh.ICRNL: 1, - ssh.OPOST: 1, - ssh.ONLCR: 1, - ssh.ISIG: 1, - ssh.ICANON: 1, - ssh.VINTR: 3, // Ctrl+C - ssh.VQUIT: 28, // Ctrl+\ - ssh.VERASE: 127, // Backspace - ssh.VKILL: 21, // Ctrl+U - ssh.VEOF: 4, // Ctrl+D - ssh.VEOL: 0, - ssh.VEOL2: 0, - ssh.VSTART: 17, // Ctrl+Q - ssh.VSTOP: 19, // Ctrl+S - ssh.VSUSP: 26, // Ctrl+Z - ssh.VDISCARD: 15, // Ctrl+O - ssh.VWERASE: 23, // Ctrl+W - ssh.VLNEXT: 22, // Ctrl+V - ssh.VREPRINT: 18, // Ctrl+R + ssh.ICRNL: 1, + ssh.OPOST: 1, + ssh.ONLCR: 1, + ssh.ISIG: 1, + ssh.ICANON: 1, + } + for mode, value := range nbssh.DefaultTerminalModes { + modes[mode] = value } if err := session.RequestPty("xterm-256color", h, w, modes); err != nil { diff --git a/client/ssh/common.go b/client/ssh/common.go index 3f4f3e9d1..4ebf8842a 100644 --- a/client/ssh/common.go +++ b/client/ssh/common.go @@ -35,6 +35,19 @@ type HostKeyVerifier interface { VerifySSHHostKey(peerAddress string, key []byte) error } +// PeerKeyLookup returns the stored SSH host key for a peer address. +type PeerKeyLookup func(peerAddress string) ([]byte, bool) + +// VerifySSHHostKey implements HostKeyVerifier by looking up the stored key +// and comparing it against the presented key. +func (l PeerKeyLookup) VerifySSHHostKey(peerAddress string, presentedKey []byte) error { + storedKey, found := l(peerAddress) + if !found { + return ErrPeerNotFound + } + return VerifyHostKey(storedKey, presentedKey, peerAddress) +} + // DaemonHostKeyVerifier implements HostKeyVerifier using the NetBird daemon type DaemonHostKeyVerifier struct { client proto.DaemonServiceClient diff --git a/client/ssh/handshake.go b/client/ssh/handshake.go new file mode 100644 index 000000000..e78a806be --- /dev/null +++ b/client/ssh/handshake.go @@ -0,0 +1,45 @@ +package ssh + +import ( + "context" + "fmt" + "io" + "net" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" +) + +// Handshake runs the SSH client handshake on an already dialed conn and +// returns the resulting client. Dialing bounds only the TCP establishment; +// without a deadline on the socket a peer that accepts and then goes silent +// blocks the handshake forever, so the context deadline is applied to conn +// for the duration of the handshake. conn is closed on any error. +func Handshake(ctx context.Context, conn net.Conn, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { + if deadline, ok := ctx.Deadline(); ok { + if err := conn.SetDeadline(deadline); err != nil { + closeHandshake(conn, "conn after deadline error") + return nil, fmt.Errorf("set handshake deadline: %w", err) + } + } + + sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) + if err != nil { + closeHandshake(conn, "conn after handshake error") + return nil, fmt.Errorf("ssh handshake: %w", err) + } + + if err := conn.SetDeadline(time.Time{}); err != nil { + closeHandshake(sshConn, "ssh conn after deadline clear error") + return nil, fmt.Errorf("clear handshake deadline: %w", err) + } + + return ssh.NewClient(sshConn, chans, reqs), nil +} + +func closeHandshake(c io.Closer, label string) { + if err := c.Close(); err != nil { + log.Debugf("ssh: close %s: %v", label, err) + } +} diff --git a/client/ssh/proxy/proxy.go b/client/ssh/proxy/proxy.go index 721810edb..070515b57 100644 --- a/client/ssh/proxy/proxy.go +++ b/client/ssh/proxy/proxy.go @@ -610,13 +610,10 @@ func (p *SSHProxy) dialBackend(ctx context.Context, addr, user, jwtToken string) return nil, fmt.Errorf("connect to server: %w", err) } - clientConn, chans, reqs, err := cryptossh.NewClientConn(conn, addr, config) - if err != nil { - _ = conn.Close() - return nil, fmt.Errorf("SSH handshake: %w", err) - } + handshakeCtx, cancel := context.WithTimeout(ctx, sshHandshakeTimeout) + defer cancel() - return cryptossh.NewClient(clientConn, chans, reqs), nil + return nbssh.Handshake(handshakeCtx, conn, addr, config) } func (p *SSHProxy) verifyHostKey(hostname string, remote net.Addr, key cryptossh.PublicKey) error { diff --git a/client/ssh/session.go b/client/ssh/session.go new file mode 100644 index 000000000..999b6f251 --- /dev/null +++ b/client/ssh/session.go @@ -0,0 +1,84 @@ +package ssh + +import ( + "fmt" + "io" + + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" +) + +// DefaultTerminalModes are the PTY modes used by the interactive terminal clients. +var DefaultTerminalModes = ssh.TerminalModes{ + ssh.ECHO: 1, + ssh.TTY_OP_ISPEED: 14400, + ssh.TTY_OP_OSPEED: 14400, + ssh.VINTR: 3, // Ctrl+C + ssh.VQUIT: 28, // Ctrl+\ + ssh.VERASE: 127, // Backspace + ssh.VKILL: 21, // Ctrl+U + ssh.VEOF: 4, // Ctrl+D + ssh.VEOL: 0, + ssh.VEOL2: 0, + ssh.VSTART: 17, // Ctrl+Q + ssh.VSTOP: 19, // Ctrl+S + ssh.VSUSP: 26, // Ctrl+Z + ssh.VDISCARD: 15, // Ctrl+O + ssh.VREPRINT: 18, // Ctrl+R + ssh.VWERASE: 23, // Ctrl+W + ssh.VLNEXT: 22, // Ctrl+V +} + +// PTYSession is an interactive shell session with a PTY and its I/O pipes. +type PTYSession struct { + Session *ssh.Session + Stdin io.WriteCloser + Stdout io.Reader + Stderr io.Reader +} + +// StartPTYSession opens a session on the client, requests an xterm-256color PTY +// with the default terminal modes, wires up the I/O pipes and starts a shell. +// The session is closed on any error. +func StartPTYSession(client *ssh.Client, cols, rows int) (*PTYSession, error) { + session, err := client.NewSession() + if err != nil { + return nil, fmt.Errorf("new session: %w", err) + } + + pty, err := setupPTYSession(session, cols, rows) + if err != nil { + if closeErr := session.Close(); closeErr != nil { + log.Debugf("ssh: session close after setup error: %v", closeErr) + } + return nil, err + } + return pty, nil +} + +// setupPTYSession requests the PTY, opens the pipes and starts the shell on an +// already created session. +func setupPTYSession(session *ssh.Session, cols, rows int) (*PTYSession, error) { + if err := session.RequestPty("xterm-256color", rows, cols, DefaultTerminalModes); err != nil { + return nil, fmt.Errorf("request pty: %w", err) + } + + stdin, err := session.StdinPipe() + if err != nil { + return nil, fmt.Errorf("stdin pipe: %w", err) + } + stdout, err := session.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("stdout pipe: %w", err) + } + stderr, err := session.StderrPipe() + if err != nil { + return nil, fmt.Errorf("stderr pipe: %w", err) + } + + if err := session.Shell(); err != nil { + return nil, fmt.Errorf("start shell: %w", err) + } + + return &PTYSession{Session: session, Stdin: stdin, Stdout: stdout, Stderr: stderr}, nil +} diff --git a/client/system/info_android.go b/client/system/info_android.go index 3c71573bb..d4f479386 100644 --- a/client/system/info_android.go +++ b/client/system/info_android.go @@ -30,6 +30,11 @@ func GetInfo(ctx context.Context) *Info { kernelVersion = osInfo[2] } + addrs, err := networkAddresses() + if err != nil { + log.Warnf("discover network addresses: %s", err) + } + gio := &Info{ GoOS: runtime.GOOS, Kernel: kernel, @@ -41,6 +46,7 @@ func GetInfo(ctx context.Context) *Info { NetbirdVersion: version.NetbirdVersion(), UIVersion: extractUIVersion(ctx), KernelVersion: kernelVersion, + NetworkAddresses: addrs, SystemSerialNumber: serial(), SystemProductName: productModel(), SystemManufacturer: productManufacturer(), diff --git a/client/system/network_addr.go b/client/system/network_addr.go index 44260a938..505a6f0ea 100644 --- a/client/system/network_addr.go +++ b/client/system/network_addr.go @@ -1,4 +1,4 @@ -//go:build !ios +//go:build !ios && !android package system diff --git a/client/system/network_addr_android.go b/client/system/network_addr_android.go new file mode 100644 index 000000000..99a71e105 --- /dev/null +++ b/client/system/network_addr_android.go @@ -0,0 +1,89 @@ +package system + +import ( + "net/netip" + "strings" +) + +var iFaceDiscover IFaceDiscover + +type IFaceDiscover interface { + IFaces() (string, error) +} + +// SetIFaceDiscover configures the Android interface discovery provider. +func SetIFaceDiscover(discover IFaceDiscover) { + iFaceDiscover = discover +} + +func networkAddresses() ([]NetworkAddress, error) { + if iFaceDiscover == nil { + return nil, nil + } + ifaces, err := iFaceDiscover.IFaces() + if err != nil { + return nil, err + } + + var netAddresses []NetworkAddress + for _, line := range strings.Split(ifaces, "\n") { + addresses, ok := interfaceAddresses(line) + if !ok { + continue + } + for _, address := range addresses { + netAddr, ok := toNetworkAddress(address) + if !ok { + continue + } + if isDuplicated(netAddresses, netAddr) { + continue + } + netAddresses = append(netAddresses, netAddr) + } + } + return netAddresses, nil +} + +func interfaceAddresses(line string) ([]string, bool) { + parts := strings.Split(line, "|") + if len(parts) != 2 { + return nil, false + } + flags := strings.Fields(parts[0]) + if len(flags) != 8 { + return nil, false + } + up, loopback := flags[3], flags[5] + if up != "true" || loopback == "true" { + return nil, false + } + return strings.Fields(parts[1]), true +} + +func toNetworkAddress(address string) (NetworkAddress, bool) { + prefix, err := netip.ParsePrefix(address) + if err != nil { + return NetworkAddress{}, false + } + if prefix.Addr().Is4In6() { + if prefix.Bits() < 96 { + return NetworkAddress{}, false + } + prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96) + } + ip := prefix.Addr() + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsMulticast() { + return NetworkAddress{}, false + } + return NetworkAddress{NetIP: prefix}, true +} + +func isDuplicated(addresses []NetworkAddress, addr NetworkAddress) bool { + for _, duplicated := range addresses { + if duplicated.NetIP == addr.NetIP { + return true + } + } + return false +} diff --git a/client/system/network_addr_test.go b/client/system/network_addr_test.go index a5f9c4279..b0be40f0a 100644 --- a/client/system/network_addr_test.go +++ b/client/system/network_addr_test.go @@ -1,4 +1,4 @@ -//go:build !ios +//go:build !ios && !android package system diff --git a/client/wasm/internal/ssh/client.go b/client/wasm/internal/ssh/client.go index 9cfe65266..28ae95ec0 100644 --- a/client/wasm/internal/ssh/client.go +++ b/client/wasm/internal/ssh/client.go @@ -80,13 +80,12 @@ func (c *Client) Connect(host string, port int, username, jwtToken string, ipVer return fmt.Errorf("dial %s: %w", addr, err) } - sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) + sshClient, err := nbssh.Handshake(ctx, conn, addr, config) if err != nil { - closeWithLog(conn, "connection after handshake error") - return fmt.Errorf("SSH handshake: %w", err) + return err } - c.sshClient = ssh.NewClient(sshConn, chans, reqs) + c.sshClient = sshClient logrus.Infof("SSH: Connected to %s", addr) return nil @@ -119,57 +118,26 @@ func (c *Client) getAuthMethods(jwtToken string) ([]ssh.AuthMethod, error) { return []ssh.AuthMethod{ssh.PublicKeys(signer)}, nil } -// StartSession starts an SSH session with PTY +// StartSession starts an SSH session with PTY. It holds the client lock for +// the whole startup so Close cannot tear the client down mid-setup and the +// new session cannot be installed into an already closed client. func (c *Client) StartSession(cols, rows int) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.sshClient == nil { return fmt.Errorf("SSH client not connected") } - session, err := c.sshClient.NewSession() + pty, err := nbssh.StartPTYSession(c.sshClient, cols, rows) if err != nil { - return fmt.Errorf("create session: %w", err) + return err } - c.mu.Lock() - defer c.mu.Unlock() - c.session = session - - modes := ssh.TerminalModes{ - ssh.ECHO: 1, - ssh.TTY_OP_ISPEED: 14400, - ssh.TTY_OP_OSPEED: 14400, - ssh.VINTR: 3, - ssh.VQUIT: 28, - ssh.VERASE: 127, - } - - if err := session.RequestPty("xterm-256color", rows, cols, modes); err != nil { - closeWithLog(session, "session after PTY error") - return fmt.Errorf("PTY request: %w", err) - } - - c.stdin, err = session.StdinPipe() - if err != nil { - closeWithLog(session, "session after stdin error") - return fmt.Errorf("get stdin: %w", err) - } - - c.stdout, err = session.StdoutPipe() - if err != nil { - closeWithLog(session, "session after stdout error") - return fmt.Errorf("get stdout: %w", err) - } - - c.stderr, err = session.StderrPipe() - if err != nil { - closeWithLog(session, "session after stderr error") - return fmt.Errorf("get stderr: %w", err) - } - - if err := session.Shell(); err != nil { - closeWithLog(session, "session after shell error") - return fmt.Errorf("start shell: %w", err) - } + c.session = pty.Session + c.stdin = pty.Stdin + c.stdout = pty.Stdout + c.stderr = pty.Stderr logrus.Info("SSH: Session started with PTY") return nil diff --git a/go.mod b/go.mod index f119d4a92..beca63bfe 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/vishvananda/netlink v1.3.1 - golang.org/x/crypto v0.54.0 + golang.org/x/crypto v0.55.0 golang.org/x/sys v0.47.0 golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 @@ -127,9 +127,9 @@ require ( go.uber.org/zap v1.27.0 goauthentik.io/api/v3 v3.2023051.3 golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f - golang.org/x/mobile v0.0.0-20251113184115-a159579294ab - golang.org/x/mod v0.37.0 - golang.org/x/net v0.56.0 + golang.org/x/mobile v0.0.0-20260816165457-f98cc9b3c733 + golang.org/x/mod v0.39.0 + golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 golang.org/x/term v0.45.0 @@ -313,8 +313,8 @@ require ( go.opentelemetry.io/otel/trace v1.43.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/text v0.40.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.49.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index 31e8b5454..99adaa2cb 100644 --- a/go.sum +++ b/go.sum @@ -728,13 +728,13 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20251113184115-a159579294ab h1:Iqyc+2zr7aGyLuEadIm0KRJP0Wwt+fhlXLa51Fxf1+Q= -golang.org/x/mobile v0.0.0-20251113184115-a159579294ab/go.mod h1:Eq3Nh/5pFSWug2ohiudJ1iyU59SO78QFuh4qTTN++I0= +golang.org/x/mobile v0.0.0-20260816165457-f98cc9b3c733 h1:XKMObIaAElmkdO+4SQh1iCfzwciZHJi1OblnX9BED9k= +golang.org/x/mobile v0.0.0-20260816165457-f98cc9b3c733/go.mod h1:jMwjxoDSx9jqhNaZqPnr6nnKzb7cs+Dy1Czk7wdX+R8= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -744,8 +744,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74= +golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= @@ -764,8 +764,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= @@ -843,8 +843,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -858,8 +858,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/management/server/http/handlers/setup_keys/setupkeys_handler.go b/management/server/http/handlers/setup_keys/setupkeys_handler.go index d267b6eea..bb498f46b 100644 --- a/management/server/http/handlers/setup_keys/setupkeys_handler.go +++ b/management/server/http/handlers/setup_keys/setupkeys_handler.go @@ -64,6 +64,19 @@ func (h *handler) createSetupKey(w http.ResponseWriter, r *http.Request) { return } + // A one-off key can be used once, and GenerateSetupKey pins its usage limit + // at 1 whatever the request says. Silently overriding a caller that asked + // for a different number leaves them holding a key that does not do what + // they configured, and no way to find out except by using it. Only values + // above 1 are refused: usage_limit is a required field with no null, so 0 + // cannot be told apart from a caller that has nothing to say about it. + if types.SetupKeyType(req.Type) == types.SetupKeyOneOff && req.UsageLimit > 1 { + util.WriteError(r.Context(), status.Errorf(status.InvalidArgument, + "usage_limit %d is not valid for a one-off setup key, which can be used once; use type reusable for a key that can be used more than once", + req.UsageLimit), w) + return + } + expiresIn := time.Duration(req.ExpiresIn) * time.Second if expiresIn < 0 { diff --git a/management/server/http/handlers/setup_keys/setupkeys_handler_test.go b/management/server/http/handlers/setup_keys/setupkeys_handler_test.go index b137b6dd1..a9cfd4bd3 100644 --- a/management/server/http/handlers/setup_keys/setupkeys_handler_test.go +++ b/management/server/http/handlers/setup_keys/setupkeys_handler_test.go @@ -134,6 +134,40 @@ func TestSetupKeysHandlers(t *testing.T) { expectedBody: true, expectedSetupKey: expectedNewKey, }, + { + // A one-off key is used once. Asking for more used to be accepted + // and then quietly reduced to 1. + name: "Create One-Off Setup Key With Conflicting Usage Limit", + requestType: http.MethodPost, + requestPath: "/api/setup-keys", + requestBody: bytes.NewBuffer( + []byte(fmt.Sprintf("{\"name\":\"%s\",\"type\":\"one-off\",\"expires_in\":86400,\"usage_limit\":5}", newSetupKeyName))), + expectedStatus: http.StatusUnprocessableEntity, + expectedBody: false, + }, + { + // 0 is what a caller sends when it has nothing to say about the + // usage limit, since the field is required and has no null, so it + // has to keep working. + name: "Create One-Off Setup Key Without Usage Limit", + requestType: http.MethodPost, + requestPath: "/api/setup-keys", + requestBody: bytes.NewBuffer( + []byte(fmt.Sprintf("{\"name\":\"%s\",\"type\":\"one-off\",\"expires_in\":86400,\"usage_limit\":0}", newSetupKeyName))), + expectedStatus: http.StatusOK, + expectedBody: false, + }, + { + // Only one-off keys are constrained; a reusable key means what it + // says. + name: "Create Reusable Setup Key With Usage Limit", + requestType: http.MethodPost, + requestPath: "/api/setup-keys", + requestBody: bytes.NewBuffer( + []byte(fmt.Sprintf("{\"name\":\"%s\",\"type\":\"reusable\",\"expires_in\":86400,\"usage_limit\":5}", newSetupKeyName))), + expectedStatus: http.StatusOK, + expectedBody: false, + }, { name: "Update Setup Key", requestType: http.MethodPut, diff --git a/management/server/http/testing/integration/setupkeys_handler_integration_test.go b/management/server/http/testing/integration/setupkeys_handler_integration_test.go index 0d3aaac82..21ad9d347 100644 --- a/management/server/http/testing/integration/setupkeys_handler_integration_test.go +++ b/management/server/http/testing/integration/setupkeys_handler_integration_test.go @@ -136,7 +136,10 @@ func Test_SetupKeys_Create(t *testing.T) { }, }, { - name: "Create Setup Key as on-off with more than one usage", + // The key used to be created anyway, with its usage limit quietly + // reduced to 1, so the caller was told a key they had not asked for + // was what they asked for. + name: "Create Setup Key as one-off with more than one usage", requestType: http.MethodPost, requestPath: "/api/setup-keys", requestBody: &api.CreateSetupKeyRequest{ @@ -146,23 +149,7 @@ func Test_SetupKeys_Create(t *testing.T) { Type: "one-off", UsageLimit: 3, }, - expectedStatus: http.StatusOK, - expectedResponse: &api.SetupKey{ - AutoGroups: []string{}, - Ephemeral: false, - Expires: time.Time{}, - Id: "", - Key: "", - LastUsed: time.Time{}, - Name: testing_tools.NewKeyName, - Revoked: false, - State: "valid", - Type: "one-off", - UpdatedAt: time.Now(), - UsageLimit: 1, - UsedTimes: 0, - Valid: true, - }, + expectedStatus: http.StatusUnprocessableEntity, }, { name: "Create Setup Key with expiration in the past", diff --git a/shared/management/client/grpc.go b/shared/management/client/grpc.go index 81f25900a..cd250b5f7 100644 --- a/shared/management/client/grpc.go +++ b/shared/management/client/grpc.go @@ -21,6 +21,8 @@ import ( "google.golang.org/grpc/connectivity" nbgrpc "github.com/netbirdio/netbird/client/grpc" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/shared/management/domain" @@ -62,6 +64,13 @@ type GrpcClient struct { connStateCallbackLock sync.RWMutex serverURL string + // netState gates the stream retry loop on OS-reported network + // availability; nil (the default) disables gating. + netState *netstate.State + + // sweeper cuts the transport connections on network change; nil disables it. + sweeper *netsweep.Sweeper + // syncStreamErr holds the last Sync stream error, or nil while the stream // is established and healthy. GetServerKey succeeds even when the peer // cannot sync (e.g. the server returns "settings not found"), so the @@ -111,16 +120,43 @@ func MaxRecvMsgSize() int { return size } +// Option configures optional GrpcClient behavior. +type Option func(*GrpcClient) + +// WithNetworkState injects the OS network availability state that gates the +// stream retry loop; without it gating is disabled. +func WithNetworkState(netState *netstate.State) Option { + return func(c *GrpcClient) { c.netState = netState } +} + +// WithSweeper injects the network change sweeper. +func WithSweeper(sweeper *netsweep.Sweeper) Option { + return func(c *GrpcClient) { c.sweeper = sweeper } +} + // NewClient creates a new client to Management service -func NewClient(ctx context.Context, addr string, ourPrivateKey wgtypes.Key, tlsEnabled bool) (*GrpcClient, error) { - var conn *grpc.ClientConn +func NewClient(ctx context.Context, addr string, ourPrivateKey wgtypes.Key, tlsEnabled bool, opts ...Option) (*GrpcClient, error) { + // Options apply before dialing: the sweeper must wrap the first connection too. + c := &GrpcClient{ + key: ourPrivateKey, + ctx: ctx, + connStateCallbackLock: sync.RWMutex{}, + serverURL: addr, + } + for _, opt := range opts { + opt(c) + } var extraOpts []grpc.DialOption if maxSize := MaxRecvMsgSize(); maxSize > 0 { extraOpts = append(extraOpts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxSize))) log.Infof("management gRPC max receive message size set to %d bytes", maxSize) } + if c.sweeper != nil { + extraOpts = append(extraOpts, nbgrpc.WithSweeper(c.sweeper)) + } + var conn *grpc.ClientConn operation := func() error { var err error conn, err = nbgrpc.CreateConnection(ctx, addr, tlsEnabled, wsproxy.ManagementComponent, extraOpts...) @@ -136,16 +172,9 @@ func NewClient(ctx context.Context, addr string, ourPrivateKey wgtypes.Key, tlsE return nil, err } - realClient := proto.NewManagementServiceClient(conn) - - return &GrpcClient{ - key: ourPrivateKey, - realClient: realClient, - ctx: ctx, - conn: conn, - connStateCallbackLock: sync.RWMutex{}, - serverURL: addr, - }, nil + c.conn = conn + c.realClient = proto.NewManagementServiceClient(conn) + return c, nil } // GetServerURL returns the management server URL @@ -206,16 +235,33 @@ func (c *GrpcClient) withMgmtStream( ctx context.Context, handler func(ctx context.Context, serverPubKey wgtypes.Key, backOff backoff.BackOff) error, ) error { - backOff := defaultBackoff(ctx) + backOff := c.sweeper.QuickRetryBackoff(ctx, defaultBackoff(ctx), c.netState) operation := func() error { - log.Debugf("management connection state %v", c.conn.GetState()) - connState := c.conn.GetState() + // suspend reconnect attempts while the OS reports no usable network. + // Wait only errors on a cancelled context, which means shutdown, so + // stop the loop without reporting a failure. + if waited, err := c.netState.Wait(ctx); err != nil { + log.Debugf("management connection context has been canceled while offline, this usually indicates shutdown") + return nil //nolint:nilerr // a cancelled context means shutdown, not a retryable failure + } else if waited { + backOff.Reset() + } + connState := c.conn.GetState() + log.Debugf("management connection state %v", connState) if connState == connectivity.Shutdown { return backoff.Permanent(fmt.Errorf("connection to management has been shut down")) - } else if !(connState == connectivity.Ready || connState == connectivity.Idle) { + } + if !(connState == connectivity.Ready || connState == connectivity.Idle) { + // A dial may already be in flight (e.g. the other stream triggered + // it after a network change); wait for it to settle and proceed if + // the channel became usable, instead of burning a backoff round on + // a successful dial. A failed dial errors out as before. c.conn.WaitForStateChange(ctx, connState) - return fmt.Errorf("connection to management is not ready and in %s state", connState) + connState = c.conn.GetState() + if !(connState == connectivity.Ready || connState == connectivity.Idle) { + return fmt.Errorf("connection to management is not ready and in %s state", connState) + } } serverPubKey, err := c.getServerPublicKey() @@ -227,7 +273,7 @@ func (c *GrpcClient) withMgmtStream( return handler(ctx, *serverPubKey, backOff) } - err := backoff.Retry(operation, backOff) + err := nbgrpc.Retry(ctx, operation, backOff, c.netState) if err != nil { log.Warnf("exiting the Management service connection retry loop due to the unrecoverable error: %s", err) } diff --git a/shared/relay/client/client.go b/shared/relay/client/client.go index 8d4aa6020..4fb30b8d9 100644 --- a/shared/relay/client/client.go +++ b/shared/relay/client/client.go @@ -14,6 +14,7 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/netsweep" auth "github.com/netbirdio/netbird/shared/relay/auth/hmac" "github.com/netbirdio/netbird/shared/relay/client/dialer" netErr "github.com/netbirdio/netbird/shared/relay/client/dialer/net" @@ -184,6 +185,10 @@ type Client struct { // datagram-sized transport is avoided on subsequent connects. Shared via // the manager. transportFallback *transportFallback + + // sweeper cuts the relay connection on network change; the read loop + // reports the disconnect and the guard reconnects. Shared via the manager. + sweeper *netsweep.Sweeper // datagramFallbackTriggered guards a single fallback per connection so a // burst of oversized datagrams triggers one reconnect, not many. datagramFallbackTriggered atomic.Bool @@ -393,6 +398,12 @@ func (c *Client) Close() error { } func (c *Client) connect(ctx context.Context) (*RelayAddr, error) { + // A sweep cancels this context, so a dial started on the old network + // aborts instead of waiting out its handshake timeout. + dial := c.sweeper.StartDial(ctx) + defer dial.Release() + ctx = dial.Ctx() + mode := transportModeFromEnv() dialers := c.getDialers(mode) @@ -417,12 +428,19 @@ func (c *Client) connect(ctx context.Context) (*RelayAddr, error) { return nil, fmt.Errorf("dial via FQDN: %w", err) } } - c.relayConn = conn - c.datagramFallbackTriggered.Store(false) + // Read the transport off the concrete connection: the sweeper's wrapper + // embeds net.Conn only, so it does not promote Protocol(). if tc, ok := conn.(transportConn); ok { c.transport = tc.Protocol() } + conn, err := dial.WrapConn(conn) + if err != nil { + return nil, fmt.Errorf("register connection: %w", err) + } + c.relayConn = conn + c.datagramFallbackTriggered.Store(false) + instanceURL, err := c.handShake(ctx) if err != nil { cErr := conn.Close() diff --git a/shared/relay/client/guard.go b/shared/relay/client/guard.go index d18534d9d..a62f8772d 100644 --- a/shared/relay/client/guard.go +++ b/shared/relay/client/guard.go @@ -7,9 +7,22 @@ import ( "github.com/cenkalti/backoff/v4" log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/netstate" ) -const defaultMaxBackoffInterval = 60 * time.Second +const ( + defaultMaxBackoffInterval = 60 * time.Second + + // quickReconnectBudget bounds how long a quick reconnect waits for the + // network before handing the retry over to the ticker. + quickReconnectBudget = 1500 * time.Millisecond + + // verdictSettleWindow is how long an online verdict must hold before it + // is trusted: the disconnect often precedes the OS offline flag by a few + // milliseconds. + verdictSettleWindow = 200 * time.Millisecond +) // Guard manage the reconnection tries to the Relay server in case of disconnection event. type Guard struct { @@ -22,14 +35,19 @@ type Guard struct { // attempts. maxBackoffInterval time.Duration + // netState gates reconnect attempts on OS-reported network availability; + // nil disables gating. + netState *netstate.State + // lastErr is the error from the most recent failed reconnect attempt, // surfaced as the home relay status while disconnected. lastErr atomic.Pointer[error] } // NewGuard creates a new guard for the relay client. A non-positive -// maxBackoffInterval falls back to defaultMaxBackoffInterval. -func NewGuard(sp *ServerPicker, maxBackoffInterval time.Duration) *Guard { +// maxBackoffInterval falls back to defaultMaxBackoffInterval. A nil netState +// disables network availability gating. +func NewGuard(sp *ServerPicker, maxBackoffInterval time.Duration, netState *netstate.State) *Guard { if maxBackoffInterval <= 0 { maxBackoffInterval = defaultMaxBackoffInterval } @@ -38,6 +56,7 @@ func NewGuard(sp *ServerPicker, maxBackoffInterval time.Duration) *Guard { OnReconnected: make(chan struct{}, 1), serverPicker: sp, maxBackoffInterval: maxBackoffInterval, + netState: netState, } return g } @@ -70,11 +89,21 @@ func (g *Guard) StartReconnectTrys(ctx context.Context, relayClient *Client) { // start a ticker to pick a new server ticker := g.exponentTicker(ctx) - defer ticker.Stop() + defer func() { + ticker.Stop() + }() for { select { case <-ticker.C: + // suspend reconnect attempts while the OS reports no usable network + if waited, err := g.netState.Wait(ctx); err != nil { + return + } else if waited { + ticker.Stop() + ticker = g.exponentTicker(ctx) + continue + } if err := g.retry(ctx); err != nil { log.Errorf("failed to pick new Relay server: %s", err) g.setLastError(err) @@ -100,7 +129,12 @@ func (g *Guard) tryToQuickReconnect(parentCtx context.Context, rc *Client) bool return false } - if cancelled := waiteBeforeRetry(parentCtx); !cancelled { + if ok := g.waitForNetwork(parentCtx); !ok { + return false + } + + // Still offline after the budget: leave the retry to the ticker. + if !g.netState.IsOnline() { return false } @@ -166,14 +200,47 @@ func (g *Guard) exponentTicker(ctx context.Context) *backoff.Ticker { return backoff.NewTicker(bo) } -func waiteBeforeRetry(ctx context.Context) bool { - timer := time.NewTimer(1500 * time.Millisecond) - defer timer.Stop() +// waitForNetwork waits out the settle window while online, or waits for the +// network to return while offline, within the budget. Returns false when ctx +// is cancelled. Without an injected netState it degrades to a fixed +// budget-long sleep, the pre-netstate behavior. +func (g *Guard) waitForNetwork(ctx context.Context) bool { + budget := time.NewTimer(quickReconnectBudget) + defer budget.Stop() - select { - case <-timer.C: - return true - case <-ctx.Done(): - return false + settleWindow := verdictSettleWindow + if g.netState == nil { + settleWindow = quickReconnectBudget + } + settle := time.NewTimer(settleWindow) + defer settle.Stop() + + for { + // Channel first, flag second: a flip in between still fires the channel. + changedCh := g.netState.Changed() + if g.netState.IsOnline() { + select { + case <-settle.C: + return true + case <-changedCh: + case <-ctx.Done(): + return false + } + } else { + select { + case <-budget.C: + return true + case <-changedCh: + case <-ctx.Done(): + return false + } + } + if !settle.Stop() { + select { + case <-settle.C: + default: + } + } + settle.Reset(settleWindow) } } diff --git a/shared/relay/client/guard_test.go b/shared/relay/client/guard_test.go new file mode 100644 index 000000000..0e05783e0 --- /dev/null +++ b/shared/relay/client/guard_test.go @@ -0,0 +1,30 @@ +package client + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/netbirdio/netbird/client/netstate" +) + +func TestWaitForNetworkSettlesAfterOutage(t *testing.T) { + ns := netstate.New() + ns.Set(false) + g := NewGuard(nil, 0, ns) + + const outage = 2 * verdictSettleWindow + start := time.Now() + go func() { + time.Sleep(outage) + ns.Set(true) + }() + + ok := g.waitForNetwork(context.Background()) + elapsed := time.Since(start) + + assert.True(t, ok, "recovered network must let the quick reconnect proceed") + assert.GreaterOrEqual(t, elapsed, outage+verdictSettleWindow, "reconnect must wait a full settle window after the network returns") +} diff --git a/shared/relay/client/manager.go b/shared/relay/client/manager.go index 2f2839d94..80e38ae2d 100644 --- a/shared/relay/client/manager.go +++ b/shared/relay/client/manager.go @@ -12,6 +12,8 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" relayAuth "github.com/netbirdio/netbird/shared/relay/auth/hmac" ) @@ -65,6 +67,17 @@ func WithMaxBackoffInterval(d time.Duration) ManagerOption { return func(m *Manager) { m.maxBackoffInterval = d } } +// WithNetworkState injects the OS network availability state that gates the +// reconnect guard; without it reconnect attempts are not gated. +func WithNetworkState(netState *netstate.State) ManagerOption { + return func(m *Manager) { m.netState = netState } +} + +// WithSweeper injects the network change sweeper. +func WithSweeper(sweeper *netsweep.Sweeper) ManagerOption { + return func(m *Manager) { m.sweeper = sweeper } +} + // Manager is a manager for the relay client instances. It establishes one persistent connection to the given relay URL // and automatically reconnect to them in case disconnection. // The manager also manage temporary relay connection. If a client wants to communicate with a client on a @@ -92,6 +105,8 @@ type Manager struct { mtu uint16 maxBackoffInterval time.Duration + netState *netstate.State + sweeper *netsweep.Sweeper cleanupInterval time.Duration keepUnusedServerTime time.Duration @@ -128,8 +143,9 @@ func NewManager(ctx context.Context, serverURLs []string, peerID string, mtu uin for _, opt := range opts { opt(m) } + m.serverPicker.Sweeper = m.sweeper m.serverPicker.ServerURLs.Store(serverURLs) - m.reconnectGuard = NewGuard(m.serverPicker, m.maxBackoffInterval) + m.reconnectGuard = NewGuard(m.serverPicker, m.maxBackoffInterval, m.netState) return m } @@ -354,6 +370,7 @@ func (m *Manager) openConnVia(ctx context.Context, serverAddress, peerKey string relayClient := NewClientWithServerIP(serverAddress, serverIP, m.tokenStore, m.peerID, m.mtu) relayClient.SetTransportFallback(m.transportFallback) + relayClient.sweeper = m.sweeper err := relayClient.Connect(m.ctx) if err != nil { rt.Lock() diff --git a/shared/relay/client/picker.go b/shared/relay/client/picker.go index bb721e4ad..72789fadc 100644 --- a/shared/relay/client/picker.go +++ b/shared/relay/client/picker.go @@ -9,6 +9,7 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/netsweep" auth "github.com/netbirdio/netbird/shared/relay/auth/hmac" ) @@ -30,6 +31,7 @@ type ServerPicker struct { MTU uint16 ConnectionTimeout time.Duration TransportFallback *transportFallback + Sweeper *netsweep.Sweeper } func (sp *ServerPicker) PickServer(parentCtx context.Context) (*Client, error) { @@ -73,6 +75,7 @@ func (sp *ServerPicker) startConnection(ctx context.Context, resultChan chan con log.Infof("try to connecting to relay server: %s", url) relayClient := NewClient(url, sp.TokenStore, sp.PeerID, sp.MTU) relayClient.SetTransportFallback(sp.TransportFallback) + relayClient.sweeper = sp.Sweeper err := relayClient.Connect(ctx) resultChan <- connResult{ RelayClient: relayClient, diff --git a/shared/signal/client/grpc.go b/shared/signal/client/grpc.go index a07867263..73c482e8f 100644 --- a/shared/signal/client/grpc.go +++ b/shared/signal/client/grpc.go @@ -19,6 +19,8 @@ import ( "google.golang.org/grpc/status" nbgrpc "github.com/netbirdio/netbird/client/grpc" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/shared/management/client" "github.com/netbirdio/netbird/shared/signal/proto" @@ -65,6 +67,13 @@ type GrpcClient struct { connStateCallback ConnStateNotifier connStateCallbackLock sync.RWMutex + // netState gates the Receive retry loop on OS-reported network + // availability; nil (the default) disables gating. + netState *netstate.State + + // sweeper cuts the transport connections on network change; nil disables it. + sweeper *netsweep.Sweeper + onReconnectedListenerFn func() decryptionWorker *Worker @@ -88,13 +97,43 @@ type GrpcClient struct { watchdogWg sync.WaitGroup } -// NewClient creates a new Signal client -func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled bool) (*GrpcClient, error) { - var conn *grpc.ClientConn +// Option configures optional GrpcClient behavior. +type Option func(*GrpcClient) +// WithNetworkState injects the OS network availability state that gates the +// Receive retry loop; without it gating is disabled. +func WithNetworkState(netState *netstate.State) Option { + return func(c *GrpcClient) { c.netState = netState } +} + +// WithSweeper injects the network change sweeper. +func WithSweeper(sweeper *netsweep.Sweeper) Option { + return func(c *GrpcClient) { c.sweeper = sweeper } +} + +// NewClient creates a new Signal client +func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled bool, opts ...Option) (*GrpcClient, error) { + // Options apply before dialing: the sweeper must wrap the first connection too. + c := &GrpcClient{ + ctx: ctx, + key: key, + mux: sync.Mutex{}, + status: StreamDisconnected, + connStateCallbackLock: sync.RWMutex{}, + } + for _, opt := range opts { + opt(c) + } + + var extraOpts []grpc.DialOption + if c.sweeper != nil { + extraOpts = append(extraOpts, nbgrpc.WithSweeper(c.sweeper)) + } + + var conn *grpc.ClientConn operation := func() error { var err error - conn, err = nbgrpc.CreateConnection(ctx, addr, tlsEnabled, wsproxy.SignalComponent) + conn, err = nbgrpc.CreateConnection(ctx, addr, tlsEnabled, wsproxy.SignalComponent, extraOpts...) if err != nil { return fmt.Errorf("create connection: %w", err) } @@ -109,15 +148,9 @@ func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled boo log.Debugf("connected to Signal Service: %v", conn.Target()) - return &GrpcClient{ - realClient: proto.NewSignalExchangeClient(conn), - ctx: ctx, - signalConn: conn, - key: key, - mux: sync.Mutex{}, - status: StreamDisconnected, - connStateCallbackLock: sync.RWMutex{}, - }, nil + c.signalConn = conn + c.realClient = proto.NewSignalExchangeClient(conn) + return c, nil } func (c *GrpcClient) StreamConnected() bool { @@ -165,19 +198,36 @@ func defaultBackoff(ctx context.Context) backoff.BackOff { // The connection retry logic will try to reconnect for 30 min and if wasn't successful will propagate the error to the function caller. func (c *GrpcClient) Receive(ctx context.Context, msgHandler func(msg *proto.Message) error) error { - var backOff = defaultBackoff(ctx) + backOff := c.sweeper.QuickRetryBackoff(ctx, defaultBackoff(ctx), c.netState) operation := func() error { + // suspend reconnect attempts while the OS reports no usable network. + // Wait only errors on a cancelled context, which means shutdown, so + // stop the loop without reporting a failure. + if waited, err := c.netState.Wait(ctx); err != nil { + log.Debugf("signal connection context has been canceled while offline, this usually indicates shutdown") + return nil + } else if waited { + backOff.Reset() + } c.notifyStreamDisconnected() - log.Debugf("signal connection state %v", c.signalConn.GetState()) connState := c.signalConn.GetState() + log.Debugf("signal connection state %v", connState) if connState == connectivity.Shutdown { return backoff.Permanent(fmt.Errorf("connection to signal has been shut down")) - } else if !(connState == connectivity.Ready || connState == connectivity.Idle) { + } + if !(connState == connectivity.Ready || connState == connectivity.Idle) { + // A dial may already be in flight (e.g. triggered by another RPC + // after a network change); wait for it to settle and proceed if + // the channel became usable, instead of burning a backoff round on + // a successful dial. A failed dial errors out as before. c.signalConn.WaitForStateChange(ctx, connState) - return fmt.Errorf("connection to signal is not ready and in %s state", connState) + connState = c.signalConn.GetState() + if !(connState == connectivity.Ready || connState == connectivity.Idle) { + return fmt.Errorf("connection to signal is not ready and in %s state", connState) + } } // connect to Signal stream identifying ourselves with a public WireGuard key @@ -231,7 +281,7 @@ func (c *GrpcClient) Receive(ctx context.Context, msgHandler func(msg *proto.Mes return nil } - err := backoff.Retry(operation, backOff) + err := nbgrpc.Retry(ctx, operation, backOff, c.netState) if err != nil { log.Errorf("exiting the Signal service connection retry loop due to the unrecoverable error: %v", err) return err