From ecfbd686b807f2e47295e1dfad3cd86b47af3c84 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 18 Aug 2026 16:49:13 +0000 Subject: [PATCH 1/3] [client, android] Expose ssh functionality for Android (#7156) Adds an SSHClient gomobile binding so the Android app can run an SSH session over the tunnel with a PTY, exposed through a listener interface for the in-app terminal. Server type is auto-detected from the SSH banner, which selects the auth path: JWT device-code flow, NetBird key, or a regular server (NetBird key first, then password). Host keys are verified against the peer registry for NetBird servers and trust-on-first-use for regular ones. --- client/android/login.go | 57 +- client/android/profile_prefs.go | 38 ++ client/android/ssh_client.go | 649 +++++++++++++++++++ client/android/ssh_known_hosts.go | 168 +++++ client/android/ssh_sessions.go | 104 +++ client/embed/embed.go | 9 +- client/internal/auth/auth.go | 27 +- client/internal/auth/oauth.go | 8 +- client/internal/profilemanager/prefs.go | 130 ++++ client/internal/profilemanager/prefs_test.go | 138 ++++ client/internal/profilemanager/service.go | 5 + client/ios/NetBirdSDK/login.go | 2 +- client/ssh/client/client.go | 14 +- client/ssh/client/terminal_unix.go | 34 +- client/ssh/client/terminal_windows.go | 32 +- client/ssh/common.go | 13 + client/ssh/handshake.go | 45 ++ client/ssh/proxy/proxy.go | 9 +- client/ssh/session.go | 84 +++ client/wasm/internal/ssh/client.go | 62 +- 20 files changed, 1470 insertions(+), 158 deletions(-) create mode 100644 client/android/profile_prefs.go create mode 100644 client/android/ssh_client.go create mode 100644 client/android/ssh_known_hosts.go create mode 100644 client/android/ssh_sessions.go create mode 100644 client/internal/profilemanager/prefs.go create mode 100644 client/internal/profilemanager/prefs_test.go create mode 100644 client/ssh/handshake.go create mode 100644 client/ssh/session.go diff --git a/client/android/login.go b/client/android/login.go index 897b1561e..24c911eb5 100644 --- a/client/android/login.go +++ b/client/android/login.go @@ -191,40 +191,49 @@ func (a *Auth) login(urlOpener URLOpener, isAndroidTV bool) error { return nil } -// loginHintSetter is implemented by both concrete flows (PKCE and device code) -// but absent from the OAuthFlow interface, hence the assertion below — the same -// way internal/auth wires it in authenticateWithPKCEFlow. -type loginHintSetter interface { - SetLoginHint(hint string) -} - func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener, isAndroidTV bool) (*auth.TokenInfo, error) { - oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV) + oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV, profileLoginHint(a.cfgPath)) if err != nil { return nil, fmt.Errorf("failed to get OAuth flow: %v", err) } - // An empty hint is deliberate, not a fallback: a fresh profile leaves the - // choice to the IdP. Switching accounts is done by switching or removing - // profiles, not by logging out — logout keeps the email. - if a.cfgPath != "" { - if hint := readProfileEmail(a.cfgPath); hint != "" { - if setter, ok := oAuthFlow.(loginHintSetter); ok { - setter.SetLoginHint(hint) - } - } + return runOAuthFlow(a.ctx, oAuthFlow, urlOpener, nil) +} + +// profileLoginHint returns the stored account email for the profile at cfgPath. +// An empty hint is deliberate, not a fallback: a fresh profile leaves the +// choice to the IdP. Switching accounts is done by switching or removing +// profiles, not by logging out — logout keeps the email. +func profileLoginHint(cfgPath string) string { + if cfgPath == "" { + return "" + } + return readProfileEmail(cfgPath) +} + +// runOAuthFlow drives an already acquired OAuth flow to a token: requests the +// flow info, presents the verification URL through the opener and waits for +// the browser round-trip. Open is called synchronously — it is what marks the +// surface as opened on the client side, and a fast token's OnLoginSuccess is +// a no-op until it has, so the dismissal would be dropped rather than +// delayed. Openers must therefore not block: they post their UI work and +// return. onWaiting, when set, runs after the URL is shown, right before the +// blocking wait. +func runOAuthFlow(ctx context.Context, flow auth.OAuthFlow, urlOpener URLOpener, onWaiting func()) (*auth.TokenInfo, error) { + flowInfo, err := flow.RequestAuthInfo(ctx) + if err != nil { + return nil, fmt.Errorf("request auth info: %w", err) } - flowInfo, err := oAuthFlow.RequestAuthInfo(context.TODO()) - if err != nil { - return nil, fmt.Errorf("getting a request OAuth flow info failed: %v", err) + urlOpener.Open(flowInfo.VerificationURIComplete, flowInfo.UserCode) + + if onWaiting != nil { + onWaiting() } - go urlOpener.Open(flowInfo.VerificationURIComplete, flowInfo.UserCode) - - tokenInfo, err := oAuthFlow.WaitToken(a.ctx, flowInfo) + tokenInfo, err := flow.WaitToken(ctx, flowInfo) if err != nil { - return nil, fmt.Errorf("waiting for browser login failed: %v", err) + return nil, fmt.Errorf("wait for token: %w", err) } return &tokenInfo, nil diff --git a/client/android/profile_prefs.go b/client/android/profile_prefs.go new file mode 100644 index 000000000..9c1fd307b --- /dev/null +++ b/client/android/profile_prefs.go @@ -0,0 +1,38 @@ +//go:build android + +package android + +import ( + "fmt" + + "github.com/netbirdio/netbird/client/internal/profilemanager" +) + +type prefsStore interface { + Get(namespace string, v any) (bool, error) + Put(namespace string, v any) error +} + +type profilePrefs struct { + prefs *profilemanager.Prefs +} + +func newProfilePrefs(configDir, profileID string) (*profilePrefs, error) { + if configDir == "" || profileID == "" { + return nil, fmt.Errorf("profile prefs require a config dir and profile ID") + } + pm := NewProfileManager(configDir) + prefs, err := pm.serviceMgr.ProfilePrefs(profilemanager.ID(profileID), androidUsername) + if err != nil { + return nil, fmt.Errorf("resolve profile prefs: %w", err) + } + return &profilePrefs{prefs: prefs}, nil +} + +func (p *profilePrefs) Get(namespace string, v any) (bool, error) { + return p.prefs.Get(namespace, v) +} + +func (p *profilePrefs) Put(namespace string, v any) error { + return p.prefs.Put(namespace, v) +} diff --git a/client/android/ssh_client.go b/client/android/ssh_client.go new file mode 100644 index 000000000..2822b6539 --- /dev/null +++ b/client/android/ssh_client.go @@ -0,0 +1,649 @@ +//go:build android + +package android + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" + gossh "golang.org/x/crypto/ssh" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/auth" + "github.com/netbirdio/netbird/client/internal/profilemanager" + nbssh "github.com/netbirdio/netbird/client/ssh" + "github.com/netbirdio/netbird/client/ssh/detection" +) + +const ( + sshDialTimeout = 30 * time.Second + sshDetectionTimeout = 5 * time.Second +) + +// PasswordRequiredMarker tells Java to prompt for a password and retry. It is +// a string because gomobile flattens errors to their message, so a sentinel +// value would not survive the binding. +const PasswordRequiredMarker = "netbird-ssh-password-required" + +// HostKeyUnknownMarker tells Java to show the fingerprint and, on confirmation, +// retry with TrustHostKey set. The presented fingerprint is appended after the +// marker so the prompt can display it and the retry can guard against a key +// that changed between the two connects. Only regular (non-NetBird) servers +// reach this: NetBird peers verify against the registry. +const HostKeyUnknownMarker = "netbird-ssh-hostkey-unknown" + +var ( + errPasswordRequired = errors.New(PasswordRequiredMarker) + errClientClosed = errors.New("ssh client closed") +) + +// errHostKeyUnknown carries the presented fingerprint so Connect can build the +// marker message the Java side parses. +type errHostKeyUnknown struct { + fingerprint string +} + +func (e *errHostKeyUnknown) Error() string { + return HostKeyUnknownMarker + ":" + e.fingerprint +} + +// SSHTerminalListener receives SSH session events. It is implemented in Java. +// +// All callbacks are invoked from goroutines and may run concurrently with each +// other; the implementation must be safe to call from any thread. +type SSHTerminalListener interface { + OnConnected() + OnData(data []byte) + OnClose(reason string) + OnError(message string) +} + +// SSHClient is a NetBird-aware SSH client exposed to Java via gomobile. +// +// It dials through the running NetBird tunnel and runs a standard SSH session +// on top with PTY enabled. Host-key verification uses the NetBird-provided +// peer SSH host keys, identical to the desktop client. +type SSHClient struct { + nb *Client + mu sync.Mutex + listener SSHTerminalListener + urlOpener URLOpener + + sshClient *gossh.Client + session *gossh.Session + stdin io.WriteCloser + closed bool + + // gen identifies the current connection attempt. Connect and Close bump it, + // so an in-flight dial or a reader left over from a previous connection + // finds itself stale and stays silent instead of publishing OnConnected or + // OnClose for a connection the caller already abandoned. + gen uint64 + dialCancel context.CancelFunc + + // knownHostsConfigDir and knownHostsProfile locate the TOFU store for + // regular SSH servers in the profile's preferences. Java supplies them, + // since an overlay IP is a different host under a different profile. Empty + // until set: without them a regular server cannot be verified and Connect + // refuses one. + knownHostsConfigDir string + knownHostsProfile string + // trustHostKey carries the fingerprint the user confirmed on a previous + // attempt, so the retry accepts exactly that key and persists it. + trustHostKey string +} + +// NewSSHClient creates a new SSH client bound to the running NetBird Client. +func NewSSHClient(c *Client) *SSHClient { + return &SSHClient{nb: c} +} + +// SetListener registers the Java listener. Must be called before Connect to +// receive any events. +func (s *SSHClient) SetListener(l SSHTerminalListener) { + s.mu.Lock() + s.listener = l + s.mu.Unlock() +} + +// SetURLOpener registers the Java URL opener used to display the device-code +// authorization page in a Custom Tabs window when the target peer requires +// JWT authentication. Must be set before Connect to be effective. +func (s *SSHClient) SetURLOpener(opener URLOpener) { + s.mu.Lock() + s.urlOpener = opener + s.mu.Unlock() +} + +// SetKnownHostsStore points the TOFU host-key store at a profile's preferences. +// Must be set before connecting to a regular SSH server; without it such a +// server cannot be verified and Connect refuses one. +func (s *SSHClient) SetKnownHostsStore(configDir, profileID string) { + s.mu.Lock() + s.knownHostsConfigDir = configDir + s.knownHostsProfile = profileID + s.mu.Unlock() +} + +// TrustHostKey records the fingerprint the user confirmed for a regular server, +// so the next Connect accepts that exact key and adds it to the known-hosts +// store. Passing a fingerprint that no longer matches makes the connect fail +// rather than trust a key that changed since the prompt. +func (s *SSHClient) TrustHostKey(fingerprint string) { + s.mu.Lock() + s.trustHostKey = fingerprint + s.mu.Unlock() +} + +// Connect dials the SSH server through the NetBird tunnel and performs the +// SSH handshake. It auto-detects the server type via SSH banner inspection +// and selects the appropriate authentication path: +// +// - NetBird-SSH server requiring JWT: launches the OAuth 2.0 device-code +// flow, opens the verification URL through the registered URLOpener, and +// uses the resulting token as the SSH password. Host-key verification +// uses the NetBird peer registry. +// - NetBird-SSH server without JWT: authenticates with the NetBird SSH +// private key. Host-key verification uses the NetBird peer registry. +// - Regular SSH server (e.g. OpenSSH): authenticates with the NetBird key +// first (so a user-installed NetBird public key works), then falls back +// to the supplied password if non-empty. Host-key verification is +// trust-on-first-use against the per-profile known-hosts store. +// +// The password parameter is only consulted for regular SSH servers. +func (s *SSHClient) Connect(host string, port int, user, password string) error { + if port < 1 || port > 65535 { + return fmt.Errorf("invalid port: %d", port) + } + + cfg, cfgPath, cc := s.nb.authSnapshot() + if cc == nil { + return errors.New("netbird client not running") + } + if cfg == nil { + return errors.New("netbird config not loaded") + } + engine := cc.Engine() + if engine == nil { + return errors.New("netbird engine not available") + } + + s.mu.Lock() + s.gen++ + gen := s.gen + s.mu.Unlock() + + serverType := detectServerType(host, port) + log.Debugf("SSH server type: %s", serverType) + + authMethods, hostKeyCallback, err := s.buildAuth(cfg, cfgPath, engine, serverType, password) + if err != nil { + return err + } + + clientConfig := &gossh.ClientConfig{ + User: user, + Auth: authMethods, + HostKeyCallback: hostKeyCallback, + Timeout: sshDialTimeout, + } + err = s.dialAndHandshake(gen, host, port, clientConfig) + + // An unknown host key is a prompt, not a failure: return the marker intact + // (rootCause would unwrap it) so Java can show the fingerprint and retry. + var unknownHost *errHostKeyUnknown + if errors.As(err, &unknownHost) { + return errors.New(unknownHost.Error()) + } + + // A regular server may still accept a password, so let the caller ask for + // one instead of failing. NetBird servers never use a password, so a + // failure there is genuine. + if err != nil && serverType != detection.ServerTypeNetBirdJWT && + serverType != detection.ServerTypeNetBirdNoJWT && isAuthFailure(err) && + passwordCouldHelp(err, password != "") { + return errPasswordRequired + } + if err != nil { + return rootCause(err) + } + return nil +} + +// StartSession requests a PTY and starts an interactive shell. Output from +// the session is forwarded to the listener via OnData. +func (s *SSHClient) StartSession(cols, rows int) error { + err := s.startSession(cols, rows) + if err != nil { + log.Infof("SSH: start session failed: %v", err) + return rootCause(err) + } + return nil +} + +// Write sends data to the SSH session stdin. +func (s *SSHClient) Write(data []byte) error { + s.mu.Lock() + stdin := s.stdin + s.mu.Unlock() + if stdin == nil { + return errors.New("ssh session not started") + } + if _, err := stdin.Write(data); err != nil { + return fmt.Errorf("write stdin: %w", err) + } + return nil +} + +// Resize updates the PTY window size. +func (s *SSHClient) Resize(cols, rows int) error { + s.mu.Lock() + session := s.session + s.mu.Unlock() + if session == nil { + return errors.New("ssh session not started") + } + return session.WindowChange(rows, cols) +} + +// Reset makes a closed client usable for another Connect: Close leaves the +// one-shot guard set, and clearing it lets the same client back a reconnect. +func (s *SSHClient) Reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = false +} + +// Close terminates the SSH session and underlying connection. Safe to call +// multiple times. +func (s *SSHClient) Close() error { + s.mu.Lock() + s.gen++ + if s.dialCancel != nil { + s.dialCancel() + s.dialCancel = nil + } + sshClient := s.sshClient + session := s.session + stdin := s.stdin + s.sshClient = nil + s.session = nil + s.stdin = nil + notify := !s.closed + s.closed = true + listener := s.listener + s.mu.Unlock() + + if stdin != nil { + if err := stdin.Close(); err != nil { + log.Debugf("ssh: stdin close: %v", err) + } + } + if session != nil { + if err := session.Close(); err != nil && !errors.Is(err, io.EOF) { + log.Debugf("ssh: session close: %v", err) + } + } + var firstErr error + if sshClient != nil { + if err := sshClient.Close(); err != nil { + firstErr = err + } + } + if notify && listener != nil { + listener.OnClose("closed by client") + } + return firstErr +} + +func (s *SSHClient) startSession(cols, rows int) error { + log.Debugf("SSH: starting session %dx%d", cols, rows) + s.mu.Lock() + sshClient := s.sshClient + gen := s.gen + s.mu.Unlock() + + if sshClient == nil { + return errors.New("ssh client not connected") + } + + pty, err := nbssh.StartPTYSession(sshClient, cols, rows) + if err != nil { + return err + } + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + closeQuiet(pty.Session, "stale session") + return errClientClosed + } + s.session = pty.Session + s.stdin = pty.Stdin + s.mu.Unlock() + + readerDone := make(chan string, 2) + go func() { readerDone <- s.readLoop(pty.Stdout, "stdout") }() + go func() { readerDone <- s.readLoop(pty.Stderr, "stderr") }() + go func() { + reason := <-readerDone + if second := <-readerDone; reason == "" { + reason = second + } + s.notifyClose(gen, reason) + }() + log.Debug("SSH: session started, shell running") + return nil +} + +func (s *SSHClient) buildAuth(cfg *profilemanager.Config, cfgPath string, engine *internal.Engine, + serverType detection.ServerType, password string) ([]gossh.AuthMethod, gossh.HostKeyCallback, error) { + + switch serverType { + case detection.ServerTypeNetBirdJWT: + token, err := s.requestJWTToken(cfg, cfgPath) + if err != nil { + return nil, nil, fmt.Errorf("jwt: %w", err) + } + auths := []gossh.AuthMethod{gossh.Password(token)} + return auths, nbssh.CreateHostKeyCallback(nbssh.PeerKeyLookup(engine.GetPeerSSHKey)), nil + + case detection.ServerTypeNetBirdNoJWT: + if cfg.SSHKey == "" { + return nil, nil, errors.New("no NetBird SSH key available") + } + signer, err := gossh.ParsePrivateKey([]byte(cfg.SSHKey)) + if err != nil { + return nil, nil, fmt.Errorf("parse netbird ssh key: %w", err) + } + auths := []gossh.AuthMethod{gossh.PublicKeys(signer)} + return auths, nbssh.CreateHostKeyCallback(nbssh.PeerKeyLookup(engine.GetPeerSSHKey)), nil + + case detection.ServerTypeRegular: + var auths []gossh.AuthMethod + if cfg.SSHKey != "" { + if signer, err := gossh.ParsePrivateKey([]byte(cfg.SSHKey)); err == nil { + auths = append(auths, gossh.PublicKeys(signer)) + } else { + log.Debugf("ssh: parse netbird key for regular auth: %v", err) + } + } + if password != "" { + pw := password + auths = append(auths, gossh.Password(pw)) + auths = append(auths, gossh.KeyboardInteractive(func(_, _ string, questions []string, _ []bool) ([]string, error) { + answers := make([]string, len(questions)) + for i := range questions { + answers[i] = pw + } + return answers, nil + })) + } + if len(auths) == 0 { + // Nothing to offer at all: ask for a password rather than failing, + // so the caller can retry once the user supplies one. + return nil, nil, errPasswordRequired + } + callback, err := s.tofuHostKeyCallback() + if err != nil { + return nil, nil, err + } + return auths, callback, nil + + default: + return nil, nil, fmt.Errorf("unsupported SSH server type: %v", serverType) + } +} + +// tofuHostKeyCallback verifies a regular server's host key against the +// per-profile known-hosts store. An unknown host returns errHostKeyUnknown so +// Java can show the fingerprint and, once confirmed, retry with the key +// trusted; a changed key is rejected outright, as OpenSSH does. When the user +// has confirmed a fingerprint, the callback accepts exactly that key and +// appends it to the store. +func (s *SSHClient) tofuHostKeyCallback() (gossh.HostKeyCallback, error) { + s.mu.Lock() + configDir := s.knownHostsConfigDir + profileID := s.knownHostsProfile + trusted := s.trustHostKey + s.mu.Unlock() + + if configDir == "" || profileID == "" { + return nil, errors.New("no known-hosts store configured for regular SSH") + } + + store, err := openKnownHostsStore(configDir, profileID) + if err != nil { + return nil, fmt.Errorf("load known-hosts store: %w", err) + } + + return func(hostname string, remote net.Addr, key gossh.PublicKey) error { + verdict, err := store.verify(hostname, remote, key) + if err != nil { + return err + } + if verdict == hostKeyMatched { + return nil + } + if verdict == hostKeyChanged { + return fmt.Errorf("SSH host key changed for %s (possible attack)", hostname) + } + + fingerprint := gossh.FingerprintSHA256(key) + if trusted == "" { + return &errHostKeyUnknown{fingerprint: fingerprint} + } + if trusted != fingerprint { + return fmt.Errorf("SSH host key changed since it was confirmed for %s", hostname) + } + if err := store.append(hostname, remote, key); err != nil { + return fmt.Errorf("persist trusted host key: %w", err) + } + // The confirmation is spent: now that the key is stored, a later + // reconnect must verify against the file, not re-accept this fingerprint. + s.mu.Lock() + s.trustHostKey = "" + s.mu.Unlock() + return nil + }, nil +} + +func (s *SSHClient) requestJWTToken(cfg *profilemanager.Config, cfgPath string) (string, error) { + s.mu.Lock() + urlOpener := s.urlOpener + s.mu.Unlock() + if urlOpener == nil { + return "", errors.New("URL opener not configured for JWT auth") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + flow, err := auth.NewOAuthFlow(ctx, cfg, false, true, profileLoginHint(cfgPath)) + if err != nil { + return "", fmt.Errorf("create oauth flow: %w", err) + } + + // The status callback covers the browser round-trip, which would + // otherwise leave the terminal blank. + tokenInfo, err := runOAuthFlow(ctx, flow, urlOpener, func() { + s.notifyStatus("Waiting for browser authentication...") + }) + if err != nil { + return "", err + } + + token := tokenInfo.GetTokenToUse() + if token == "" { + return "", errors.New("empty token returned by IdP") + } + + // Tells the client the browser round-trip is over so it can dismiss the + // surface it opened, the same way the login and session-extend flows do. + // Without it the Custom Tab stays in front of the terminal even though the + // token has already been collected. + urlOpener.OnLoginSuccess() + + return token, nil +} + +func (s *SSHClient) dialAndHandshake(gen uint64, host string, port int, clientConfig *gossh.ClientConfig) error { + addr := net.JoinHostPort(host, strconv.Itoa(port)) + ctx, cancel := context.WithTimeout(context.Background(), sshDialTimeout) + defer cancel() + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + return errClientClosed + } + s.dialCancel = cancel + s.mu.Unlock() + + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + return fmt.Errorf("dial %s: %w", addr, err) + } + + client, err := nbssh.Handshake(ctx, conn, addr, clientConfig) + if err != nil { + return err + } + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + closeQuiet(client, "stale ssh client") + return errClientClosed + } + s.sshClient = client + listener := s.listener + s.mu.Unlock() + + if listener != nil { + listener.OnConnected() + } + return nil +} + +func (s *SSHClient) readLoop(r io.Reader, name string) string { + buf := make([]byte, 4096) + for { + n, err := r.Read(buf) + if n > 0 { + s.mu.Lock() + listener := s.listener + s.mu.Unlock() + if listener != nil { + chunk := make([]byte, n) + copy(chunk, buf[:n]) + listener.OnData(chunk) + } + } + if err != nil { + // EOF is a normal shell exit, so report it without a reason. + if errors.Is(err, io.EOF) { + return "" + } + log.Debugf("ssh %s read: %v", name, err) + return rootCause(err).Error() + } + } +} + +// notifyStatus writes a progress line to the terminal through the normal +// output path, so long steps are visible while nothing else is arriving. +func (s *SSHClient) notifyStatus(text string) { + s.mu.Lock() + listener := s.listener + s.mu.Unlock() + if listener != nil { + listener.OnData([]byte("\r\n\x1b[33m" + text + "\x1b[0m\r\n")) + } +} + +func (s *SSHClient) notifyClose(gen uint64, reason string) { + s.mu.Lock() + if gen != s.gen || s.closed { + s.mu.Unlock() + return + } + s.closed = true + listener := s.listener + s.mu.Unlock() + if listener != nil { + listener.OnClose(reason) + } +} + +func closeQuiet(c io.Closer, label string) { + if c == nil { + return + } + if err := c.Close(); err != nil && !errors.Is(err, io.EOF) { + log.Debugf("ssh: close %s: %v", label, err) + } +} + +func detectServerType(host string, port int) detection.ServerType { + ctx, cancel := context.WithTimeout(context.Background(), sshDetectionTimeout) + defer cancel() + + dialer := &net.Dialer{} + serverType, err := detection.DetectSSHServerType(ctx, dialer, host, port) + if err != nil { + log.Debugf("ssh: server detection failed: %v (assuming regular SSH)", err) + return detection.ServerTypeRegular + } + return serverType +} + +// rootCause returns the innermost error of a %w chain, so the terminal shows +// "i/o timeout" rather than every layer that added context on the way up. +func rootCause(err error) error { + for { + // A joined error has no single root, so keep it as-is. + if _, ok := err.(interface{ Unwrap() []error }); ok { + return err + } + next := errors.Unwrap(err) + if next == nil { + return err + } + err = next + } +} + +// isAuthFailure distinguishes credential rejection from dial, timeout and +// host-key errors, which retrying with a password would not fix. +func isAuthFailure(err error) bool { + if errors.Is(err, errPasswordRequired) { + return true + } + var partial *gossh.PartialSuccessError + if errors.As(err, &partial) { + return true + } + return strings.Contains(err.Error(), "unable to authenticate") +} + +// passwordCouldHelp reports whether prompting for a password again can change +// the outcome. gossh lists a method under "attempted methods" only when the +// server offered it, so a supplied password that was never attempted means the +// server does not accept passwords and the real error should surface instead. +func passwordCouldHelp(err error, passwordOffered bool) bool { + if !passwordOffered { + return true + } + msg := err.Error() + return strings.Contains(msg, "password") || strings.Contains(msg, "keyboard-interactive") +} diff --git a/client/android/ssh_known_hosts.go b/client/android/ssh_known_hosts.go new file mode 100644 index 000000000..eea90fd32 --- /dev/null +++ b/client/android/ssh_known_hosts.go @@ -0,0 +1,168 @@ +//go:build android + +package android + +import ( + "bytes" + "net" + "strconv" + "strings" + "sync" + + gossh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +const knownHostsNamespace = "ssh" + +const ( + hostKeyUnknown hostKeyVerdict = iota + hostKeyMatched + hostKeyChanged +) + +var knownHostsMu sync.Mutex + +type hostKeyVerdict uint8 + +type knownHostsSection struct { + KnownHosts []string `json:"knownHosts"` +} + +type knownHostsStore struct { + prefs prefsStore +} + +// RemoveKnownHost deletes every known-hosts entry for host:port from the +// profile's store, so a host trusted for a session that is being deleted does +// not linger. Java calls this only once no session targets that host, so a +// shared host stays trusted. A missing entry is not an error: the goal state +// is "absent". +func RemoveKnownHost(configDir, profileID, host string, port int) error { + store, err := openKnownHostsStore(configDir, profileID) + if err != nil { + return err + } + return store.removeHost(host, port) +} + +func openKnownHostsStore(configDir, profileID string) (*knownHostsStore, error) { + prefs, err := newProfilePrefs(configDir, profileID) + if err != nil { + return nil, err + } + return &knownHostsStore{prefs: prefs}, nil +} + +func (st *knownHostsStore) verify(hostname string, remote net.Addr, key gossh.PublicKey) (hostKeyVerdict, error) { + lines, err := st.lines() + if err != nil { + return hostKeyUnknown, err + } + targets := knownHostsTargets(hostname, remote) + + verdict := hostKeyUnknown + for _, line := range lines { + pubKey, ok := knownHostsLineKey(line, targets) + if !ok { + continue + } + if pubKey.Type() == key.Type() && bytes.Equal(pubKey.Marshal(), key.Marshal()) { + return hostKeyMatched, nil + } + verdict = hostKeyChanged + } + return verdict, nil +} + +func (st *knownHostsStore) append(hostname string, remote net.Addr, key gossh.PublicKey) error { + line := knownhosts.Line(knownHostsTargets(hostname, remote), key) + + knownHostsMu.Lock() + defer knownHostsMu.Unlock() + + lines, err := st.lines() + if err != nil { + return err + } + return st.prefs.Put(knownHostsNamespace, knownHostsSection{KnownHosts: append(lines, line)}) +} + +func (st *knownHostsStore) removeHost(host string, port int) error { + target := knownhosts.Normalize(net.JoinHostPort(host, strconv.Itoa(port))) + + knownHostsMu.Lock() + defer knownHostsMu.Unlock() + + lines, err := st.lines() + if err != nil { + return err + } + kept := make([]string, 0, len(lines)) + for _, line := range lines { + if knownHostsLineMatches(line, target) { + continue + } + kept = append(kept, line) + } + if len(kept) == len(lines) { + return nil + } + return st.prefs.Put(knownHostsNamespace, knownHostsSection{KnownHosts: kept}) +} + +func (st *knownHostsStore) lines() ([]string, error) { + var section knownHostsSection + if _, err := st.prefs.Get(knownHostsNamespace, §ion); err != nil { + return nil, err + } + return section.KnownHosts, nil +} + +func knownHostsTargets(hostname string, remote net.Addr) []string { + targets := []string{knownhosts.Normalize(hostname)} + if remote != nil { + if normalized := knownhosts.Normalize(remote.String()); normalized != targets[0] { + targets = append(targets, normalized) + } + } + return targets +} + +func knownHostsLineKey(line string, targets []string) (gossh.PublicKey, bool) { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + return nil, false + } + _, hosts, pubKey, _, _, err := gossh.ParseKnownHosts([]byte(trimmed)) + if err != nil { + return nil, false + } + for _, host := range hosts { + for _, target := range targets { + if host == target { + return pubKey, true + } + } + } + return nil, false +} + +// knownHostsLineMatches reports whether a known-hosts line's address list +// contains the normalized target. Comment and blank lines never match. +func knownHostsLineMatches(line, target string) bool { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + return false + } + fields := strings.Fields(trimmed) + if len(fields) == 0 { + return false + } + for _, addr := range strings.Split(fields[0], ",") { + if addr == target { + return true + } + } + return false +} diff --git a/client/android/ssh_sessions.go b/client/android/ssh_sessions.go new file mode 100644 index 000000000..44b5464e9 --- /dev/null +++ b/client/android/ssh_sessions.go @@ -0,0 +1,104 @@ +//go:build android + +package android + +const ( + sshSessionsNamespace = "ssh-sessions" + maxStoredSSHSessions = 50 +) + +type sshSessionRecord struct { + ID string `json:"id"` + Host string `json:"host"` + Port int `json:"port"` + User string `json:"user"` +} + +type sshSessionsSection struct { + Sessions []sshSessionRecord `json:"sessions"` +} + +// SSHSessionEntry is one stored SSH session, without any credential. +type SSHSessionEntry struct { + ID string + Host string + Port int + User string +} + +// SSHSessionArray wraps stored SSH sessions for gomobile compatibility. +type SSHSessionArray struct { + items []*SSHSessionEntry +} + +// NewSSHSessionArray creates an empty session array to fill via Add. +func NewSSHSessionArray() *SSHSessionArray { + return &SSHSessionArray{} +} + +// Add appends a session entry, oldest first. +func (a *SSHSessionArray) Add(id, host string, port int, user string) { + a.items = append(a.items, &SSHSessionEntry{ID: id, Host: host, Port: port, User: user}) +} + +// Length returns the number of entries. +func (a *SSHSessionArray) Length() int { + return len(a.items) +} + +// Get returns the entry at index i, or nil when out of range. +func (a *SSHSessionArray) Get(i int) *SSHSessionEntry { + if i < 0 || i >= len(a.items) { + return nil + } + return a.items[i] +} + +// SSHSessionStore reads and writes a profile's stored SSH sessions. +type SSHSessionStore struct { + prefs prefsStore +} + +// NewSSHSessionStore opens the session store of the given profile. +func NewSSHSessionStore(configDir, profileID string) (*SSHSessionStore, error) { + prefs, err := newProfilePrefs(configDir, profileID) + if err != nil { + return nil, err + } + return &SSHSessionStore{prefs: prefs}, nil +} + +// Load returns the stored sessions, oldest first. +func (s *SSHSessionStore) Load() (*SSHSessionArray, error) { + var section sshSessionsSection + if _, err := s.prefs.Get(sshSessionsNamespace, §ion); err != nil { + return nil, err + } + + out := NewSSHSessionArray() + for _, record := range section.Sessions { + if record.ID == "" || record.Host == "" { + continue + } + out.Add(record.ID, record.Host, record.Port, record.User) + } + return out, nil +} + +// Save replaces the stored sessions, keeping only the newest entries when the +// list exceeds the storage cap. +func (s *SSHSessionStore) Save(sessions *SSHSessionArray) error { + var items []*SSHSessionEntry + if sessions != nil { + items = sessions.items + } + if len(items) > maxStoredSSHSessions { + items = items[len(items)-maxStoredSSHSessions:] + } + + records := make([]sshSessionRecord, 0, len(items)) + for _, item := range items { + records = append(records, sshSessionRecord{ID: item.ID, Host: item.Host, Port: item.Port, User: item.User}) + } + return s.prefs.Put(sshSessionsNamespace, sshSessionsSection{Sessions: records}) +} diff --git a/client/embed/embed.go b/client/embed/embed.go index 99a6b8229..1b2d84d7e 100644 --- a/client/embed/embed.go +++ b/client/embed/embed.go @@ -21,7 +21,7 @@ import ( "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/profilemanager" - sshcommon "github.com/netbirdio/netbird/client/ssh" + nbssh "github.com/netbirdio/netbird/client/ssh" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/shared/management/domain" mgmProto "github.com/netbirdio/netbird/shared/management/proto" @@ -521,12 +521,7 @@ func (c *Client) VerifySSHHostKey(peerAddress string, key []byte) error { return err } - storedKey, found := engine.GetPeerSSHKey(peerAddress) - if !found { - return sshcommon.ErrPeerNotFound - } - - return sshcommon.VerifyHostKey(storedKey, key, peerAddress) + return nbssh.PeerKeyLookup(engine.GetPeerSSHKey).VerifySSHHostKey(peerAddress, key) } // SetPerformance retunes a running Client. Only PreallocatedBuffersPerPool diff --git a/client/internal/auth/auth.go b/client/internal/auth/auth.go index 153727a6c..b3a9e1158 100644 --- a/client/internal/auth/auth.go +++ b/client/internal/auth/auth.go @@ -138,26 +138,37 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) { // GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection // This avoids creating a new connection to the management server -func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool) (OAuthFlow, error) { +func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, hint string) (OAuthFlow, error) { var flow OAuthFlow - var err error - err = a.withRetry(ctx, func(client *mgm.GrpcClient) error { + err := a.withRetry(ctx, func(client *mgm.GrpcClient) error { if forceDeviceAuth { - flow, err = a.getDeviceFlow(client) - return err + deviceFlow, err := a.getDeviceFlow(client) + if err != nil { + return err + } + deviceFlow.SetLoginHint(hint) + flow = deviceFlow + return nil } // Try PKCE flow first - flow, err = a.getPKCEFlow(client) + pkceFlow, err := a.getPKCEFlow(client) if err != nil { // If PKCE not supported, try Device flow if s, ok := status.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) { - flow, err = a.getDeviceFlow(client) - return err + deviceFlow, err := a.getDeviceFlow(client) + if err != nil { + return err + } + deviceFlow.SetLoginHint(hint) + flow = deviceFlow + return nil } return err } + pkceFlow.SetLoginHint(hint) + flow = pkceFlow return nil }) diff --git a/client/internal/auth/oauth.go b/client/internal/auth/oauth.go index a50a2ce6f..91329c98b 100644 --- a/client/internal/auth/oauth.go +++ b/client/internal/auth/oauth.go @@ -97,9 +97,7 @@ func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config return nil, fmt.Errorf("getting pkce authorization flow info failed with error: %v", err) } - if hint != "" { - pkceFlowInfo.SetLoginHint(hint) - } + pkceFlowInfo.SetLoginHint(hint) return pkceFlowInfo, nil } @@ -127,9 +125,7 @@ func authenticateWithDeviceCodeFlow(ctx context.Context, config *profilemanager. } } - if hint != "" { - deviceFlowInfo.SetLoginHint(hint) - } + deviceFlowInfo.SetLoginHint(hint) return deviceFlowInfo, nil } diff --git a/client/internal/profilemanager/prefs.go b/client/internal/profilemanager/prefs.go new file mode 100644 index 000000000..5613b0be3 --- /dev/null +++ b/client/internal/profilemanager/prefs.go @@ -0,0 +1,130 @@ +package profilemanager + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/netbirdio/netbird/util" +) + +const prefsFileSuffix = ".prefs.json" + +var prefsMu sync.Mutex + +// Prefs is a namespaced per-profile preference store backed by a single JSON +// file next to the profile config; it is deleted together with the profile. +type Prefs struct { + path string +} + +// ProfilePrefs returns the preference store of the profile identified by id. +func (s *ServiceManager) ProfilePrefs(id ID, username string) (*Prefs, error) { + if !IsValidProfileFilenameStem(id) { + return nil, fmt.Errorf("invalid profile ID: %q", id) + } + if id == defaultProfileName { + return &Prefs{path: filepath.Join(filepath.Dir(DefaultConfigPath), id.String()+prefsFileSuffix)}, nil + } + configDir, err := s.getConfigDir(username) + if err != nil { + return nil, fmt.Errorf("get config directory for user %s: %w", username, err) + } + return &Prefs{path: filepath.Join(configDir, id.String()+prefsFileSuffix)}, nil +} + +// Get unmarshals the namespace section into v and reports whether it exists. +func (p *Prefs) Get(namespace string, v any) (bool, error) { + if namespace == "" { + return false, fmt.Errorf("empty prefs namespace") + } + + prefsMu.Lock() + defer prefsMu.Unlock() + + sections, err := readPrefsFile(p.path) + if err != nil { + return false, err + } + raw, ok := sections[namespace] + if !ok { + return false, nil + } + if err := json.Unmarshal(raw, v); err != nil { + return false, fmt.Errorf("decode prefs namespace %q: %w", namespace, err) + } + return true, nil +} + +// Put stores v as the namespace section, replacing any previous value. +func (p *Prefs) Put(namespace string, v any) error { + if namespace == "" { + return fmt.Errorf("empty prefs namespace") + } + raw, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("encode prefs namespace %q: %w", namespace, err) + } + + prefsMu.Lock() + defer prefsMu.Unlock() + + sections, err := readPrefsFile(p.path) + if err != nil { + return err + } + sections[namespace] = raw + return writePrefsFile(p.path, sections) +} + +// Remove deletes the namespace section; a missing one is not an error. +func (p *Prefs) Remove(namespace string) error { + if namespace == "" { + return fmt.Errorf("empty prefs namespace") + } + + prefsMu.Lock() + defer prefsMu.Unlock() + + sections, err := readPrefsFile(p.path) + if err != nil { + return err + } + if _, ok := sections[namespace]; !ok { + return nil + } + delete(sections, namespace) + return writePrefsFile(p.path, sections) +} + +func removePrefsFile(path string) error { + prefsMu.Lock() + defer prefsMu.Unlock() + return os.Remove(path) +} + +func readPrefsFile(path string) (map[string]json.RawMessage, error) { + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return map[string]json.RawMessage{}, nil + } + if err != nil { + return nil, fmt.Errorf("read prefs: %w", err) + } + + sections := map[string]json.RawMessage{} + if err := json.Unmarshal(data, §ions); err != nil { + return nil, fmt.Errorf("decode prefs: %w", err) + } + return sections, nil +} + +func writePrefsFile(path string, sections map[string]json.RawMessage) error { + if err := util.WriteJsonWithRestrictedPermission(context.Background(), path, sections); err != nil { + return fmt.Errorf("write prefs: %w", err) + } + return nil +} diff --git a/client/internal/profilemanager/prefs_test.go b/client/internal/profilemanager/prefs_test.go new file mode 100644 index 000000000..692ade70f --- /dev/null +++ b/client/internal/profilemanager/prefs_test.go @@ -0,0 +1,138 @@ +package profilemanager + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testPrefsSection struct { + Mode uint8 `json:"mode"` + Dest string `json:"dest"` +} + +func TestProfilePrefs_RoundTrip(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + + require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 2, Dest: "/tmp/x"})) + require.NoError(t, prefs.Put("other", map[string]int{"n": 1})) + + var got testPrefsSection + found, err := prefs.Get("filedrop", &got) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, testPrefsSection{Mode: 2, Dest: "/tmp/x"}, got) + + var other map[string]int + found, err = prefs.Get("other", &other) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, map[string]int{"n": 1}, other) + }) +} + +func TestProfilePrefs_GetMissingNamespace(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + + var got testPrefsSection + found, err := prefs.Get("filedrop", &got) + require.NoError(t, err) + assert.False(t, found) + }) +} + +func TestProfilePrefs_RemoveNamespace(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + + require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 1})) + require.NoError(t, prefs.Put("other", map[string]int{"n": 1})) + require.NoError(t, prefs.Remove("filedrop")) + require.NoError(t, prefs.Remove("missing")) + + var got testPrefsSection + found, err := prefs.Get("filedrop", &got) + require.NoError(t, err) + assert.False(t, found) + + var other map[string]int + found, err = prefs.Get("other", &other) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, map[string]int{"n": 1}, other) + }) +} + +func TestProfilePrefs_RejectsInvalidID(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + _, err := sm.ProfilePrefs("../escape", username) + assert.Error(t, err) + }) +} + +func TestProfilePrefs_RejectsEmptyNamespace(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + + _, err = prefs.Get("", &testPrefsSection{}) + assert.Error(t, err) + assert.Error(t, prefs.Put("", testPrefsSection{})) + assert.Error(t, prefs.Remove("")) + }) +} + +func TestProfilePrefs_DefaultProfile(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + prefs, err := sm.ProfilePrefs(defaultProfileName, username) + require.NoError(t, err) + + require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 1})) + + expected := filepath.Join(filepath.Dir(DefaultConfigPath), "default"+prefsFileSuffix) + _, err = os.Stat(expected) + require.NoError(t, err) + }) +} + +func TestRemoveProfile_DeletesPrefsFile(t *testing.T) { + withTestSM(t, func(sm *ServiceManager, username string) { + created, err := sm.AddProfile("work", username) + require.NoError(t, err) + + prefs, err := sm.ProfilePrefs(created.ID, username) + require.NoError(t, err) + require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 2})) + + configDir, err := sm.getConfigDir(username) + require.NoError(t, err) + prefsPath := filepath.Join(configDir, created.ID.String()+prefsFileSuffix) + _, err = os.Stat(prefsPath) + require.NoError(t, err) + + require.NoError(t, sm.RemoveProfile(created.ID, username)) + _, err = os.Stat(prefsPath) + assert.True(t, errors.Is(err, os.ErrNotExist), "prefs file should be removed") + }) +} diff --git a/client/internal/profilemanager/service.go b/client/internal/profilemanager/service.go index 696a60310..ec287f01a 100644 --- a/client/internal/profilemanager/service.go +++ b/client/internal/profilemanager/service.go @@ -420,6 +420,11 @@ func (s *ServiceManager) RemoveProfile(id ID, username string) error { log.Warnf("failed to remove profile state file %s: %v", stateFile, err) } + prefsFile := filepath.Join(filepath.Dir(target.Path), id.String()+prefsFileSuffix) + if err := removePrefsFile(prefsFile); err != nil && !os.IsNotExist(err) { + log.Warnf("failed to remove profile prefs file %s: %v", prefsFile, err) + } + return nil } diff --git a/client/ios/NetBirdSDK/login.go b/client/ios/NetBirdSDK/login.go index 6cba0c411..42a575359 100644 --- a/client/ios/NetBirdSDK/login.go +++ b/client/ios/NetBirdSDK/login.go @@ -323,7 +323,7 @@ func (a *Auth) login(urlOpener URLOpener, forceDeviceAuth bool, deviceName strin const authInfoRequestTimeout = 30 * time.Second func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener, forceDeviceAuth bool) (*auth.TokenInfo, error) { - oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, forceDeviceAuth) + oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, forceDeviceAuth, "") if err != nil { return nil, fmt.Errorf("failed to get OAuth flow: %v", err) } diff --git a/client/ssh/client/client.go b/client/ssh/client/client.go index 4180849cd..31143a4f4 100644 --- a/client/ssh/client/client.go +++ b/client/ssh/client/client.go @@ -313,21 +313,23 @@ func Dial(ctx context.Context, addr, user string, opts DialOptions) (*Client, er // dialSSH establishes an SSH connection without JWT authentication func dialSSH(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*Client, error) { + if config.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, config.Timeout) + defer cancel() + } + dialer := &net.Dialer{} conn, err := dialer.DialContext(ctx, network, addr) if err != nil { return nil, fmt.Errorf("dial %s: %w", addr, err) } - clientConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) + client, err := nbssh.Handshake(ctx, conn, addr, config) if err != nil { - if closeErr := conn.Close(); closeErr != nil { - log.Debugf("connection close after handshake failure: %v", closeErr) - } - return nil, fmt.Errorf("ssh handshake: %w", err) + return nil, err } - client := ssh.NewClient(clientConn, chans, reqs) return &Client{ client: client, }, nil diff --git a/client/ssh/client/terminal_unix.go b/client/ssh/client/terminal_unix.go index aaa3418f9..a963dc8be 100644 --- a/client/ssh/client/terminal_unix.go +++ b/client/ssh/client/terminal_unix.go @@ -12,6 +12,8 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" "golang.org/x/term" + + nbssh "github.com/netbirdio/netbird/client/ssh" ) func (c *Client) setupTerminalMode(ctx context.Context, session *ssh.Session) error { @@ -82,37 +84,7 @@ func (c *Client) setupTerminal(session *ssh.Session, fd int) error { return fmt.Errorf("get terminal size: %w", err) } - modes := ssh.TerminalModes{ - ssh.ECHO: 1, - ssh.TTY_OP_ISPEED: 14400, - ssh.TTY_OP_OSPEED: 14400, - // Ctrl+C - ssh.VINTR: 3, - // Ctrl+\ - ssh.VQUIT: 28, - // Backspace - ssh.VERASE: 127, - // Ctrl+U - ssh.VKILL: 21, - // Ctrl+D - ssh.VEOF: 4, - ssh.VEOL: 0, - ssh.VEOL2: 0, - // Ctrl+Q - ssh.VSTART: 17, - // Ctrl+S - ssh.VSTOP: 19, - // Ctrl+Z - ssh.VSUSP: 26, - // Ctrl+O - ssh.VDISCARD: 15, - // Ctrl+R - ssh.VREPRINT: 18, - // Ctrl+W - ssh.VWERASE: 23, - // Ctrl+V - ssh.VLNEXT: 22, - } + modes := nbssh.DefaultTerminalModes terminal := os.Getenv("TERM") if terminal == "" { diff --git a/client/ssh/client/terminal_windows.go b/client/ssh/client/terminal_windows.go index 462438317..c6156fc26 100644 --- a/client/ssh/client/terminal_windows.go +++ b/client/ssh/client/terminal_windows.go @@ -10,6 +10,8 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" + + nbssh "github.com/netbirdio/netbird/client/ssh" ) const ( @@ -80,28 +82,14 @@ func (c *Client) setupTerminalMode(_ context.Context, session *ssh.Session) erro w, h := c.getWindowsConsoleSize() modes := ssh.TerminalModes{ - ssh.ECHO: 1, - ssh.TTY_OP_ISPEED: 14400, - ssh.TTY_OP_OSPEED: 14400, - ssh.ICRNL: 1, - ssh.OPOST: 1, - ssh.ONLCR: 1, - ssh.ISIG: 1, - ssh.ICANON: 1, - ssh.VINTR: 3, // Ctrl+C - ssh.VQUIT: 28, // Ctrl+\ - ssh.VERASE: 127, // Backspace - ssh.VKILL: 21, // Ctrl+U - ssh.VEOF: 4, // Ctrl+D - ssh.VEOL: 0, - ssh.VEOL2: 0, - ssh.VSTART: 17, // Ctrl+Q - ssh.VSTOP: 19, // Ctrl+S - ssh.VSUSP: 26, // Ctrl+Z - ssh.VDISCARD: 15, // Ctrl+O - ssh.VWERASE: 23, // Ctrl+W - ssh.VLNEXT: 22, // Ctrl+V - ssh.VREPRINT: 18, // Ctrl+R + ssh.ICRNL: 1, + ssh.OPOST: 1, + ssh.ONLCR: 1, + ssh.ISIG: 1, + ssh.ICANON: 1, + } + for mode, value := range nbssh.DefaultTerminalModes { + modes[mode] = value } if err := session.RequestPty("xterm-256color", h, w, modes); err != nil { diff --git a/client/ssh/common.go b/client/ssh/common.go index 3f4f3e9d1..4ebf8842a 100644 --- a/client/ssh/common.go +++ b/client/ssh/common.go @@ -35,6 +35,19 @@ type HostKeyVerifier interface { VerifySSHHostKey(peerAddress string, key []byte) error } +// PeerKeyLookup returns the stored SSH host key for a peer address. +type PeerKeyLookup func(peerAddress string) ([]byte, bool) + +// VerifySSHHostKey implements HostKeyVerifier by looking up the stored key +// and comparing it against the presented key. +func (l PeerKeyLookup) VerifySSHHostKey(peerAddress string, presentedKey []byte) error { + storedKey, found := l(peerAddress) + if !found { + return ErrPeerNotFound + } + return VerifyHostKey(storedKey, presentedKey, peerAddress) +} + // DaemonHostKeyVerifier implements HostKeyVerifier using the NetBird daemon type DaemonHostKeyVerifier struct { client proto.DaemonServiceClient diff --git a/client/ssh/handshake.go b/client/ssh/handshake.go new file mode 100644 index 000000000..e78a806be --- /dev/null +++ b/client/ssh/handshake.go @@ -0,0 +1,45 @@ +package ssh + +import ( + "context" + "fmt" + "io" + "net" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" +) + +// Handshake runs the SSH client handshake on an already dialed conn and +// returns the resulting client. Dialing bounds only the TCP establishment; +// without a deadline on the socket a peer that accepts and then goes silent +// blocks the handshake forever, so the context deadline is applied to conn +// for the duration of the handshake. conn is closed on any error. +func Handshake(ctx context.Context, conn net.Conn, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { + if deadline, ok := ctx.Deadline(); ok { + if err := conn.SetDeadline(deadline); err != nil { + closeHandshake(conn, "conn after deadline error") + return nil, fmt.Errorf("set handshake deadline: %w", err) + } + } + + sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) + if err != nil { + closeHandshake(conn, "conn after handshake error") + return nil, fmt.Errorf("ssh handshake: %w", err) + } + + if err := conn.SetDeadline(time.Time{}); err != nil { + closeHandshake(sshConn, "ssh conn after deadline clear error") + return nil, fmt.Errorf("clear handshake deadline: %w", err) + } + + return ssh.NewClient(sshConn, chans, reqs), nil +} + +func closeHandshake(c io.Closer, label string) { + if err := c.Close(); err != nil { + log.Debugf("ssh: close %s: %v", label, err) + } +} diff --git a/client/ssh/proxy/proxy.go b/client/ssh/proxy/proxy.go index 721810edb..070515b57 100644 --- a/client/ssh/proxy/proxy.go +++ b/client/ssh/proxy/proxy.go @@ -610,13 +610,10 @@ func (p *SSHProxy) dialBackend(ctx context.Context, addr, user, jwtToken string) return nil, fmt.Errorf("connect to server: %w", err) } - clientConn, chans, reqs, err := cryptossh.NewClientConn(conn, addr, config) - if err != nil { - _ = conn.Close() - return nil, fmt.Errorf("SSH handshake: %w", err) - } + handshakeCtx, cancel := context.WithTimeout(ctx, sshHandshakeTimeout) + defer cancel() - return cryptossh.NewClient(clientConn, chans, reqs), nil + return nbssh.Handshake(handshakeCtx, conn, addr, config) } func (p *SSHProxy) verifyHostKey(hostname string, remote net.Addr, key cryptossh.PublicKey) error { diff --git a/client/ssh/session.go b/client/ssh/session.go new file mode 100644 index 000000000..999b6f251 --- /dev/null +++ b/client/ssh/session.go @@ -0,0 +1,84 @@ +package ssh + +import ( + "fmt" + "io" + + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" +) + +// DefaultTerminalModes are the PTY modes used by the interactive terminal clients. +var DefaultTerminalModes = ssh.TerminalModes{ + ssh.ECHO: 1, + ssh.TTY_OP_ISPEED: 14400, + ssh.TTY_OP_OSPEED: 14400, + ssh.VINTR: 3, // Ctrl+C + ssh.VQUIT: 28, // Ctrl+\ + ssh.VERASE: 127, // Backspace + ssh.VKILL: 21, // Ctrl+U + ssh.VEOF: 4, // Ctrl+D + ssh.VEOL: 0, + ssh.VEOL2: 0, + ssh.VSTART: 17, // Ctrl+Q + ssh.VSTOP: 19, // Ctrl+S + ssh.VSUSP: 26, // Ctrl+Z + ssh.VDISCARD: 15, // Ctrl+O + ssh.VREPRINT: 18, // Ctrl+R + ssh.VWERASE: 23, // Ctrl+W + ssh.VLNEXT: 22, // Ctrl+V +} + +// PTYSession is an interactive shell session with a PTY and its I/O pipes. +type PTYSession struct { + Session *ssh.Session + Stdin io.WriteCloser + Stdout io.Reader + Stderr io.Reader +} + +// StartPTYSession opens a session on the client, requests an xterm-256color PTY +// with the default terminal modes, wires up the I/O pipes and starts a shell. +// The session is closed on any error. +func StartPTYSession(client *ssh.Client, cols, rows int) (*PTYSession, error) { + session, err := client.NewSession() + if err != nil { + return nil, fmt.Errorf("new session: %w", err) + } + + pty, err := setupPTYSession(session, cols, rows) + if err != nil { + if closeErr := session.Close(); closeErr != nil { + log.Debugf("ssh: session close after setup error: %v", closeErr) + } + return nil, err + } + return pty, nil +} + +// setupPTYSession requests the PTY, opens the pipes and starts the shell on an +// already created session. +func setupPTYSession(session *ssh.Session, cols, rows int) (*PTYSession, error) { + if err := session.RequestPty("xterm-256color", rows, cols, DefaultTerminalModes); err != nil { + return nil, fmt.Errorf("request pty: %w", err) + } + + stdin, err := session.StdinPipe() + if err != nil { + return nil, fmt.Errorf("stdin pipe: %w", err) + } + stdout, err := session.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("stdout pipe: %w", err) + } + stderr, err := session.StderrPipe() + if err != nil { + return nil, fmt.Errorf("stderr pipe: %w", err) + } + + if err := session.Shell(); err != nil { + return nil, fmt.Errorf("start shell: %w", err) + } + + return &PTYSession{Session: session, Stdin: stdin, Stdout: stdout, Stderr: stderr}, nil +} diff --git a/client/wasm/internal/ssh/client.go b/client/wasm/internal/ssh/client.go index 9cfe65266..28ae95ec0 100644 --- a/client/wasm/internal/ssh/client.go +++ b/client/wasm/internal/ssh/client.go @@ -80,13 +80,12 @@ func (c *Client) Connect(host string, port int, username, jwtToken string, ipVer return fmt.Errorf("dial %s: %w", addr, err) } - sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) + sshClient, err := nbssh.Handshake(ctx, conn, addr, config) if err != nil { - closeWithLog(conn, "connection after handshake error") - return fmt.Errorf("SSH handshake: %w", err) + return err } - c.sshClient = ssh.NewClient(sshConn, chans, reqs) + c.sshClient = sshClient logrus.Infof("SSH: Connected to %s", addr) return nil @@ -119,57 +118,26 @@ func (c *Client) getAuthMethods(jwtToken string) ([]ssh.AuthMethod, error) { return []ssh.AuthMethod{ssh.PublicKeys(signer)}, nil } -// StartSession starts an SSH session with PTY +// StartSession starts an SSH session with PTY. It holds the client lock for +// the whole startup so Close cannot tear the client down mid-setup and the +// new session cannot be installed into an already closed client. func (c *Client) StartSession(cols, rows int) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.sshClient == nil { return fmt.Errorf("SSH client not connected") } - session, err := c.sshClient.NewSession() + pty, err := nbssh.StartPTYSession(c.sshClient, cols, rows) if err != nil { - return fmt.Errorf("create session: %w", err) + return err } - c.mu.Lock() - defer c.mu.Unlock() - c.session = session - - modes := ssh.TerminalModes{ - ssh.ECHO: 1, - ssh.TTY_OP_ISPEED: 14400, - ssh.TTY_OP_OSPEED: 14400, - ssh.VINTR: 3, - ssh.VQUIT: 28, - ssh.VERASE: 127, - } - - if err := session.RequestPty("xterm-256color", rows, cols, modes); err != nil { - closeWithLog(session, "session after PTY error") - return fmt.Errorf("PTY request: %w", err) - } - - c.stdin, err = session.StdinPipe() - if err != nil { - closeWithLog(session, "session after stdin error") - return fmt.Errorf("get stdin: %w", err) - } - - c.stdout, err = session.StdoutPipe() - if err != nil { - closeWithLog(session, "session after stdout error") - return fmt.Errorf("get stdout: %w", err) - } - - c.stderr, err = session.StderrPipe() - if err != nil { - closeWithLog(session, "session after stderr error") - return fmt.Errorf("get stderr: %w", err) - } - - if err := session.Shell(); err != nil { - closeWithLog(session, "session after shell error") - return fmt.Errorf("start shell: %w", err) - } + c.session = pty.Session + c.stdin = pty.Stdin + c.stdout = pty.Stdout + c.stderr = pty.Stderr logrus.Info("SSH: Session started with PTY") return nil From 070a0a7bf1681890e2998b14fd1f9b1ee6d96444 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 19 Aug 2026 08:05:12 +0000 Subject: [PATCH 2/3] [client, android] Handle network changes without restarting the engine (#7144) On network changes the client restarted the whole engine. That is heavy-handed and slow: it tears down working state to recover from a transition the engine could handle itself. This replaces the restart with proper network event handling. Suspend the retry loops while no network is available. Instead of burning through backoff intervals against an unreachable network, the reconnection loops park until the OS reports a usable network again. Reconnect immediately on a network switch. When the OS hands us a new network, connections bound to the old one are swept and re-dialed right away, rather than waiting for a timeout to notice they are dead. --- client/android/client.go | 46 ++- client/android/connection_listener.go | 41 +++ client/grpc/dialer_generic.go | 51 ++-- client/grpc/dialer_js.go | 6 + client/grpc/retry.go | 49 ++++ client/grpc/retry_test.go | 91 ++++++ client/internal/connect.go | 63 ++++- client/internal/engine.go | 12 +- client/internal/peer/conn.go | 7 +- client/internal/peer/guard/guard.go | 42 ++- client/internal/peer/guard/guard_leak_test.go | 2 +- .../peer/guard/guard_netstate_test.go | 107 +++++++ client/internal/peer/listener.go | 29 ++ client/internal/peer/notifier.go | 111 ++++++-- .../internal/peer/notifier_concurrent_test.go | 108 +++++++ client/internal/peer/notifier_test.go | 27 +- client/internal/peer/status.go | 6 + client/ios/NetBirdSDK/client.go | 43 ++- client/ios/NetBirdSDK/connection_listener.go | 43 +++ client/netstate/netstate.go | 110 ++++++++ client/netstate/netstate_test.go | 170 +++++++++++ client/netsweep/netsweep.go | 267 ++++++++++++++++++ client/netsweep/netsweep_test.go | 241 ++++++++++++++++ client/netsweep/quick_retry.go | 39 +++ client/netsweep/quick_retry_test.go | 58 ++++ shared/management/client/grpc.go | 82 ++++-- shared/relay/client/client.go | 22 +- shared/relay/client/guard.go | 93 +++++- shared/relay/client/guard_test.go | 30 ++ shared/relay/client/manager.go | 19 +- shared/relay/client/picker.go | 3 + shared/signal/client/grpc.go | 86 ++++-- 32 files changed, 1965 insertions(+), 139 deletions(-) create mode 100644 client/android/connection_listener.go create mode 100644 client/grpc/retry.go create mode 100644 client/grpc/retry_test.go create mode 100644 client/internal/peer/guard/guard_netstate_test.go create mode 100644 client/internal/peer/notifier_concurrent_test.go create mode 100644 client/ios/NetBirdSDK/connection_listener.go create mode 100644 client/netstate/netstate.go create mode 100644 client/netstate/netstate_test.go create mode 100644 client/netsweep/netsweep.go create mode 100644 client/netsweep/netsweep_test.go create mode 100644 client/netsweep/quick_retry.go create mode 100644 client/netsweep/quick_retry_test.go create mode 100644 shared/relay/client/guard_test.go diff --git a/client/android/client.go b/client/android/client.go index a21348f27..71bbe4380 100644 --- a/client/android/client.go +++ b/client/android/client.go @@ -26,6 +26,8 @@ import ( "github.com/netbirdio/netbird/client/internal/routemanager" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/formatter" "github.com/netbirdio/netbird/route" @@ -40,11 +42,6 @@ const ( AnonymizeLevelStrict = nbAnonymize.LevelStrictString ) -// ConnectionListener export internal Listener for mobile -type ConnectionListener interface { - peer.Listener -} - // TunAdapter export internal TunAdapter for mobile type TunAdapter interface { device.TunAdapter @@ -85,6 +82,13 @@ type Client struct { deviceName string uiVersion string networkChangeListener listener.NetworkChangeListener + // netState outlives engine restarts: it mirrors the OS connectivity, not + // the engine lifecycle. Run and RunWithoutLogin inject it into each new + // ConnectClient, which distributes it to every reconnection loop. + netState *netstate.State + + // sweeper also outlives engine restarts; NotifyNetworkChange sweeps it. + sweeper *netsweep.Sweeper stateMu sync.RWMutex connectClient *internal.ConnectClient @@ -156,6 +160,8 @@ func NewClient(androidSDKVersion int, deviceName string, uiVersion string, tunAd recorder: peer.NewRecorder(""), ctxCancelLock: &sync.Mutex{}, networkChangeListener: networkChangeListener, + netState: netstate.New(), + sweeper: netsweep.New(), } } @@ -196,7 +202,8 @@ func (c *Client) Run(platformFiles PlatformFiles, urlOpener URLOpener, isAndroid } // todo do not throw error in case of cancelled context ctx = internal.CtxInitState(ctx) - connectClient := internal.NewConnectClient(ctx, cfg, c.recorder) + connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, + internal.WithNetworkState(c.netState), internal.WithSweeper(c.sweeper)) c.setState(cfg, cacheDir, cfgFile, connectClient) // This path runs the interactive SSO flow, so reaching here means the peer // is authenticated again — release the latch Status() reports from. Clear @@ -237,7 +244,8 @@ func (c *Client) RunWithoutLogin(platformFiles PlatformFiles, dns *DNSList, dnsR // todo do not throw error in case of cancelled context ctx = internal.CtxInitState(ctx) - connectClient := internal.NewConnectClient(ctx, cfg, c.recorder) + connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, + internal.WithNetworkState(c.netState), internal.WithSweeper(c.sweeper)) c.setState(cfg, cacheDir, cfgFile, connectClient) return connectClient.RunOnAndroid(c.tunAdapter, c.iFaceDiscover, c.networkChangeListener, slices.Clone(dns.items), dnsReadyListener, stateFile, cacheDir) } @@ -285,6 +293,24 @@ func (c *Client) GetTunSettings() (*TunSettings, error) { }, nil } +// SetNetworkAvailable feeds OS-reported network availability into the client. +// While unavailable, the internal reconnect loops suspend their attempts and +// the connection listener reports NoNetwork instead of Connecting; when +// availability returns, the loops resume immediately with a fresh backoff. +func (c *Client) SetNetworkAvailable(available bool) { + c.netState.Set(available) + c.recorder.SetNetworkAvailable(available) +} + +// NotifyNetworkChange marks the management, signal and relay connections +// stale after the OS switched networks and schedules a sweep that cuts +// whatever has not redialed on the new network by then. The engine and the +// TUN device stay untouched. +func (c *Client) NotifyNetworkChange() { + c.sweeper.MarkNetworkChange() + log.Infof("network change: connections marked stale") +} + // DebugBundle generates a debug bundle, uploads it, and returns the upload key. // It works both with and without a running engine. anonymizeLevel is "default" // or "strict"; strict also anonymizes internal IP ranges, peer names, and @@ -525,7 +551,11 @@ func (c *Client) OnUpdatedHostDNS(list *DNSList) error { // SetConnectionListener set the network connection listener func (c *Client) SetConnectionListener(listener ConnectionListener) { - c.recorder.SetConnectionListener(listener) + if listener == nil { + c.recorder.RemoveConnectionListener() + return + } + c.recorder.SetConnectionListener(connectionListenerAdapter{listener}) } // RemoveConnectionListener remove connection listener diff --git a/client/android/connection_listener.go b/client/android/connection_listener.go new file mode 100644 index 000000000..77c47574b --- /dev/null +++ b/client/android/connection_listener.go @@ -0,0 +1,41 @@ +//go:build android + +package android + +import ( + "github.com/netbirdio/netbird/client/internal/peer" +) + +// Client state values delivered via ConnectionListener.OnStateChanged, +// re-exported as basic constants so gomobile emits them into the generated +// Java bindings. They mirror peer.ClientState*: append-only, never reorder. +const ( + ClientStateDisconnected = int(peer.ClientStateDisconnected) + ClientStateConnected = int(peer.ClientStateConnected) + ClientStateConnecting = int(peer.ClientStateConnecting) + ClientStateDisconnecting = int(peer.ClientStateDisconnecting) + ClientStateNoNetwork = int(peer.ClientStateNoNetwork) +) + +// ConnectionListener export internal Listener for mobile. It mirrors +// peer.Listener with OnStateChanged taking a plain int (one of the +// ClientState* constants), because gomobile cannot bind named types. +type ConnectionListener interface { + OnStateChanged(state int) + OnConnected() + OnDisconnected() + OnConnecting() + OnDisconnecting() + OnAddressChanged(string, string) + OnPeersListChanged(int) +} + +// connectionListenerAdapter adapts the gomobile-facing ConnectionListener to +// peer.Listener, converting the typed state to the int the binding carries. +type connectionListenerAdapter struct { + ConnectionListener +} + +func (a connectionListenerAdapter) OnStateChanged(state peer.ClientState) { + a.ConnectionListener.OnStateChanged(int(state)) +} diff --git a/client/grpc/dialer_generic.go b/client/grpc/dialer_generic.go index 479575996..8a80525e9 100644 --- a/client/grpc/dialer_generic.go +++ b/client/grpc/dialer_generic.go @@ -16,28 +16,47 @@ import ( "google.golang.org/grpc" nbnet "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/netsweep" ) func WithCustomDialer(_ bool, _ string) grpc.DialOption { + return grpc.WithContextDialer(dialContext) +} + +// WithSweeper dials like WithCustomDialer but registers connections and +// dials with the sweeper. Append it after WithCustomDialer: gRPC applies +// dial options in order, so the later context dialer wins. +func WithSweeper(sweeper *netsweep.Sweeper) grpc.DialOption { return grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { - if runtime.GOOS == "linux" { - currentUser, err := user.Current() - if err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "failed to get current user: %v", err) - } + dial := sweeper.StartDial(ctx) + defer dial.Release() - // the custom dialer requires root permissions which are not required for use cases run as non-root - if currentUser.Uid != "0" { - log.Debug("Not running as root, using standard dialer") - dialer := &net.Dialer{} - return dialer.DialContext(ctx, "tcp", addr) - } - } - - conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) + conn, err := dialContext(dial.Ctx(), addr) if err != nil { - return nil, fmt.Errorf("nbnet.NewDialer().DialContext: %w", err) + return nil, err } - return conn, nil + return dial.WrapConn(conn) }) } + +func dialContext(ctx context.Context, addr string) (net.Conn, error) { + if runtime.GOOS == "linux" { + currentUser, err := user.Current() + if err != nil { + return nil, status.Errorf(codes.FailedPrecondition, "failed to get current user: %v", err) + } + + // the custom dialer requires root permissions which are not required for use cases run as non-root + if currentUser.Uid != "0" { + log.Debug("Not running as root, using standard dialer") + dialer := &net.Dialer{} + return dialer.DialContext(ctx, "tcp", addr) + } + } + + conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) + if err != nil { + return nil, fmt.Errorf("nbnet.NewDialer().DialContext: %w", err) + } + return conn, nil +} diff --git a/client/grpc/dialer_js.go b/client/grpc/dialer_js.go index b89ec3c21..8863756d7 100644 --- a/client/grpc/dialer_js.go +++ b/client/grpc/dialer_js.go @@ -3,6 +3,7 @@ package grpc import ( "google.golang.org/grpc" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/util/wsproxy/client" ) @@ -11,3 +12,8 @@ import ( func WithCustomDialer(tlsEnabled bool, component string) grpc.DialOption { return client.WithWebSocketDialer(tlsEnabled, component) } + +// WithSweeper is a no-op on WASM/JS: there is no network change signal. +func WithSweeper(_ *netsweep.Sweeper) grpc.DialOption { + return grpc.EmptyDialOption{} +} diff --git a/client/grpc/retry.go b/client/grpc/retry.go new file mode 100644 index 000000000..754ffa341 --- /dev/null +++ b/client/grpc/retry.go @@ -0,0 +1,49 @@ +package grpc + +import ( + "context" + "errors" + "time" + + "github.com/cenkalti/backoff/v4" + + "github.com/netbirdio/netbird/client/netstate" +) + +// Retry mirrors backoff.Retry, but the sleep between attempts also wakes on +// OS network availability transitions: an operation cut down by a network +// change retries the moment the network settles instead of sleeping through +// the recovery. A nil netState never fires, leaving plain backoff.Retry +// behavior. +func Retry(ctx context.Context, operation backoff.Operation, bo backoff.BackOff, netState *netstate.State) error { + bo.Reset() + for { + err := operation() + if err == nil { + return nil + } + + var permanent *backoff.PermanentError + if errors.As(err, &permanent) { + return permanent.Err + } + + next := bo.NextBackOff() + if next == backoff.Stop { + if cerr := ctx.Err(); cerr != nil { + return cerr + } + return err + } + + timer := time.NewTimer(next) + select { + case <-timer.C: + case <-netState.Changed(): + timer.Stop() + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + } + } +} diff --git a/client/grpc/retry_test.go b/client/grpc/retry_test.go new file mode 100644 index 000000000..4edca47b6 --- /dev/null +++ b/client/grpc/retry_test.go @@ -0,0 +1,91 @@ +package grpc + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/netstate" +) + +func TestRetryWakesOnNetworkChange(t *testing.T) { + ns := netstate.New() + attempts := 0 + operation := func() error { + attempts++ + if attempts == 1 { + return errors.New("cut by network change") + } + return nil + } + + go func() { + time.Sleep(20 * time.Millisecond) + ns.Set(false) + }() + + start := time.Now() + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Minute), ns) + + require.NoError(t, err) + assert.Equal(t, 2, attempts, "network change must cause one immediate retry") + assert.Less(t, time.Since(start), time.Second, "the transition must cut the minute-long sleep short") +} + +func TestRetryPermanentError(t *testing.T) { + sentinel := errors.New("permission denied") + operation := func() error { + return backoff.Permanent(sentinel) + } + + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Millisecond), nil) + assert.ErrorIs(t, err, sentinel, "permanent errors must stop retries") +} + +func TestRetryNilNetState(t *testing.T) { + attempts := 0 + operation := func() error { + attempts++ + if attempts < 3 { + return errors.New("transient") + } + return nil + } + + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Millisecond), nil) + require.NoError(t, err) + assert.Equal(t, 3, attempts, "nil network state must preserve timed retries") +} + +func TestRetryStops(t *testing.T) { + failure := errors.New("still failing") + operation := func() error { + return failure + } + + err := Retry(context.Background(), operation, &backoff.StopBackOff{}, nil) + assert.ErrorIs(t, err, failure, "stop backoff must return the operation error") +} + +func TestRetryCtxCancelDuringSleep(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + operation := func() error { + return errors.New("failing") + } + + go func() { + time.Sleep(20 * time.Millisecond) + cancel() + }() + + start := time.Now() + err := Retry(ctx, operation, backoff.NewConstantBackOff(time.Minute), netstate.New()) + + assert.ErrorIs(t, err, context.Canceled, "context cancellation must stop the retry loop") + assert.Less(t, time.Since(start), time.Second, "context cancellation must interrupt backoff sleep") +} diff --git a/client/internal/connect.go b/client/internal/connect.go index ceb39419e..e45ecca44 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -38,6 +38,8 @@ import ( "github.com/netbirdio/netbird/client/internal/updater" "github.com/netbirdio/netbird/client/internal/updater/installer" nbnet "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" cProto "github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/client/ssh" sshconfig "github.com/netbirdio/netbird/client/ssh/config" @@ -70,18 +72,42 @@ type ConnectClient struct { updateManager *updater.Manager persistSyncResponse bool + + // netState gates every reconnection loop on OS-reported network + // availability. Nil (the default) disables gating; mobile platforms + // inject it via WithNetworkState. + netState *netstate.State + + // sweeper cuts the management, signal and relay connections on network + // change; nil disables it. + sweeper *netsweep.Sweeper +} + +// ConnectClientOption configures optional ConnectClient behavior. +type ConnectClientOption func(*ConnectClient) + +// WithNetworkState injects the OS network availability state that gates every +// reconnection loop; without it gating is disabled. +func WithNetworkState(netState *netstate.State) ConnectClientOption { + return func(c *ConnectClient) { c.netState = netState } +} + +// WithSweeper injects the network change sweeper. +func WithSweeper(sweeper *netsweep.Sweeper) ConnectClientOption { + return func(c *ConnectClient) { c.sweeper = sweeper } } func NewConnectClient( ctx context.Context, config *profilemanager.Config, statusRecorder *peer.Status, + opts ...ConnectClientOption, ) *ConnectClient { // Derive the run context here so Stop owns the cancel that unblocks the run // loop. runCancel is set once at construction, so Stop can call it without // racing the run loop's startup. Callers therefore need not cancel before Stop. runCtx, runCancel := context.WithCancel(ctx) - return &ConnectClient{ + c := &ConnectClient{ ctx: runCtx, runCancel: runCancel, runExited: make(chan struct{}), @@ -89,6 +115,10 @@ func NewConnectClient( statusRecorder: statusRecorder, engineMutex: sync.Mutex{}, } + for _, opt := range opts { + opt(c) + } + return c } func (c *ConnectClient) SetUpdateManager(um *updater.Manager) { @@ -274,6 +304,13 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan return nil } + // suspend connection attempts while the OS reports no usable network + if waited, err := c.netState.Wait(c.ctx); err != nil { + return nil + } else if waited { + backOff.Reset() + } + state.Set(StatusConnecting) engineCtx, cancel := context.WithCancel(c.ctx) @@ -285,7 +322,8 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan }() log.Debugf("connecting to the Management service %s", c.config.ManagementURL.Host) - mgmClient, err := mgm.NewClient(engineCtx, c.config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled) + mgmClient, err := mgm.NewClient(engineCtx, c.config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled, + mgm.WithNetworkState(c.netState), mgm.WithSweeper(c.sweeper)) if err != nil { // On daemon shutdown / Down() the parent context is cancelled // and the dial fails with "context canceled". Wrapping that @@ -360,7 +398,7 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan }() // with the global Netbird config in hand connect (just a connection, no stream yet) Signal - signalClient, err := connectToSignal(engineCtx, loginResp.GetNetbirdConfig(), myPrivateKey) + signalClient, err := connectToSignal(engineCtx, loginResp.GetNetbirdConfig(), myPrivateKey, c.netState, c.sweeper) if err != nil { log.Error(err) return wrapErr(err) @@ -396,7 +434,8 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan engineConfig.StateDir = filepath.Dir(path) } - relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String(), engineConfig.MTU) + relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String(), engineConfig.MTU, + relayClient.WithNetworkState(c.netState), relayClient.WithSweeper(c.sweeper)) c.statusRecorder.SetRelayMgr(relayManager) if len(relayURLs) > 0 { if token != nil { @@ -424,6 +463,7 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan UpdateManager: c.updateManager, ClientMetrics: c.clientMetrics, MetricsCtx: c.ctx, + NetState: c.netState, }, mobileDependency) engine.SetSyncResponsePersistence(c.persistSyncResponse) c.engine = engine @@ -480,6 +520,16 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan // status stream stuck at Connecting. err = backoff.Retry(operation, backoff.WithContext(backOff, c.ctx)) if err != nil { + // Once the client context is cancelled backoff.WithContext surfaces the + // bare context error, and any attempt torn down mid-flight reports the + // same. That cancellation is the caller asking us to stop (Stop, Down or + // an engine restart), so exit cleanly instead of handing back a failure + // the caller would have to distinguish from a real one. + if c.ctx.Err() != nil && errors.Is(err, context.Canceled) { + log.Info("exiting client retry loop, context cancelled") + return nil + } + log.Debugf("exiting client retry loop due to unrecoverable error: %s", err) if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.PermissionDenied) { state.Set(StatusNeedsLogin) @@ -673,7 +723,7 @@ func selectMTU(localMTU uint16, peerMTU int32) uint16 { } // connectToSignal creates Signal Service client and established a connection -func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourPrivateKey wgtypes.Key) (*signal.GrpcClient, error) { +func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourPrivateKey wgtypes.Key, netState *netstate.State, sweeper *netsweep.Sweeper) (*signal.GrpcClient, error) { var sigTLSEnabled bool if wtConfig.Signal.Protocol == mgmProto.HostConfig_HTTPS { sigTLSEnabled = true @@ -681,7 +731,8 @@ func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourP sigTLSEnabled = false } - signalClient, err := signal.NewClient(ctx, wtConfig.Signal.Uri, ourPrivateKey, sigTLSEnabled) + signalClient, err := signal.NewClient(ctx, wtConfig.Signal.Uri, ourPrivateKey, sigTLSEnabled, + signal.WithNetworkState(netState), signal.WithSweeper(sweeper)) if err != nil { log.Errorf("error while connecting to the Signal Exchange Service %s: %s", wtConfig.Signal.Uri, err) return nil, gstatus.Errorf(codes.FailedPrecondition, "failed connecting to Signal Service : %s", err) diff --git a/client/internal/engine.go b/client/internal/engine.go index d92c360f2..5380651a5 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -59,6 +59,7 @@ import ( "github.com/netbirdio/netbird/client/internal/syncstore" "github.com/netbirdio/netbird/client/internal/updater" "github.com/netbirdio/netbird/client/jobexec" + "github.com/netbirdio/netbird/client/netstate" cProto "github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/client/system" nbdns "github.com/netbirdio/netbird/dns" @@ -181,6 +182,9 @@ type EngineServices struct { UpdateManager *updater.Manager ClientMetrics *metrics.ClientMetrics MetricsCtx context.Context + // NetState gates the reconnection loops on OS-reported network + // availability; nil disables gating. + NetState *netstate.State } // Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers. @@ -204,6 +208,10 @@ type Engine struct { config *EngineConfig mobileDep MobileDependency + // netState gates the peer reconnection guards on OS-reported network + // availability; nil disables gating. + netState *netstate.State + // STUNs is a list of STUN servers used by ICE STUNs []*stun.URI // TURNs is a list of STUN servers used by ICE @@ -337,6 +345,7 @@ func NewEngine( syncMsgMux: &sync.Mutex{}, config: config, mobileDep: mobileDep, + netState: services.NetState, STUNs: []*stun.URI{}, TURNs: []*stun.URI{}, networkSerial: 0, @@ -1893,7 +1902,8 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs []netip.Prefix, agentV Addr: e.getRosenpassAddr(), PermissiveMode: e.config.RosenpassPermissive, }, - ICEConfig: e.createICEConfig(), + ICEConfig: e.createICEConfig(), + NetworkState: e.netState, } serviceDependencies := peer.ServiceDependencies{ diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index f3235ec7f..a3c320027 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -26,6 +26,7 @@ import ( "github.com/netbirdio/netbird/client/internal/portforward" "github.com/netbirdio/netbird/client/internal/rosenpass" "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/client/netstate" "github.com/netbirdio/netbird/route" relayClient "github.com/netbirdio/netbird/shared/relay/client" ) @@ -93,6 +94,10 @@ type ConnConfig struct { // ICEConfig ICE protocol configuration ICEConfig icemaker.Config + + // NetworkState gates the reconnection guard on OS-reported network + // availability; nil disables gating. + NetworkState *netstate.State } type Conn struct { @@ -254,7 +259,7 @@ func (conn *Conn) open(engineCtx context.Context, firstPacket []byte) error { conn.handshaker.AddICEListener(conn.workerICE.OnNewOffer) } - conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher) + conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher, conn.config.NetworkState) conn.wg.Add(1) go func() { diff --git a/client/internal/peer/guard/guard.go b/client/internal/peer/guard/guard.go index 6c2e846a9..68d77d318 100644 --- a/client/internal/peer/guard/guard.go +++ b/client/internal/peer/guard/guard.go @@ -6,6 +6,8 @@ import ( "github.com/cenkalti/backoff/v4" log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/netstate" ) // ConnStatus represents the connection state as seen by the guard. @@ -31,20 +33,26 @@ type connStatusFunc func() ConnStatus // - Relayed connection disconnected // - ICE candidate changes type Guard struct { - log *log.Entry - isConnectedOnAllWay connStatusFunc - timeout time.Duration - srWatcher *SRWatcher + log *log.Entry + isConnectedOnAllWay connStatusFunc + timeout time.Duration + srWatcher *SRWatcher + // netState gates reconnect attempts on OS-reported network availability; + // nil disables gating. + netState *netstate.State relayedConnDisconnected chan struct{} iCEConnDisconnected chan struct{} } -func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher) *Guard { +// NewGuard creates a reconnection guard for a peer connection. A nil netState +// disables network availability gating. +func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher, netState *netstate.State) *Guard { return &Guard{ log: log, isConnectedOnAllWay: isConnectedFn, timeout: timeout, srWatcher: srWatcher, + netState: netState, relayedConnDisconnected: make(chan struct{}, 1), iCEConnDisconnected: make(chan struct{}, 1), } @@ -96,9 +104,16 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) { iceState := &iceRetryState{log: g.log} defer iceState.reset() + netChanged := g.netState.Changed() + for { select { case <-tickerChannel: + // skip attempts while the OS reports no usable network; the + // netChanged case below resumes the loop once it returns + if !g.netState.IsOnline() { + continue + } switch g.isConnectedOnAllWay() { case ConnStatusConnected: // all good, nothing to do @@ -135,6 +150,23 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) { tickerChannel = ticker.C iceState.reset() + case <-netChanged: + // Re-arm for the next transition before acting on this one. + netChanged = g.netState.Changed() + if !g.netState.IsOnline() { + continue + } + // Ticks skipped while offline drove the backoff towards its + // maximum without ever attempting, and left the ICE budget + // frozen — possibly in hourly mode. Recover on our own so the + // peer does not depend on a signal or relay event that never + // comes when both stayed up across the outage. + g.log.Debugf("network is back, reset reconnection ticker") + ticker.Stop() + ticker = g.newReconnectTicker(ctx) + tickerChannel = ticker.C + iceState.reset() + case <-ctx.Done(): g.log.Debugf("context is done, stop reconnect loop") return diff --git a/client/internal/peer/guard/guard_leak_test.go b/client/internal/peer/guard/guard_leak_test.go index ded3e4aea..3d82ec591 100644 --- a/client/internal/peer/guard/guard_leak_test.go +++ b/client/internal/peer/guard/guard_leak_test.go @@ -15,7 +15,7 @@ import ( func newTestGuard(status connStatusFunc) *Guard { srw := NewSRWatcher(nil, nil, nil, ice.Config{}) - return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw) + return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw, nil) } // countBackoffTickerGoroutines returns how many goroutines are currently sitting diff --git a/client/internal/peer/guard/guard_netstate_test.go b/client/internal/peer/guard/guard_netstate_test.go new file mode 100644 index 000000000..2ab736428 --- /dev/null +++ b/client/internal/peer/guard/guard_netstate_test.go @@ -0,0 +1,107 @@ +package guard + +import ( + "context" + "sync/atomic" + "testing" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/peer/ice" + "github.com/netbirdio/netbird/client/netstate" +) + +// newTestGuardWithNetState builds a guard with a realistic MaxInterval: the +// backoff must be able to grow well past the outage, as it does in production +// where the timeout is seconds to minutes. +func newTestGuardWithNetState(status connStatusFunc, netState *netstate.State) *Guard { + srw := NewSRWatcher(nil, nil, nil, ice.Config{}) + return NewGuard(log.WithField("test", "guard"), status, 30*time.Second, srw, netState) +} + +// TestGuard_RecoversAfterOfflineToOnline covers a peer that stays disconnected +// across a network outage while neither signal nor relay reports an event — +// both stayed up, as on a short airplane mode toggle over Wi-Fi. +// +// Every tick taken while offline is skipped, but it still advances the +// exponential backoff, so by the time the network returns the next tick can be +// tens of seconds away. Without an explicit reaction to the transition the +// peer waits out that interval for a recovery that could start immediately. +func TestGuard_RecoversAfterOfflineToOnline(t *testing.T) { + netState := netstate.New() + + var attempts atomic.Int32 + g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Start from the reconnect ticker (800ms initial interval), the state a + // peer is in after it loses its connection. + go g.Start(ctx, func() { attempts.Add(1) }) + g.SetRelayedConnDisconnected() + + // Let the backoff climb: 0.8s, 1.6s, 3.2s, 6.4s ... every tick is skipped + // while offline, but each one doubles the wait for the next. + netState.Set(false) + time.Sleep(8 * time.Second) + + offlineAttempts := attempts.Load() + if offlineAttempts != 0 { + t.Fatalf("callback ran %d times while offline, want 0", offlineAttempts) + } + + netState.Set(true) + + // The next organic tick is now several seconds out, so anything within + // this window can only come from reacting to the transition itself. + pollCtx, stopPolling := context.WithTimeout(ctx, 2*time.Second) + defer stopPolling() + + select { + case <-pollCtx.Done(): + t.Fatal("peer was not retried within 2s of the network coming back, " + + "with neither a signal nor a relay event to fall back on") + case <-pollUntil(pollCtx, func() bool { return attempts.Load() > 0 }): + } +} + +// TestGuard_OfflineTransitionDoesNotRetry checks the other direction: going +// offline must not itself trigger an attempt. +func TestGuard_OfflineTransitionDoesNotRetry(t *testing.T) { + netState := netstate.New() + + var attempts atomic.Int32 + g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go g.Start(ctx, func() { attempts.Add(1) }) + + netState.Set(false) + time.Sleep(5 * time.Second) + + if got := attempts.Load(); got != 0 { + t.Fatalf("callback ran %d times after going offline, want 0", got) + } +} + +// pollUntil closes the returned channel once cond holds. It gives up when ctx +// is done, so the polling goroutine never outlives the test that started it. +func pollUntil(ctx context.Context, cond func() bool) <-chan struct{} { + done := make(chan struct{}) + go func() { + for { + if cond() { + close(done) + return + } + select { + case <-ctx.Done(): + return + case <-time.After(10 * time.Millisecond): + } + } + }() + return done +} diff --git a/client/internal/peer/listener.go b/client/internal/peer/listener.go index c601fe534..2bb7fcf32 100644 --- a/client/internal/peer/listener.go +++ b/client/internal/peer/listener.go @@ -1,11 +1,40 @@ package peer +// ClientState identifies the client connection state delivered via +// Listener.OnStateChanged. +type ClientState int + +// Client states. The numeric values cross the gomobile boundary (the mobile +// bindings re-export them as integer constants), so they are a wire format: +// append new states at the end, never reorder or insert. +const ( + ClientStateDisconnected ClientState = iota + ClientStateConnected + ClientStateConnecting + ClientStateDisconnecting + // ClientStateNoNetwork is an overlay state: it is never stored as the + // last notification, only derived from ClientStateConnecting while the + // OS reports no usable network (see notifier.effectiveState). + ClientStateNoNetwork +) + // Listener is a callback type about the NetBird network connection state type Listener interface { + // OnStateChanged reports every client state transition. New states are + // delivered only through this callback; the per-state callbacks below + // are kept for compatibility and will be removed once all consumers + // have migrated. + OnStateChanged(state ClientState) + + // Deprecated: consume OnStateChanged instead. OnConnected() + // Deprecated: consume OnStateChanged instead. OnDisconnected() + // Deprecated: consume OnStateChanged instead. OnConnecting() + // Deprecated: consume OnStateChanged instead. OnDisconnecting() + OnAddressChanged(string, string) OnPeersListChanged(int) } diff --git a/client/internal/peer/notifier.go b/client/internal/peer/notifier.go index 8d1954fe5..1ee1d32ea 100644 --- a/client/internal/peer/notifier.go +++ b/client/internal/peer/notifier.go @@ -4,31 +4,64 @@ import ( "sync" ) -const ( - stateDisconnected = iota - stateConnected - stateConnecting - stateDisconnecting -) - type notifier struct { + // publishLock orders state publication: it is held across computing the + // effective state and handing it to the listener, so a transition cannot + // overtake a newer one and leave the listener on a stale state. + publishLock sync.Mutex serverStateLock sync.Mutex listenersLock sync.Mutex listener Listener currentClientState bool - lastNotification int + lastNotification ClientState lastNumberOfPeers int lastFqdnAddress string lastIPAddress string + networkAvailable bool } func newNotifier() *notifier { - return ¬ifier{} + return ¬ifier{ + networkAvailable: true, + } +} + +// effectiveState maps the computed state to what listeners should see: +// while the OS reports no usable network, "Connecting" would be a lie — +// connection attempts are suspended — so it is reported as NoNetwork. +// Caller must hold serverStateLock. +func (n *notifier) effectiveState(state ClientState) ClientState { + if !n.networkAvailable && state == ClientStateConnecting { + return ClientStateNoNetwork + } + return state +} + +// setNetworkAvailable records the OS network availability and re-notifies +// the listener when the flag flips the effective state (Connecting <-> +// NoNetwork). +func (n *notifier) setNetworkAvailable(available bool) { + n.publishLock.Lock() + defer n.publishLock.Unlock() + + n.serverStateLock.Lock() + if n.networkAvailable == available { + n.serverStateLock.Unlock() + return + } + previous := n.effectiveState(n.lastNotification) + n.networkAvailable = available + current := n.effectiveState(n.lastNotification) + n.serverStateLock.Unlock() + + if previous != current { + n.notify(current) + } } func (n *notifier) setListener(listener Listener) { n.serverStateLock.Lock() - lastNotification := n.lastNotification + lastNotification := n.effectiveState(n.lastNotification) numOfPeers := n.lastNumberOfPeers fqdnAddress := n.lastFqdnAddress address := n.lastIPAddress @@ -52,6 +85,9 @@ func (n *notifier) removeListener() { } func (n *notifier) updateServerStates(mgmState bool, signalState bool) { + n.publishLock.Lock() + defer n.publishLock.Unlock() + n.serverStateLock.Lock() calculatedState := n.calculateState(mgmState, signalState) @@ -61,43 +97,54 @@ func (n *notifier) updateServerStates(mgmState bool, signalState bool) { } n.lastNotification = calculatedState + effective := n.effectiveState(calculatedState) n.serverStateLock.Unlock() - n.notify(calculatedState) + n.notify(effective) } func (n *notifier) clientStart() { + n.publishLock.Lock() + defer n.publishLock.Unlock() + n.serverStateLock.Lock() n.currentClientState = true - n.lastNotification = stateConnecting + n.lastNotification = ClientStateConnecting + effective := n.effectiveState(ClientStateConnecting) n.serverStateLock.Unlock() - n.notify(stateConnecting) + n.notify(effective) } func (n *notifier) clientStop() { + n.publishLock.Lock() + defer n.publishLock.Unlock() + n.serverStateLock.Lock() n.currentClientState = false - n.lastNotification = stateDisconnected + n.lastNotification = ClientStateDisconnected n.serverStateLock.Unlock() - n.notify(stateDisconnected) + n.notify(ClientStateDisconnected) } func (n *notifier) clientTearDown() { + n.publishLock.Lock() + defer n.publishLock.Unlock() + n.serverStateLock.Lock() n.currentClientState = false - n.lastNotification = stateDisconnecting + n.lastNotification = ClientStateDisconnecting n.serverStateLock.Unlock() - n.notify(stateDisconnecting) + n.notify(ClientStateDisconnecting) } -func (n *notifier) isServerStateChanged(newState int) bool { +func (n *notifier) isServerStateChanged(newState ClientState) bool { return n.lastNotification != newState } -func (n *notifier) notify(state int) { +func (n *notifier) notify(state ClientState) { n.listenersLock.Lock() listener := n.listener n.listenersLock.Unlock() @@ -109,20 +156,20 @@ func (n *notifier) notify(state int) { notifyListener(listener, state) } -func (n *notifier) calculateState(managementConn, signalConn bool) int { +func (n *notifier) calculateState(managementConn, signalConn bool) ClientState { if managementConn && signalConn { - return stateConnected + return ClientStateConnected } if !managementConn && !signalConn && !n.currentClientState { - return stateDisconnected + return ClientStateDisconnected } - if n.lastNotification == stateDisconnecting { - return stateDisconnecting + if n.lastNotification == ClientStateDisconnecting { + return ClientStateDisconnecting } - return stateConnecting + return ClientStateConnecting } func (n *notifier) peerListChanged(numOfPeers int) { @@ -159,15 +206,19 @@ func (n *notifier) localAddressChanged(fqdn, address string) { listener.OnAddressChanged(fqdn, address) } -func notifyListener(l Listener, state int) { +func notifyListener(l Listener, state ClientState) { + // legacy per-state callbacks; NoNetwork is delivered only via + // OnStateChanged below switch state { - case stateDisconnected: + case ClientStateDisconnected: l.OnDisconnected() - case stateConnected: + case ClientStateConnected: l.OnConnected() - case stateConnecting: + case ClientStateConnecting: l.OnConnecting() - case stateDisconnecting: + case ClientStateDisconnecting: l.OnDisconnecting() } + + l.OnStateChanged(state) } diff --git a/client/internal/peer/notifier_concurrent_test.go b/client/internal/peer/notifier_concurrent_test.go new file mode 100644 index 000000000..fcaaaad3b --- /dev/null +++ b/client/internal/peer/notifier_concurrent_test.go @@ -0,0 +1,108 @@ +package peer + +import ( + "sync" + "testing" + "time" +) + +type recordingListener struct { + mu sync.Mutex + states []ClientState + onState func(ClientState) +} + +func (l *recordingListener) OnStateChanged(state ClientState) { + l.mu.Lock() + l.states = append(l.states, state) + hook := l.onState + l.mu.Unlock() + + if hook != nil { + hook(state) + } +} + +func (l *recordingListener) last() (ClientState, bool) { + l.mu.Lock() + defer l.mu.Unlock() + if len(l.states) == 0 { + return 0, false + } + return l.states[len(l.states)-1], true +} + +func (l *recordingListener) snapshot() []ClientState { + l.mu.Lock() + defer l.mu.Unlock() + return append([]ClientState(nil), l.states...) +} + +func (l *recordingListener) OnConnected() {} +func (l *recordingListener) OnDisconnected() {} +func (l *recordingListener) OnConnecting() {} +func (l *recordingListener) OnDisconnecting() {} +func (l *recordingListener) OnAddressChanged(string, string) {} +func (l *recordingListener) OnPeersListChanged(int) {} + +// TestNotifier_ConcurrentAvailabilityFlipOrdersPublication holds the first +// transition inside the listener callback and flips availability again from +// another goroutine while it is parked. The second flip must not publish +// ahead of the one in flight, otherwise the listener ends up on a state the +// notifier already superseded. +func TestNotifier_ConcurrentAvailabilityFlipOrdersPublication(t *testing.T) { + n := newNotifier() + n.currentClientState = true + n.lastNotification = ClientStateConnecting + + entered := make(chan struct{}) + release := make(chan struct{}) + + l := &recordingListener{} + l.onState = func(state ClientState) { + if state != ClientStateNoNetwork { + return + } + l.mu.Lock() + l.onState = nil + l.mu.Unlock() + close(entered) + <-release + } + n.listener = l + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + n.setNetworkAvailable(false) + }() + + <-entered + + flipped := make(chan struct{}) + go func() { + defer close(flipped) + n.setNetworkAvailable(true) + }() + + select { + case <-flipped: + t.Fatal("the online transition published while the offline one was " + + "still in flight; publication is not serialized") + case <-time.After(200 * time.Millisecond): + } + + close(release) + <-flipped + wg.Wait() + + got, ok := l.last() + if !ok { + t.Fatal("listener never observed a state") + } + if got != ClientStateConnecting { + t.Fatalf("listener holds %v after the network came back, want Connecting; sequence: %v", + got, l.snapshot()) + } +} diff --git a/client/internal/peer/notifier_test.go b/client/internal/peer/notifier_test.go index 0b7722b0c..a73016b05 100644 --- a/client/internal/peer/notifier_test.go +++ b/client/internal/peer/notifier_test.go @@ -6,29 +6,32 @@ import ( ) type mocListener struct { - lastState int + lastState ClientState wg sync.WaitGroup peersWg sync.WaitGroup peers int } func (l *mocListener) OnConnected() { - l.lastState = stateConnected + l.lastState = ClientStateConnected l.wg.Done() } func (l *mocListener) OnDisconnected() { - l.lastState = stateDisconnected + l.lastState = ClientStateDisconnected l.wg.Done() } func (l *mocListener) OnConnecting() { - l.lastState = stateConnecting + l.lastState = ClientStateConnecting l.wg.Done() } func (l *mocListener) OnDisconnecting() { - l.lastState = stateDisconnecting + l.lastState = ClientStateDisconnecting l.wg.Done() } +func (l *mocListener) OnStateChanged(state ClientState) { + +} func (l *mocListener) OnAddressChanged(host, addr string) { } @@ -57,15 +60,15 @@ func Test_notifier_serverState(t *testing.T) { type scenario struct { name string - expected int + expected ClientState mgmState bool signalState bool } scenarios := []scenario{ - {"connected", stateConnected, true, true}, - {"mgm down", stateConnecting, false, true}, - {"signal down", stateConnecting, true, false}, - {"disconnected", stateDisconnected, false, false}, + {"connected", ClientStateConnected, true, true}, + {"mgm down", ClientStateConnecting, false, true}, + {"signal down", ClientStateConnecting, true, false}, + {"disconnected", ClientStateDisconnected, false, false}, } for _, tt := range scenarios { @@ -85,7 +88,7 @@ func Test_notifier_SetListener(t *testing.T) { listener.setPeersWaiter() n := newNotifier() - n.lastNotification = stateConnecting + n.lastNotification = ClientStateConnecting n.setListener(listener) listener.wait() listener.waitPeers() @@ -99,7 +102,7 @@ func Test_notifier_RemoveListener(t *testing.T) { listener.setWaiter() listener.setPeersWaiter() n := newNotifier() - n.lastNotification = stateConnecting + n.lastNotification = ClientStateConnecting n.setListener(listener) // setListener replays cached state on a goroutine; wait for both the state // and peers callbacks to finish so we don't race on listener.peers. diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 423ce9b23..24e3e7fac 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -1211,6 +1211,12 @@ func (d *Status) ClientTeardown() { d.notifyStateChange() } +// SetNetworkAvailable records the OS-reported network availability; while +// unavailable, listeners see NoNetwork instead of Connecting. +func (d *Status) SetNetworkAvailable(available bool) { + d.notifier.setNetworkAvailable(available) +} + // SetConnectionListener set a listener to the notifier func (d *Status) SetConnectionListener(listener Listener) { d.notifier.setListener(listener) diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index 5df6f92f1..f92f085ab 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -22,6 +22,8 @@ import ( "github.com/netbirdio/netbird/client/internal/listener" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/formatter" "github.com/netbirdio/netbird/route" @@ -36,11 +38,6 @@ const ( AnonymizeLevelStrict = nbAnonymize.LevelStrictString ) -// ConnectionListener export internal Listener for mobile -type ConnectionListener interface { - peer.Listener -} - // RouteListener export internal RouteListener for mobile type NetworkChangeListener interface { listener.NetworkChangeListener @@ -87,6 +84,12 @@ type Client struct { onHostDnsFn func([]string) dnsManager dns.IosDnsManager loginComplete bool + // netState outlives engine restarts: it mirrors the OS connectivity, not + // the engine lifecycle. Run injects it into each new ConnectClient, which + // distributes it to every reconnection loop. + netState *netstate.State + // sweeper also outlives engine restarts; NotifyNetworkChange sweeps it. + sweeper *netsweep.Sweeper // preloadedConfig holds config loaded from JSON (used on tvOS where file writes are blocked) preloadedConfig *profilemanager.Config @@ -109,6 +112,8 @@ func NewClient(cfgFile, stateFile, cacheDir, logFilePath, deviceName string, osV ctxCancelLock: &sync.Mutex{}, networkChangeListener: networkChangeListener, dnsManager: dnsManager, + netState: netstate.New(), + sweeper: netsweep.New(), } } @@ -184,7 +189,8 @@ func (c *Client) Run(fd int32, interfaceName string, envList *EnvList) error { c.onHostDnsFn = func([]string) {} cfg.WgIface = interfaceName - connectClient := internal.NewConnectClient(ctx, cfg, c.recorder) + connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, + internal.WithNetworkState(c.netState), internal.WithSweeper(c.sweeper)) c.setState(cfg, connectClient) // Persist the latest sync response so DebugBundle can include the network // map. On iOS this is backed by disk to keep it out of the constrained @@ -193,6 +199,25 @@ func (c *Client) Run(fd int32, interfaceName string, envList *EnvList) error { return connectClient.RunOniOS(fd, c.networkChangeListener, c.dnsManager, c.stateFile, c.cacheDir, c.logFilePath) } +// SetNetworkAvailable feeds OS-reported network availability into the client +// (e.g. from NWPathMonitor). While unavailable, the internal reconnect loops +// suspend their attempts and the connection listener reports NoNetwork +// instead of Connecting; when availability returns, the loops resume +// immediately with a fresh backoff. +func (c *Client) SetNetworkAvailable(available bool) { + c.netState.Set(available) + c.recorder.SetNetworkAvailable(available) +} + +// NotifyNetworkChange marks the management, signal and relay connections +// stale after the OS switched networks and schedules a sweep that cuts +// whatever has not redialed on the new network by then. The engine and the +// TUN device stay untouched. +func (c *Client) NotifyNetworkChange() { + c.sweeper.MarkNetworkChange() + log.Infof("network change: connections marked stale") +} + // Stop the internal client and free the resources func (c *Client) Stop() { c.ctxCancelLock.Lock() @@ -331,7 +356,11 @@ func (c *Client) GetStatusDetails() *StatusDetails { // SetConnectionListener set the network connection listener func (c *Client) SetConnectionListener(listener ConnectionListener) { - c.recorder.SetConnectionListener(listener) + if listener == nil { + c.recorder.RemoveConnectionListener() + return + } + c.recorder.SetConnectionListener(connectionListenerAdapter{listener}) } // RemoveConnectionListener remove connection listener diff --git a/client/ios/NetBirdSDK/connection_listener.go b/client/ios/NetBirdSDK/connection_listener.go new file mode 100644 index 000000000..d792537ba --- /dev/null +++ b/client/ios/NetBirdSDK/connection_listener.go @@ -0,0 +1,43 @@ +//go:build ios + +package NetBirdSDK + +import ( + "github.com/netbirdio/netbird/client/internal/peer" +) + +// Client state values, re-exported as basic constants so gomobile emits them +// into the generated bindings. They mirror peer.ClientState*: append-only, +// never reorder. +const ( + ClientStateDisconnected = int(peer.ClientStateDisconnected) + ClientStateConnected = int(peer.ClientStateConnected) + ClientStateConnecting = int(peer.ClientStateConnecting) + ClientStateDisconnecting = int(peer.ClientStateDisconnecting) + ClientStateNoNetwork = int(peer.ClientStateNoNetwork) +) + +// ConnectionListener export internal Listener for mobile. +// +// It intentionally lacks OnStateChanged for now: adding a method to a gomobile +// interface breaks every Swift implementation, so the iOS app keeps building +// against the legacy per-state callbacks. A follow-up will extend it together +// with the app. +type ConnectionListener interface { + OnConnected() + OnDisconnected() + OnConnecting() + OnDisconnecting() + OnAddressChanged(string, string) + OnPeersListChanged(int) +} + +// connectionListenerAdapter adapts the gomobile-facing ConnectionListener to +// peer.Listener. +type connectionListenerAdapter struct { + ConnectionListener +} + +// OnStateChanged is dropped on iOS until the app adopts the state callback; +// the legacy per-state callbacks continue to fire. +func (a connectionListenerAdapter) OnStateChanged(peer.ClientState) {} diff --git a/client/netstate/netstate.go b/client/netstate/netstate.go new file mode 100644 index 000000000..0d7a1268b --- /dev/null +++ b/client/netstate/netstate.go @@ -0,0 +1,110 @@ +// Package netstate tracks OS-reported network availability for the client. +// +// A State instance is owned by the platform integration (e.g. the Android or +// iOS bindings, fed from ConnectivityManager callbacks or NWPathMonitor) and +// is injected into the connection retry loops (management, signal, relay, +// peer guards and the top-level connect loop), which consult it to avoid +// burning CPU and battery on reconnect attempts while the device has no +// network at all (e.g. airplane mode), and to reset their backoff as soon as +// the network returns. +// +// Consumers hold a *State that may be nil — every non-mobile platform leaves +// it unset. The read methods are safe on a nil receiver: they report online +// and never block, so consumers behave as if this package did not exist. +package netstate + +import ( + "context" + "sync" + + log "github.com/sirupsen/logrus" +) + +// State holds the OS-reported network availability. The zero value is not +// usable; create instances with New. +type State struct { + mu sync.Mutex + online bool + changed chan struct{} +} + +// New creates a State that starts online. Platforms without network tracking +// pass a nil *State instead: the read methods treat nil as always online and +// never block, so consumers need no nil guards. +func New() *State { + return &State{ + online: true, + changed: make(chan struct{}), + } +} + +// Set records whether the OS reports any usable network. Transitions wake up +// all Wait callers immediately. Unlike the read methods, Set is not nil-safe: +// it is only for the platform owner that created the State with New. +func (s *State) Set(online bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.online == online { + return + } + s.online = online + close(s.changed) + s.changed = make(chan struct{}) + log.Infof("OS network availability changed: online=%t", online) +} + +// IsOnline reports whether the OS reports at least one usable network. On a +// nil receiver — no State injected — it reports online. +func (s *State) IsOnline() bool { + if s == nil { + return true + } + s.mu.Lock() + defer s.mu.Unlock() + return s.online +} + +// Changed returns a channel closed on the next availability transition, for +// callers that already own a select loop and cannot block in Wait. Re-read it +// after every fire: each transition installs a fresh channel. On a nil +// receiver — no State injected — it returns nil, which blocks forever in a +// select, so the caller simply never observes a transition. +func (s *State) Changed() <-chan struct{} { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + return s.changed +} + +// Wait blocks while the network is offline. It reports whether it had to +// wait, so callers can reset their backoff after an outage. It returns early +// with the context error when ctx is done. On a nil receiver — no State +// injected — it returns immediately. +func (s *State) Wait(ctx context.Context) (bool, error) { + if s == nil { + return false, nil + } + waited := false + for { + s.mu.Lock() + if s.online { + s.mu.Unlock() + return waited, nil + } + ch := s.changed + s.mu.Unlock() + + if !waited { + waited = true + log.Debugf("network is offline, pausing connection attempts") + } + + select { + case <-ctx.Done(): + return waited, ctx.Err() + case <-ch: + } + } +} diff --git a/client/netstate/netstate_test.go b/client/netstate/netstate_test.go new file mode 100644 index 000000000..ea7015761 --- /dev/null +++ b/client/netstate/netstate_test.go @@ -0,0 +1,170 @@ +package netstate + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewStateIsOnline(t *testing.T) { + assert.True(t, New().IsOnline(), "a fresh State should start online") +} + +func TestSetTogglesOnlineState(t *testing.T) { + s := New() + + s.Set(false) + assert.False(t, s.IsOnline(), "state should be offline after Set(false)") + + s.Set(true) + assert.True(t, s.IsOnline(), "state should be online after Set(true)") +} + +func TestWaitReturnsImmediatelyWhenOnline(t *testing.T) { + s := New() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + waited, err := s.Wait(ctx) + require.NoError(t, err) + assert.False(t, waited, "Wait should not block when the network is online") +} + +func TestWaitBlocksUntilOnline(t *testing.T) { + s := New() + s.Set(false) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + result := make(chan bool, 1) + go func() { + waited, err := s.Wait(ctx) + if err != nil { + result <- false + return + } + result <- waited + }() + + // Verify Wait is actually blocking while offline + select { + case <-result: + t.Fatal("Wait should block while the network is offline") + case <-time.After(100 * time.Millisecond): + } + + s.Set(true) + + select { + case waited := <-result: + assert.True(t, waited, "Wait should report that it had to wait for the network") + case <-time.After(2 * time.Second): + t.Fatal("Wait should return promptly after the network becomes available") + } +} + +func TestWaitReturnsOnContextCancel(t *testing.T) { + s := New() + s.Set(false) + + ctx, cancel := context.WithCancel(context.Background()) + + result := make(chan error, 1) + go func() { + _, err := s.Wait(ctx) + result <- err + }() + + cancel() + + select { + case err := <-result: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("Wait should return promptly after context cancellation") + } +} + +func TestWaitWakesAllWaiters(t *testing.T) { + s := New() + s.Set(false) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + const waiters = 10 + var wg sync.WaitGroup + results := make(chan bool, waiters) + for i := 0; i < waiters; i++ { + wg.Add(1) + go func() { + defer wg.Done() + waited, err := s.Wait(ctx) + if err != nil { + results <- false + return + } + results <- waited + }() + } + + time.Sleep(100 * time.Millisecond) + s.Set(true) + wg.Wait() + + close(results) + count := 0 + for waited := range results { + assert.True(t, waited, "every waiter should report that it waited") + count++ + } + assert.Equal(t, waiters, count, "all waiters should have returned") +} + +func TestNilStateReadsAreNoops(t *testing.T) { + var s *State + + assert.True(t, s.IsOnline(), "nil State should report online") + + waited, err := s.Wait(context.Background()) + require.NoError(t, err) + assert.False(t, waited, "nil State's Wait should not block") +} + +func TestConcurrentSetAndWait(t *testing.T) { + s := New() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + s.Set(j%2 == 0) + s.IsOnline() + } + }() + } + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + if _, err := s.Wait(ctx); err != nil { + return + } + } + }() + } + + wg.Wait() +} diff --git a/client/netsweep/netsweep.go b/client/netsweep/netsweep.go new file mode 100644 index 000000000..46bc0a709 --- /dev/null +++ b/client/netsweep/netsweep.go @@ -0,0 +1,267 @@ +// Package netsweep cuts network-bound activity when the OS switches networks: +// a sweep closes the registered connections and aborts the in-flight dials, so +// their owners redial immediately instead of waiting for the old sockets to +// time out. +// +// A nil *Sweeper disables everything: all methods are nil-safe no-ops. +package netsweep + +import ( + "context" + "errors" + "net" + "sync" + "time" + + "github.com/cenkalti/backoff/v4" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/netstate" +) + +// DefaultSweepDelay absorbs network flapping while the OS settles on a +// default network before the stale registrations are cut. +const DefaultSweepDelay = 500 * time.Millisecond + +const recentMarkWindow = 3 * time.Second + +// Config customizes a Sweeper. The zero value applies the defaults. +type Config struct { + // SweepDelay overrides DefaultSweepDelay when positive. + SweepDelay time.Duration +} + +// ErrSwept reports that a dial finished after a network change swept its +// registration. The connection is already closed; the caller must treat it +// as a failed dial and redial on the new network. +var ErrSwept = errors.New("netsweep: connection swept by network change") + +// sweepID identifies one registration in a sweeper. Connections and dials +// draw from the same counter, so an id is unique across both registries. +type sweepID uint64 + +type connEntry struct { + conn net.Conn + gen uint64 +} + +// Dial tracks one dial from start to connection registration. It hands the +// dialed connection to the sweeper atomically, so a sweep can never fall +// between the dial finishing and the connection being registered. +type Dial struct { + sweeper *Sweeper + ctx context.Context + cancel context.CancelFunc + id sweepID + done bool // set by a sweep, WrapConn or Release; guarded by sweeper.mu + gen uint64 +} + +// Ctx returns the dial's context. A sweep cancels it, so a dial started on the +// old network aborts instead of waiting out its handshake timeout. +func (d *Dial) Ctx() context.Context { + return d.ctx +} + +// Release ends the dial's registration and cancels its context. It is +// idempotent and safe after WrapConn, so callers can defer it. +func (d *Dial) Release() { + s := d.sweeper + if s == nil { + return + } + + s.mu.Lock() + d.done = true + delete(s.dials, d.id) + s.mu.Unlock() + + d.cancel() +} + +// sweptConn deregisters itself from the sweeper when closed. +type sweptConn struct { + net.Conn + sweeper *Sweeper + id sweepID +} + +func (c *sweptConn) Close() error { + c.sweeper.deregister(c.id) + return c.Conn.Close() +} + +// Sweeper registers live connections and in-flight dials so the +// network-change sweep can cut everything registered before the change. +type Sweeper struct { + mu sync.Mutex + conns map[sweepID]connEntry + dials map[sweepID]*Dial + nextID sweepID + gen uint64 + timer *time.Timer + sweepDelay time.Duration + lastMark time.Time +} + +// New creates an empty sweeper with the default configuration. +func New() *Sweeper { + return NewWithConfig(Config{}) +} + +// NewWithConfig creates an empty sweeper customized by cfg. +func NewWithConfig(cfg Config) *Sweeper { + delay := cfg.SweepDelay + if delay <= 0 { + delay = DefaultSweepDelay + } + return &Sweeper{ + conns: make(map[sweepID]connEntry), + dials: make(map[sweepID]*Dial), + sweepDelay: delay, + } +} + +// StartDial registers an in-flight dial. Dial with Ctx, hand the result to +// WrapConn, and Release the dial when the attempt is over, typically deferred. +func (s *Sweeper) StartDial(ctx context.Context) *Dial { + if s == nil { + return &Dial{ctx: ctx} + } + + ctx, cancel := context.WithCancel(ctx) + d := &Dial{sweeper: s, ctx: ctx, cancel: cancel} + + s.mu.Lock() + d.id = s.nextID + s.nextID++ + d.gen = s.gen + s.dials[d.id] = d + s.mu.Unlock() + + return d +} + +// WrapConn hands conn over to the sweeper. If a sweep ran since StartDial, +// the connection belongs to the old network: it is closed and ErrSwept is +// returned. Otherwise conn is registered against the next sweep and returned +// wrapped, deregistering itself on Close. Call it once, before Release. +func (d *Dial) WrapConn(conn net.Conn) (net.Conn, error) { + s := d.sweeper + if s == nil { + return conn, nil + } + + s.mu.Lock() + if d.done { + s.mu.Unlock() + if err := conn.Close(); err != nil { + log.Debugf("swept dial close error: %v", err) + } + return nil, ErrSwept + } + d.done = true + delete(s.dials, d.id) + id := s.nextID + s.nextID++ + // The conn inherits the dial's generation: the socket was bound to the + // network that was default when the dial started, not when it finished. + s.conns[id] = connEntry{conn: conn, gen: d.gen} + s.mu.Unlock() + + return &sweptConn{Conn: conn, sweeper: s, id: id}, nil +} + +// MarkNetworkChange records that the OS switched networks: everything +// registered so far becomes stale, and a sweep is (re)scheduled after the +// configured delay to cut whatever is still stale by then. Owners that +// redialed in the meantime hold fresh-generation registrations and survive, +// so no cancellation is needed around the sweep. +func (s *Sweeper) MarkNetworkChange() { + if s == nil { + return + } + + s.mu.Lock() + s.gen++ + cutoff := s.gen + s.lastMark = time.Now() + if s.timer != nil { + s.timer.Stop() + } + s.timer = time.AfterFunc(s.sweepDelay, func() { + n := s.sweep(cutoff) + log.Infof("network change sweep: closed %d stale connections", n) + }) + s.mu.Unlock() +} + +// QuickRetryBackoff wraps bo so that after each Reset the first retry comes +// quickly when the disconnect followed a recent network change and the +// network is online. Any other failure keeps bo's spread, so the clients of +// a restarted server still scatter their reconnects. A nil sweeper returns +// bo unchanged. +func (s *Sweeper) QuickRetryBackoff(ctx context.Context, bo backoff.BackOff, netState *netstate.State) backoff.BackOff { + if s == nil { + return bo + } + return backoff.WithContext(newQuickRetryBackoff(bo, s, netState), ctx) +} + +func (s *Sweeper) markedRecently() bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + return !s.lastMark.IsZero() && time.Since(s.lastMark) < recentMarkWindow +} + +// sweep closes the registered connections and aborts the in-flight dials +// older than cutoff, and returns how many connections it closed. A dial +// whose connection was not yet handed to WrapConn is marked, so the late +// WrapConn closes it instead of registering it. +func (s *Sweeper) sweep(cutoff uint64) int { + if s == nil { + return 0 + } + + s.mu.Lock() + var conns []net.Conn + for id, e := range s.conns { + if e.gen < cutoff { + delete(s.conns, id) + conns = append(conns, e.conn) + } + } + var dials []*Dial + for id, d := range s.dials { + if d.gen < cutoff { + d.done = true + delete(s.dials, id) + dials = append(dials, d) + } + } + s.mu.Unlock() + + if len(dials) > 0 { + log.Debugf("aborting %d in-flight dials", len(dials)) + for _, d := range dials { + d.cancel() + } + } + + for _, conn := range conns { + log.Debugf("sweeping connection %s -> %s", conn.LocalAddr(), conn.RemoteAddr()) + if err := conn.Close(); err != nil { + log.Debugf("swept connection close error: %v", err) + } + } + return len(conns) +} + +func (s *Sweeper) deregister(id sweepID) { + s.mu.Lock() + delete(s.conns, id) + s.mu.Unlock() +} diff --git a/client/netsweep/netsweep_test.go b/client/netsweep/netsweep_test.go new file mode 100644 index 000000000..88d660c2d --- /dev/null +++ b/client/netsweep/netsweep_test.go @@ -0,0 +1,241 @@ +package netsweep + +import ( + "context" + "math" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSweepClosesRegisteredConns(t *testing.T) { + sweeper := New() + + c1 := wrap(t, sweeper, connPair(t)) + c2 := wrap(t, sweeper, connPair(t)) + + assert.Equal(t, 2, sweeper.sweepAll(), "both live connections should be closed") + + // The wrappers must report closed now. + buf := make([]byte, 1) + _, err := c1.Read(buf) + assert.Error(t, err, "first connection should be unusable after the sweep") + _, err = c2.Read(buf) + assert.Error(t, err, "second connection should be unusable after the sweep") + + assert.Equal(t, 0, sweeper.sweepAll(), "second sweep should find nothing") +} + +func TestCloseDeregisters(t *testing.T) { + sweeper := New() + + conn := wrap(t, sweeper, connPair(t)) + require.NoError(t, conn.Close()) + + assert.Equal(t, 0, sweeper.sweepAll(), "closed connection must leave the registry") +} + +func TestCloseIsIdempotent(t *testing.T) { + sweeper := New() + + conn := wrap(t, sweeper, connPair(t)) + require.NoError(t, conn.Close()) + assert.Error(t, conn.Close(), "double close surfaces the underlying error but must not panic") +} + +func TestSweepOnlyAffectsOlderConns(t *testing.T) { + sweeper := New() + + _ = wrap(t, sweeper, connPair(t)) + assert.Equal(t, 1, sweeper.sweepAll()) + + // A connection dialed after the sweep must survive until the next one. + _ = wrap(t, sweeper, connPair(t)) + assert.Equal(t, 1, sweeper.sweepAll(), "post-sweep connection belongs to the next sweep") +} + +func TestSweepAbortsInFlightDials(t *testing.T) { + sweeper := New() + + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + + sweeper.sweepAll() + + assert.ErrorIs(t, dial.Ctx().Err(), context.Canceled, "sweep must cancel the in-flight dial context") +} + +func TestReleasedDialIsNotAborted(t *testing.T) { + sweeper := New() + + // Simulate a dial that finished before the sweep. + released := sweeper.StartDial(context.Background()) + released.Release() + + // A dial still in flight during the sweep. + pending := sweeper.StartDial(context.Background()) + defer pending.Release() + + sweeper.sweepAll() + assert.ErrorIs(t, pending.Ctx().Err(), context.Canceled, "pending dial must be aborted") +} + +func TestSweepBetweenDialAndHandoffClosesConn(t *testing.T) { + sweeper := New() + + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + + // The dial succeeds on the old network, then the sweep lands before the + // connection is handed over. + conn := connPair(t) + assert.Equal(t, 0, sweeper.sweepAll(), "the connection is not registered yet") + + wrapped, err := dial.WrapConn(conn) + require.ErrorIs(t, err, ErrSwept) + require.Nil(t, wrapped) + + buf := make([]byte, 1) + _, err = conn.Read(buf) + assert.Error(t, err, "the old-network connection must be closed, not leaked") + + assert.Equal(t, 0, sweeper.sweepAll(), "nothing may leak into the next sweep") +} + +func TestMarkNetworkChangeSparesFreshConns(t *testing.T) { + sweeper := NewWithConfig(Config{SweepDelay: 10 * time.Millisecond}) + + stale := wrap(t, sweeper, connPair(t)) + sweeper.MarkNetworkChange() + _ = wrap(t, sweeper, connPair(t)) + + _ = stale.SetReadDeadline(time.Now().Add(time.Second)) + buf := make([]byte, 1) + _, err := stale.Read(buf) + require.ErrorIs(t, err, net.ErrClosed, "stale connection must be closed by the delayed sweep") + + assert.Equal(t, 1, sweeper.sweepAll(), "the fresh connection must survive the stale sweep") +} + +func TestMarkNetworkChangeAbortsStaleDials(t *testing.T) { + sweeper := NewWithConfig(Config{SweepDelay: 10 * time.Millisecond}) + + stale := sweeper.StartDial(context.Background()) + defer stale.Release() + sweeper.MarkNetworkChange() + fresh := sweeper.StartDial(context.Background()) + defer fresh.Release() + + assert.Eventually(t, func() bool { + return stale.Ctx().Err() != nil + }, time.Second, 5*time.Millisecond, "stale dial must be aborted by the delayed sweep") + assert.NoError(t, fresh.Ctx().Err(), "post-mark dial must not be aborted") +} + +func TestConnInheritsDialGeneration(t *testing.T) { + sweeper := NewWithConfig(Config{SweepDelay: 20 * time.Millisecond}) + + // The dial starts before the network change but completes after it: the + // socket is bound to the old network, so the sweep must still cut it. + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + sweeper.MarkNetworkChange() + + wrapped, err := dial.WrapConn(connPair(t)) + require.NoError(t, err) + + _ = wrapped.SetReadDeadline(time.Now().Add(time.Second)) + buf := make([]byte, 1) + _, err = wrapped.Read(buf) + require.ErrorIs(t, err, net.ErrClosed, "old-generation connection must be swept") +} + +func TestRepeatedMarksCoalesce(t *testing.T) { + sweeper := NewWithConfig(Config{SweepDelay: 20 * time.Millisecond}) + + first := wrap(t, sweeper, connPair(t)) + sweeper.MarkNetworkChange() + second := wrap(t, sweeper, connPair(t)) + sweeper.MarkNetworkChange() + _ = wrap(t, sweeper, connPair(t)) + + buf := make([]byte, 1) + for _, conn := range []net.Conn{first, second} { + _ = conn.SetReadDeadline(time.Now().Add(time.Second)) + _, err := conn.Read(buf) + require.ErrorIs(t, err, net.ErrClosed, "every pre-mark connection must be swept by the rescheduled sweep") + } + assert.Equal(t, 1, sweeper.sweepAll(), "only the newest-generation connection may remain") +} + +func TestNilSweeperIsNoop(t *testing.T) { + var sweeper *Sweeper + + conn := connPair(t) + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + + wrapped, err := dial.WrapConn(conn) + require.NoError(t, err) + assert.Equal(t, conn, wrapped, "nil sweeper must return the conn unchanged") + assert.NoError(t, dial.Ctx().Err(), "nil sweeper must not cancel the dial context") + assert.Equal(t, 0, sweeper.sweepAll(), "nil sweeper closes nothing") +} + +// wrap registers conn with the sweeper through a completed dial. +func wrap(t *testing.T, sweeper *Sweeper, conn net.Conn) net.Conn { + t.Helper() + + dial := sweeper.StartDial(context.Background()) + defer dial.Release() + + wrapped, err := dial.WrapConn(conn) + require.NoError(t, err) + return wrapped +} + +// connPair dials a loopback TCP connection and keeps the accepted peer open +// until the test ends: a peer that closed early would make the connection +// unreadable on its own, so a read error after the sweep would prove nothing. +func connPair(t *testing.T) net.Conn { + t.Helper() + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { + if err := l.Close(); err != nil { + t.Logf("listener close error: %v", err) + } + }) + + accepted := make(chan net.Conn, 1) + go func() { + conn, err := l.Accept() + if err != nil { + close(accepted) + return + } + accepted <- conn + }() + + conn, err := net.Dial("tcp", l.Addr().String()) + require.NoError(t, err) + + peer, ok := <-accepted + require.True(t, ok, "listener must accept the dialed connection") + t.Cleanup(func() { + if err := peer.Close(); err != nil { + t.Logf("peer close error: %v", err) + } + }) + + return conn +} + +// sweepAll cuts every registration regardless of generation. +func (s *Sweeper) sweepAll() int { + return s.sweep(math.MaxUint64) +} diff --git a/client/netsweep/quick_retry.go b/client/netsweep/quick_retry.go new file mode 100644 index 000000000..524a5c50c --- /dev/null +++ b/client/netsweep/quick_retry.go @@ -0,0 +1,39 @@ +package netsweep + +import ( + "time" + + "github.com/cenkalti/backoff/v4" + + "github.com/netbirdio/netbird/client/netstate" +) + +const quickRetryDelay = 200 * time.Millisecond + +type quickRetryBackoff struct { + backoff.BackOff + sweeper *Sweeper + netState *netstate.State + used bool +} + +func newQuickRetryBackoff(bo backoff.BackOff, sweeper *Sweeper, netState *netstate.State) *quickRetryBackoff { + return &quickRetryBackoff{ + BackOff: bo, + sweeper: sweeper, + netState: netState, + } +} + +func (b *quickRetryBackoff) NextBackOff() time.Duration { + if !b.used && b.sweeper.markedRecently() && b.netState.IsOnline() { + b.used = true + return quickRetryDelay + } + return b.BackOff.NextBackOff() +} + +func (b *quickRetryBackoff) Reset() { + b.used = false + b.BackOff.Reset() +} diff --git a/client/netsweep/quick_retry_test.go b/client/netsweep/quick_retry_test.go new file mode 100644 index 000000000..5505862c5 --- /dev/null +++ b/client/netsweep/quick_retry_test.go @@ -0,0 +1,58 @@ +package netsweep + +import ( + "context" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/stretchr/testify/assert" +) + +func TestQuickRetryAfterRecentMark(t *testing.T) { + sweeper := New() + sweeper.MarkNetworkChange() + + slow := backoff.NewConstantBackOff(5 * time.Second) + bo := sweeper.QuickRetryBackoff(context.Background(), slow, nil) + + assert.Equal(t, quickRetryDelay, bo.NextBackOff(), "first retry after a mark must be quick") + assert.Equal(t, 5*time.Second, bo.NextBackOff(), "second retry must fall back to the wrapped backoff") + + bo.Reset() + assert.Equal(t, quickRetryDelay, bo.NextBackOff(), "reset must re-arm the quick retry") +} + +func TestQuickRetryWithoutMarkKeepsSpread(t *testing.T) { + sweeper := New() + + slow := backoff.NewConstantBackOff(5 * time.Second) + bo := sweeper.QuickRetryBackoff(context.Background(), slow, nil) + + assert.Equal(t, 5*time.Second, bo.NextBackOff(), "without a mark the wrapped backoff decides") + + sweeper.mu.Lock() + sweeper.lastMark = time.Now().Add(-recentMarkWindow) + sweeper.mu.Unlock() + assert.Equal(t, 5*time.Second, bo.NextBackOff(), "a stale mark must not trigger the quick retry") +} + +func TestQuickRetryNilSweeperPassthrough(t *testing.T) { + var sweeper *Sweeper + + slow := backoff.NewConstantBackOff(5 * time.Second) + bo := sweeper.QuickRetryBackoff(context.Background(), slow, nil) + + assert.Equal(t, backoff.BackOff(slow), bo, "nil sweeper must return the backoff unchanged") +} + +func TestQuickRetryHonorsContext(t *testing.T) { + sweeper := New() + sweeper.MarkNetworkChange() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + bo := sweeper.QuickRetryBackoff(ctx, backoff.NewConstantBackOff(time.Millisecond), nil) + + assert.Equal(t, backoff.Stop, bo.NextBackOff(), "cancelled context must stop the retry loop") +} diff --git a/shared/management/client/grpc.go b/shared/management/client/grpc.go index 81f25900a..cd250b5f7 100644 --- a/shared/management/client/grpc.go +++ b/shared/management/client/grpc.go @@ -21,6 +21,8 @@ import ( "google.golang.org/grpc/connectivity" nbgrpc "github.com/netbirdio/netbird/client/grpc" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/shared/management/domain" @@ -62,6 +64,13 @@ type GrpcClient struct { connStateCallbackLock sync.RWMutex serverURL string + // netState gates the stream retry loop on OS-reported network + // availability; nil (the default) disables gating. + netState *netstate.State + + // sweeper cuts the transport connections on network change; nil disables it. + sweeper *netsweep.Sweeper + // syncStreamErr holds the last Sync stream error, or nil while the stream // is established and healthy. GetServerKey succeeds even when the peer // cannot sync (e.g. the server returns "settings not found"), so the @@ -111,16 +120,43 @@ func MaxRecvMsgSize() int { return size } +// Option configures optional GrpcClient behavior. +type Option func(*GrpcClient) + +// WithNetworkState injects the OS network availability state that gates the +// stream retry loop; without it gating is disabled. +func WithNetworkState(netState *netstate.State) Option { + return func(c *GrpcClient) { c.netState = netState } +} + +// WithSweeper injects the network change sweeper. +func WithSweeper(sweeper *netsweep.Sweeper) Option { + return func(c *GrpcClient) { c.sweeper = sweeper } +} + // NewClient creates a new client to Management service -func NewClient(ctx context.Context, addr string, ourPrivateKey wgtypes.Key, tlsEnabled bool) (*GrpcClient, error) { - var conn *grpc.ClientConn +func NewClient(ctx context.Context, addr string, ourPrivateKey wgtypes.Key, tlsEnabled bool, opts ...Option) (*GrpcClient, error) { + // Options apply before dialing: the sweeper must wrap the first connection too. + c := &GrpcClient{ + key: ourPrivateKey, + ctx: ctx, + connStateCallbackLock: sync.RWMutex{}, + serverURL: addr, + } + for _, opt := range opts { + opt(c) + } var extraOpts []grpc.DialOption if maxSize := MaxRecvMsgSize(); maxSize > 0 { extraOpts = append(extraOpts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxSize))) log.Infof("management gRPC max receive message size set to %d bytes", maxSize) } + if c.sweeper != nil { + extraOpts = append(extraOpts, nbgrpc.WithSweeper(c.sweeper)) + } + var conn *grpc.ClientConn operation := func() error { var err error conn, err = nbgrpc.CreateConnection(ctx, addr, tlsEnabled, wsproxy.ManagementComponent, extraOpts...) @@ -136,16 +172,9 @@ func NewClient(ctx context.Context, addr string, ourPrivateKey wgtypes.Key, tlsE return nil, err } - realClient := proto.NewManagementServiceClient(conn) - - return &GrpcClient{ - key: ourPrivateKey, - realClient: realClient, - ctx: ctx, - conn: conn, - connStateCallbackLock: sync.RWMutex{}, - serverURL: addr, - }, nil + c.conn = conn + c.realClient = proto.NewManagementServiceClient(conn) + return c, nil } // GetServerURL returns the management server URL @@ -206,16 +235,33 @@ func (c *GrpcClient) withMgmtStream( ctx context.Context, handler func(ctx context.Context, serverPubKey wgtypes.Key, backOff backoff.BackOff) error, ) error { - backOff := defaultBackoff(ctx) + backOff := c.sweeper.QuickRetryBackoff(ctx, defaultBackoff(ctx), c.netState) operation := func() error { - log.Debugf("management connection state %v", c.conn.GetState()) - connState := c.conn.GetState() + // suspend reconnect attempts while the OS reports no usable network. + // Wait only errors on a cancelled context, which means shutdown, so + // stop the loop without reporting a failure. + if waited, err := c.netState.Wait(ctx); err != nil { + log.Debugf("management connection context has been canceled while offline, this usually indicates shutdown") + return nil //nolint:nilerr // a cancelled context means shutdown, not a retryable failure + } else if waited { + backOff.Reset() + } + connState := c.conn.GetState() + log.Debugf("management connection state %v", connState) if connState == connectivity.Shutdown { return backoff.Permanent(fmt.Errorf("connection to management has been shut down")) - } else if !(connState == connectivity.Ready || connState == connectivity.Idle) { + } + if !(connState == connectivity.Ready || connState == connectivity.Idle) { + // A dial may already be in flight (e.g. the other stream triggered + // it after a network change); wait for it to settle and proceed if + // the channel became usable, instead of burning a backoff round on + // a successful dial. A failed dial errors out as before. c.conn.WaitForStateChange(ctx, connState) - return fmt.Errorf("connection to management is not ready and in %s state", connState) + connState = c.conn.GetState() + if !(connState == connectivity.Ready || connState == connectivity.Idle) { + return fmt.Errorf("connection to management is not ready and in %s state", connState) + } } serverPubKey, err := c.getServerPublicKey() @@ -227,7 +273,7 @@ func (c *GrpcClient) withMgmtStream( return handler(ctx, *serverPubKey, backOff) } - err := backoff.Retry(operation, backOff) + err := nbgrpc.Retry(ctx, operation, backOff, c.netState) if err != nil { log.Warnf("exiting the Management service connection retry loop due to the unrecoverable error: %s", err) } diff --git a/shared/relay/client/client.go b/shared/relay/client/client.go index 8d4aa6020..4fb30b8d9 100644 --- a/shared/relay/client/client.go +++ b/shared/relay/client/client.go @@ -14,6 +14,7 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/netsweep" auth "github.com/netbirdio/netbird/shared/relay/auth/hmac" "github.com/netbirdio/netbird/shared/relay/client/dialer" netErr "github.com/netbirdio/netbird/shared/relay/client/dialer/net" @@ -184,6 +185,10 @@ type Client struct { // datagram-sized transport is avoided on subsequent connects. Shared via // the manager. transportFallback *transportFallback + + // sweeper cuts the relay connection on network change; the read loop + // reports the disconnect and the guard reconnects. Shared via the manager. + sweeper *netsweep.Sweeper // datagramFallbackTriggered guards a single fallback per connection so a // burst of oversized datagrams triggers one reconnect, not many. datagramFallbackTriggered atomic.Bool @@ -393,6 +398,12 @@ func (c *Client) Close() error { } func (c *Client) connect(ctx context.Context) (*RelayAddr, error) { + // A sweep cancels this context, so a dial started on the old network + // aborts instead of waiting out its handshake timeout. + dial := c.sweeper.StartDial(ctx) + defer dial.Release() + ctx = dial.Ctx() + mode := transportModeFromEnv() dialers := c.getDialers(mode) @@ -417,12 +428,19 @@ func (c *Client) connect(ctx context.Context) (*RelayAddr, error) { return nil, fmt.Errorf("dial via FQDN: %w", err) } } - c.relayConn = conn - c.datagramFallbackTriggered.Store(false) + // Read the transport off the concrete connection: the sweeper's wrapper + // embeds net.Conn only, so it does not promote Protocol(). if tc, ok := conn.(transportConn); ok { c.transport = tc.Protocol() } + conn, err := dial.WrapConn(conn) + if err != nil { + return nil, fmt.Errorf("register connection: %w", err) + } + c.relayConn = conn + c.datagramFallbackTriggered.Store(false) + instanceURL, err := c.handShake(ctx) if err != nil { cErr := conn.Close() diff --git a/shared/relay/client/guard.go b/shared/relay/client/guard.go index d18534d9d..a62f8772d 100644 --- a/shared/relay/client/guard.go +++ b/shared/relay/client/guard.go @@ -7,9 +7,22 @@ import ( "github.com/cenkalti/backoff/v4" log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/netstate" ) -const defaultMaxBackoffInterval = 60 * time.Second +const ( + defaultMaxBackoffInterval = 60 * time.Second + + // quickReconnectBudget bounds how long a quick reconnect waits for the + // network before handing the retry over to the ticker. + quickReconnectBudget = 1500 * time.Millisecond + + // verdictSettleWindow is how long an online verdict must hold before it + // is trusted: the disconnect often precedes the OS offline flag by a few + // milliseconds. + verdictSettleWindow = 200 * time.Millisecond +) // Guard manage the reconnection tries to the Relay server in case of disconnection event. type Guard struct { @@ -22,14 +35,19 @@ type Guard struct { // attempts. maxBackoffInterval time.Duration + // netState gates reconnect attempts on OS-reported network availability; + // nil disables gating. + netState *netstate.State + // lastErr is the error from the most recent failed reconnect attempt, // surfaced as the home relay status while disconnected. lastErr atomic.Pointer[error] } // NewGuard creates a new guard for the relay client. A non-positive -// maxBackoffInterval falls back to defaultMaxBackoffInterval. -func NewGuard(sp *ServerPicker, maxBackoffInterval time.Duration) *Guard { +// maxBackoffInterval falls back to defaultMaxBackoffInterval. A nil netState +// disables network availability gating. +func NewGuard(sp *ServerPicker, maxBackoffInterval time.Duration, netState *netstate.State) *Guard { if maxBackoffInterval <= 0 { maxBackoffInterval = defaultMaxBackoffInterval } @@ -38,6 +56,7 @@ func NewGuard(sp *ServerPicker, maxBackoffInterval time.Duration) *Guard { OnReconnected: make(chan struct{}, 1), serverPicker: sp, maxBackoffInterval: maxBackoffInterval, + netState: netState, } return g } @@ -70,11 +89,21 @@ func (g *Guard) StartReconnectTrys(ctx context.Context, relayClient *Client) { // start a ticker to pick a new server ticker := g.exponentTicker(ctx) - defer ticker.Stop() + defer func() { + ticker.Stop() + }() for { select { case <-ticker.C: + // suspend reconnect attempts while the OS reports no usable network + if waited, err := g.netState.Wait(ctx); err != nil { + return + } else if waited { + ticker.Stop() + ticker = g.exponentTicker(ctx) + continue + } if err := g.retry(ctx); err != nil { log.Errorf("failed to pick new Relay server: %s", err) g.setLastError(err) @@ -100,7 +129,12 @@ func (g *Guard) tryToQuickReconnect(parentCtx context.Context, rc *Client) bool return false } - if cancelled := waiteBeforeRetry(parentCtx); !cancelled { + if ok := g.waitForNetwork(parentCtx); !ok { + return false + } + + // Still offline after the budget: leave the retry to the ticker. + if !g.netState.IsOnline() { return false } @@ -166,14 +200,47 @@ func (g *Guard) exponentTicker(ctx context.Context) *backoff.Ticker { return backoff.NewTicker(bo) } -func waiteBeforeRetry(ctx context.Context) bool { - timer := time.NewTimer(1500 * time.Millisecond) - defer timer.Stop() +// waitForNetwork waits out the settle window while online, or waits for the +// network to return while offline, within the budget. Returns false when ctx +// is cancelled. Without an injected netState it degrades to a fixed +// budget-long sleep, the pre-netstate behavior. +func (g *Guard) waitForNetwork(ctx context.Context) bool { + budget := time.NewTimer(quickReconnectBudget) + defer budget.Stop() - select { - case <-timer.C: - return true - case <-ctx.Done(): - return false + settleWindow := verdictSettleWindow + if g.netState == nil { + settleWindow = quickReconnectBudget + } + settle := time.NewTimer(settleWindow) + defer settle.Stop() + + for { + // Channel first, flag second: a flip in between still fires the channel. + changedCh := g.netState.Changed() + if g.netState.IsOnline() { + select { + case <-settle.C: + return true + case <-changedCh: + case <-ctx.Done(): + return false + } + } else { + select { + case <-budget.C: + return true + case <-changedCh: + case <-ctx.Done(): + return false + } + } + if !settle.Stop() { + select { + case <-settle.C: + default: + } + } + settle.Reset(settleWindow) } } diff --git a/shared/relay/client/guard_test.go b/shared/relay/client/guard_test.go new file mode 100644 index 000000000..0e05783e0 --- /dev/null +++ b/shared/relay/client/guard_test.go @@ -0,0 +1,30 @@ +package client + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/netbirdio/netbird/client/netstate" +) + +func TestWaitForNetworkSettlesAfterOutage(t *testing.T) { + ns := netstate.New() + ns.Set(false) + g := NewGuard(nil, 0, ns) + + const outage = 2 * verdictSettleWindow + start := time.Now() + go func() { + time.Sleep(outage) + ns.Set(true) + }() + + ok := g.waitForNetwork(context.Background()) + elapsed := time.Since(start) + + assert.True(t, ok, "recovered network must let the quick reconnect proceed") + assert.GreaterOrEqual(t, elapsed, outage+verdictSettleWindow, "reconnect must wait a full settle window after the network returns") +} diff --git a/shared/relay/client/manager.go b/shared/relay/client/manager.go index 2f2839d94..80e38ae2d 100644 --- a/shared/relay/client/manager.go +++ b/shared/relay/client/manager.go @@ -12,6 +12,8 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" relayAuth "github.com/netbirdio/netbird/shared/relay/auth/hmac" ) @@ -65,6 +67,17 @@ func WithMaxBackoffInterval(d time.Duration) ManagerOption { return func(m *Manager) { m.maxBackoffInterval = d } } +// WithNetworkState injects the OS network availability state that gates the +// reconnect guard; without it reconnect attempts are not gated. +func WithNetworkState(netState *netstate.State) ManagerOption { + return func(m *Manager) { m.netState = netState } +} + +// WithSweeper injects the network change sweeper. +func WithSweeper(sweeper *netsweep.Sweeper) ManagerOption { + return func(m *Manager) { m.sweeper = sweeper } +} + // Manager is a manager for the relay client instances. It establishes one persistent connection to the given relay URL // and automatically reconnect to them in case disconnection. // The manager also manage temporary relay connection. If a client wants to communicate with a client on a @@ -92,6 +105,8 @@ type Manager struct { mtu uint16 maxBackoffInterval time.Duration + netState *netstate.State + sweeper *netsweep.Sweeper cleanupInterval time.Duration keepUnusedServerTime time.Duration @@ -128,8 +143,9 @@ func NewManager(ctx context.Context, serverURLs []string, peerID string, mtu uin for _, opt := range opts { opt(m) } + m.serverPicker.Sweeper = m.sweeper m.serverPicker.ServerURLs.Store(serverURLs) - m.reconnectGuard = NewGuard(m.serverPicker, m.maxBackoffInterval) + m.reconnectGuard = NewGuard(m.serverPicker, m.maxBackoffInterval, m.netState) return m } @@ -354,6 +370,7 @@ func (m *Manager) openConnVia(ctx context.Context, serverAddress, peerKey string relayClient := NewClientWithServerIP(serverAddress, serverIP, m.tokenStore, m.peerID, m.mtu) relayClient.SetTransportFallback(m.transportFallback) + relayClient.sweeper = m.sweeper err := relayClient.Connect(m.ctx) if err != nil { rt.Lock() diff --git a/shared/relay/client/picker.go b/shared/relay/client/picker.go index bb721e4ad..72789fadc 100644 --- a/shared/relay/client/picker.go +++ b/shared/relay/client/picker.go @@ -9,6 +9,7 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/netsweep" auth "github.com/netbirdio/netbird/shared/relay/auth/hmac" ) @@ -30,6 +31,7 @@ type ServerPicker struct { MTU uint16 ConnectionTimeout time.Duration TransportFallback *transportFallback + Sweeper *netsweep.Sweeper } func (sp *ServerPicker) PickServer(parentCtx context.Context) (*Client, error) { @@ -73,6 +75,7 @@ func (sp *ServerPicker) startConnection(ctx context.Context, resultChan chan con log.Infof("try to connecting to relay server: %s", url) relayClient := NewClient(url, sp.TokenStore, sp.PeerID, sp.MTU) relayClient.SetTransportFallback(sp.TransportFallback) + relayClient.sweeper = sp.Sweeper err := relayClient.Connect(ctx) resultChan <- connResult{ RelayClient: relayClient, diff --git a/shared/signal/client/grpc.go b/shared/signal/client/grpc.go index a07867263..73c482e8f 100644 --- a/shared/signal/client/grpc.go +++ b/shared/signal/client/grpc.go @@ -19,6 +19,8 @@ import ( "google.golang.org/grpc/status" nbgrpc "github.com/netbirdio/netbird/client/grpc" + "github.com/netbirdio/netbird/client/netstate" + "github.com/netbirdio/netbird/client/netsweep" "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/shared/management/client" "github.com/netbirdio/netbird/shared/signal/proto" @@ -65,6 +67,13 @@ type GrpcClient struct { connStateCallback ConnStateNotifier connStateCallbackLock sync.RWMutex + // netState gates the Receive retry loop on OS-reported network + // availability; nil (the default) disables gating. + netState *netstate.State + + // sweeper cuts the transport connections on network change; nil disables it. + sweeper *netsweep.Sweeper + onReconnectedListenerFn func() decryptionWorker *Worker @@ -88,13 +97,43 @@ type GrpcClient struct { watchdogWg sync.WaitGroup } -// NewClient creates a new Signal client -func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled bool) (*GrpcClient, error) { - var conn *grpc.ClientConn +// Option configures optional GrpcClient behavior. +type Option func(*GrpcClient) +// WithNetworkState injects the OS network availability state that gates the +// Receive retry loop; without it gating is disabled. +func WithNetworkState(netState *netstate.State) Option { + return func(c *GrpcClient) { c.netState = netState } +} + +// WithSweeper injects the network change sweeper. +func WithSweeper(sweeper *netsweep.Sweeper) Option { + return func(c *GrpcClient) { c.sweeper = sweeper } +} + +// NewClient creates a new Signal client +func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled bool, opts ...Option) (*GrpcClient, error) { + // Options apply before dialing: the sweeper must wrap the first connection too. + c := &GrpcClient{ + ctx: ctx, + key: key, + mux: sync.Mutex{}, + status: StreamDisconnected, + connStateCallbackLock: sync.RWMutex{}, + } + for _, opt := range opts { + opt(c) + } + + var extraOpts []grpc.DialOption + if c.sweeper != nil { + extraOpts = append(extraOpts, nbgrpc.WithSweeper(c.sweeper)) + } + + var conn *grpc.ClientConn operation := func() error { var err error - conn, err = nbgrpc.CreateConnection(ctx, addr, tlsEnabled, wsproxy.SignalComponent) + conn, err = nbgrpc.CreateConnection(ctx, addr, tlsEnabled, wsproxy.SignalComponent, extraOpts...) if err != nil { return fmt.Errorf("create connection: %w", err) } @@ -109,15 +148,9 @@ func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled boo log.Debugf("connected to Signal Service: %v", conn.Target()) - return &GrpcClient{ - realClient: proto.NewSignalExchangeClient(conn), - ctx: ctx, - signalConn: conn, - key: key, - mux: sync.Mutex{}, - status: StreamDisconnected, - connStateCallbackLock: sync.RWMutex{}, - }, nil + c.signalConn = conn + c.realClient = proto.NewSignalExchangeClient(conn) + return c, nil } func (c *GrpcClient) StreamConnected() bool { @@ -165,19 +198,36 @@ func defaultBackoff(ctx context.Context) backoff.BackOff { // The connection retry logic will try to reconnect for 30 min and if wasn't successful will propagate the error to the function caller. func (c *GrpcClient) Receive(ctx context.Context, msgHandler func(msg *proto.Message) error) error { - var backOff = defaultBackoff(ctx) + backOff := c.sweeper.QuickRetryBackoff(ctx, defaultBackoff(ctx), c.netState) operation := func() error { + // suspend reconnect attempts while the OS reports no usable network. + // Wait only errors on a cancelled context, which means shutdown, so + // stop the loop without reporting a failure. + if waited, err := c.netState.Wait(ctx); err != nil { + log.Debugf("signal connection context has been canceled while offline, this usually indicates shutdown") + return nil + } else if waited { + backOff.Reset() + } c.notifyStreamDisconnected() - log.Debugf("signal connection state %v", c.signalConn.GetState()) connState := c.signalConn.GetState() + log.Debugf("signal connection state %v", connState) if connState == connectivity.Shutdown { return backoff.Permanent(fmt.Errorf("connection to signal has been shut down")) - } else if !(connState == connectivity.Ready || connState == connectivity.Idle) { + } + if !(connState == connectivity.Ready || connState == connectivity.Idle) { + // A dial may already be in flight (e.g. triggered by another RPC + // after a network change); wait for it to settle and proceed if + // the channel became usable, instead of burning a backoff round on + // a successful dial. A failed dial errors out as before. c.signalConn.WaitForStateChange(ctx, connState) - return fmt.Errorf("connection to signal is not ready and in %s state", connState) + connState = c.signalConn.GetState() + if !(connState == connectivity.Ready || connState == connectivity.Idle) { + return fmt.Errorf("connection to signal is not ready and in %s state", connState) + } } // connect to Signal stream identifying ourselves with a public WireGuard key @@ -231,7 +281,7 @@ func (c *GrpcClient) Receive(ctx context.Context, msgHandler func(msg *proto.Mes return nil } - err := backoff.Retry(operation, backOff) + err := nbgrpc.Retry(ctx, operation, backOff, c.netState) if err != nil { log.Errorf("exiting the Signal service connection retry loop due to the unrecoverable error: %v", err) return err From 6d223042eb906bb462791522de340cfea6705700 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 19 Aug 2026 08:27:06 +0000 Subject: [PATCH 3/3] [client] Clear stale installer result before starting update (#7204) * [client] Clear stale installer result before starting update The installer result file could survive a previous update attempt (e.g. when the updater wrote it after the restarted daemon already ran its startup check). A new install attempt left the old file in place, so the GUI progress window's first GetInstallerResult poll read the outdated result: a stale success made the GUI quit mid-install, which cancelled the TriggerUpdate context and aborted the artifact verification; a stale error surfaced a bogus failure dialog for a succeeding update. Remove any leftover result file at the start of RunInstallation, before the download begins, so result watchers only see the current attempt's outcome. * [client] Align stale-result warning with log message style --- client/internal/updater/installer/installer_common.go | 3 +++ client/internal/updater/installer/result.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/client/internal/updater/installer/installer_common.go b/client/internal/updater/installer/installer_common.go index 8e44bee82..17566f7de 100644 --- a/client/internal/updater/installer/installer_common.go +++ b/client/internal/updater/installer/installer_common.go @@ -42,6 +42,9 @@ func NewWithDir(tempDir string) *Installer { // This will run by the original service process func (u *Installer) RunInstallation(ctx context.Context, targetVersion string) (err error) { resultHandler := NewResultHandler(u.tempDir) + if err := resultHandler.ClearStaleResult(); err != nil { + log.Warnf("clear stale installer result: %v", err) + } defer func() { if err != nil { diff --git a/client/internal/updater/installer/result.go b/client/internal/updater/installer/result.go index 526c3eb53..55a0d8ac8 100644 --- a/client/internal/updater/installer/result.go +++ b/client/internal/updater/installer/result.go @@ -54,6 +54,12 @@ func (rh *ResultHandler) GetErrorResultReason() string { return "" } +// ClearStaleResult removes a result file left over from a previous installation +// attempt so result watchers cannot read an outdated outcome for the current attempt. +func (rh *ResultHandler) ClearStaleResult() error { + return rh.cleanup() +} + func (rh *ResultHandler) WriteSuccess() error { result := Result{ Success: true,