[client] make AllowedIPs swap self-healing on WireGuard errors

Address CodeRabbit review on #6799.

Decrement now decides whether to reprogram based on whether the currently
installed peer still holds references (e.peers[e.active] > 0) instead of
whether the released peer was the active one. If a prior swap's remove/add
failed, e.active was left pointing at a peer with zero references and every
later Decrement returned early, permanently stranding the prefix and leaking
the entry. The active peer is now detached only when e.active != "", and a
failed hand-off is retried on the next Decrement/Increment.

Also clear currentPeerKey unconditionally in the static handler's
RemoveAllowedIPs (matching dynamic/dnsinterceptor) and assert Increment
errors in the tests. Added self-heal tests for the failed remove/add paths.

client/internal/routemanager/dnsinterceptor/handler.go

@@ -95,7 +95,7 @@ func (d *DnsInterceptor) RemoveRoute() error {
 
 			// AllowedIPs should use real IPs
 			if d.currentPeerKey != "" {
-				if _, err := d.allowedIPsRefcounter.Decrement(prefix); err != nil {
+				if _, err := d.allowedIPsRefcounter.Decrement(prefix, d.currentPeerKey); err != nil {
 					merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %v", prefix, err))
 				}
 			}
@@ -172,7 +172,7 @@ func (d *DnsInterceptor) removeAllowedIP(realPrefix netip.Prefix) error {
 	}
 
 	// AllowedIPs use real IPs
-	if _, err := d.allowedIPsRefcounter.Decrement(realPrefix); err != nil {
+	if _, err := d.allowedIPsRefcounter.Decrement(realPrefix, d.currentPeerKey); err != nil {
 		return fmt.Errorf("remove allowed IP %s: %v", realPrefix, err)
 	}
 
@@ -205,7 +205,7 @@ func (d *DnsInterceptor) RemoveAllowedIPs() error {
 	for _, prefixes := range d.interceptedDomains {
 		for _, prefix := range prefixes {
 			// AllowedIPs use real IPs
-			if _, err := d.allowedIPsRefcounter.Decrement(prefix); err != nil {
+			if _, err := d.allowedIPsRefcounter.Decrement(prefix, d.currentPeerKey); err != nil {
 				merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %v", prefix, err))
 			}
 		}

client/internal/routemanager/dynamic/route.go

@@ -135,7 +135,7 @@ func (r *Route) RemoveAllowedIPs() error {
 	var merr *multierror.Error
 	for _, domainPrefixes := range r.dynamicDomains {
 		for _, prefix := range domainPrefixes {
-			if _, err := r.allowedIPsRefcounter.Decrement(prefix); err != nil {
+			if _, err := r.allowedIPsRefcounter.Decrement(prefix, r.currentPeerKey); err != nil {
 				merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %w", prefix, err))
 			}
 		}
@@ -320,7 +320,7 @@ func (r *Route) removeRoutes(prefixes []netip.Prefix) ([]netip.Prefix, error) {
 			merr = multierror.Append(merr, fmt.Errorf("remove dynamic route for IP %s: %w", prefix, err))
 		}
 		if r.currentPeerKey != "" {
-			if _, err := r.allowedIPsRefcounter.Decrement(prefix); err != nil {
+			if _, err := r.allowedIPsRefcounter.Decrement(prefix, r.currentPeerKey); err != nil {
 				merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %w", prefix, err))
 			}
 		}

client/internal/routemanager/manager.go

@@ -215,7 +215,7 @@ func (m *DefaultManager) setupRefCounters(useNoop bool) {
 		)
 	}
 
-	m.allowedIPsRefCounter = refcounter.New(
+	m.allowedIPsRefCounter = refcounter.NewAllowedIPs(
 		func(prefix netip.Prefix, peerKey string) (string, error) {
 			// save peerKey to use it in the remove function
 			return peerKey, m.wgInterface.AddAllowedIP(peerKey, prefix)

client/internal/routemanager/refcounter/allowedips.go

@@ -0,0 +1,185 @@
+package refcounter
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+	"sort"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
+
+	nberrors "github.com/netbirdio/netbird/client/errors"
+)
+
+// allowedIPsEntry holds the per-peer reference counts for a single prefix and which peer is
+// currently installed in WireGuard. WireGuard allows a prefix on exactly one peer, so at most
+// one peer is active at a time even when several peers reference the prefix.
+type allowedIPsEntry struct {
+	// peers maps a peerKey to the number of references holding the prefix for that peer.
+	peers map[string]int
+	// active is the peerKey currently installed in WireGuard for this prefix ("" if none).
+	active string
+	// total is the sum of all per-peer reference counts (kept in sync with peers).
+	total int
+}
+
+// AllowedIPsRefCounter is a peer-aware reference counter for WireGuard AllowedIPs.
+//
+// The generic Counter keys only by prefix and remembers a single Out value set by the first
+// caller, which it never changes. That is wrong for AllowedIPs: two independent watchers (or
+// multiple resolved domains) can reference the same prefix through different peers, and when the
+// peer currently installed in WireGuard releases its last reference the prefix must be handed over
+// to a surviving peer instead of being left pointing at the released one.
+//
+// It calls add/remove (which program WireGuard) only on the transitions that matter:
+//   - add on the first reference for a prefix, or when swapping the active peer;
+//   - remove on the last reference for a prefix, or on the old peer during a swap.
+type AllowedIPsRefCounter struct {
+	mu      sync.Mutex
+	entries map[netip.Prefix]*allowedIPsEntry
+	add     AddFunc[netip.Prefix, string, string]
+	remove  RemoveFunc[netip.Prefix, string]
+}
+
+// NewAllowedIPs creates a new peer-aware AllowedIPs reference counter.
+// add programs a prefix on a peer in WireGuard and returns the peerKey to store as the active peer.
+// remove unprograms the prefix from the given peer.
+func NewAllowedIPs(add AddFunc[netip.Prefix, string, string], remove RemoveFunc[netip.Prefix, string]) *AllowedIPsRefCounter {
+	return &AllowedIPsRefCounter{
+		entries: map[netip.Prefix]*allowedIPsEntry{},
+		add:     add,
+		remove:  remove,
+	}
+}
+
+// Increment adds a reference to prefix for peerKey. WireGuard is programmed only for the first
+// reference to a prefix; while a different peer is already installed the prefix is left with it
+// (first peer wins, HA at the WireGuard layer is not possible) and only the reference count is kept.
+func (rm *AllowedIPsRefCounter) Increment(prefix netip.Prefix, peerKey string) (Ref[string], error) {
+	rm.mu.Lock()
+	defer rm.mu.Unlock()
+
+	e, ok := rm.entries[prefix]
+	if !ok {
+		e = &allowedIPsEntry{peers: map[string]int{}}
+		rm.entries[prefix] = e
+	}
+
+	logCallerF("Increasing allowed IP ref count for prefix %v peer %s [peer %d -> %d, total %d -> %d, active %q]",
+		prefix, peerKey, e.peers[peerKey], e.peers[peerKey]+1, e.total, e.total+1, e.active)
+
+	// Program WireGuard only when nothing is installed yet for this prefix.
+	if e.active == "" {
+		out, err := rm.add(prefix, peerKey)
+		if errors.Is(err, ErrIgnore) {
+			if e.total == 0 {
+				delete(rm.entries, prefix)
+			}
+			return Ref[string]{Count: e.total, Out: e.active}, nil
+		}
+		if err != nil {
+			if e.total == 0 {
+				delete(rm.entries, prefix)
+			}
+			return Ref[string]{}, fmt.Errorf("failed to add allowed IP %v for peer %s: %w", prefix, peerKey, err)
+		}
+		e.active = out
+	}
+
+	e.peers[peerKey]++
+	e.total++
+
+	return Ref[string]{Count: e.total, Out: e.active}, nil
+}
+
+// Decrement removes a reference to prefix for peerKey. When the peer currently installed in
+// WireGuard releases its last reference, the prefix is swapped to a surviving peer if one exists,
+// otherwise it is removed from WireGuard.
+func (rm *AllowedIPsRefCounter) Decrement(prefix netip.Prefix, peerKey string) (Ref[string], error) {
+	rm.mu.Lock()
+	defer rm.mu.Unlock()
+
+	e, ok := rm.entries[prefix]
+	if !ok {
+		logCallerF("No allowed IP reference found for prefix %v", prefix)
+		return Ref[string]{}, nil
+	}
+
+	if e.peers[peerKey] > 0 {
+		logCallerF("Decreasing allowed IP ref count for prefix %v peer %s [peer %d -> %d, total %d -> %d, active %q]",
+			prefix, peerKey, e.peers[peerKey], e.peers[peerKey]-1, e.total, e.total-1, e.active)
+		e.peers[peerKey]--
+		e.total--
+		if e.peers[peerKey] == 0 {
+			delete(e.peers, peerKey)
+		}
+	} else {
+		logCallerF("No allowed IP reference found for prefix %v peer %s", prefix, peerKey)
+	}
+
+	// If the peer currently installed in WireGuard still holds references, nothing to reprogram.
+	// Keying the check on the active peer (not the one just released) makes this self-healing:
+	// a prior swap whose remove/add failed leaves e.active pointing at a peer with no references,
+	// and this retries the hand-off on the next Decrement instead of getting stuck.
+	if e.active != "" && e.peers[e.active] > 0 {
+		return Ref[string]{Count: e.total, Out: e.active}, nil
+	}
+
+	// Detach the stale/gone active peer from WireGuard before reprogramming.
+	if e.active != "" {
+		if err := rm.remove(prefix, e.active); err != nil {
+			return Ref[string]{Count: e.total, Out: e.active}, fmt.Errorf("remove allowed IP %v for peer %s: %w", prefix, e.active, err)
+		}
+		e.active = ""
+	}
+
+	// Hand the prefix over to a surviving peer, or drop the entry when none remain.
+	if survivor, ok := pickSurvivor(e.peers); ok {
+		out, err := rm.add(prefix, survivor)
+		if err != nil {
+			return Ref[string]{Count: e.total, Out: ""}, fmt.Errorf("swap allowed IP %v to peer %s: %w", prefix, survivor, err)
+		}
+		e.active = out
+		return Ref[string]{Count: e.total, Out: e.active}, nil
+	}
+
+	delete(rm.entries, prefix)
+	return Ref[string]{Count: 0, Out: ""}, nil
+}
+
+// Flush removes all prefixes from WireGuard and clears the counter.
+func (rm *AllowedIPsRefCounter) Flush() error {
+	rm.mu.Lock()
+	defer rm.mu.Unlock()
+
+	var merr *multierror.Error
+	for prefix, e := range rm.entries {
+		if e.active == "" {
+			continue
+		}
+		logCallerF("Flushing allowed IP for prefix %v peer %s", prefix, e.active)
+		if err := rm.remove(prefix, e.active); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %v for peer %s: %w", prefix, e.active, err))
+		}
+	}
+
+	clear(rm.entries)
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+// pickSurvivor deterministically selects a peer still referencing the prefix. WireGuard cannot do
+// multipath for a single prefix, so any surviving peer is a valid winner; the choice is made stable
+// (lowest peerKey) for predictable behavior and testability.
+func pickSurvivor(peers map[string]int) (string, bool) {
+	if len(peers) == 0 {
+		return "", false
+	}
+	keys := make([]string, 0, len(peers))
+	for k := range peers {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys[0], true
+}

client/internal/routemanager/refcounter/allowedips_test.go

@@ -0,0 +1,241 @@
+package refcounter
+
+import (
+	"errors"
+	"net/netip"
+	"testing"
+)
+
+// fakeWG models WireGuard's cryptokey routing: a prefix can be installed on exactly one peer.
+// failAdd/failRemove make the next add/remove fail once, to exercise the self-healing error paths.
+type fakeWG struct {
+	installed  map[netip.Prefix]string
+	adds       int
+	removes    int
+	failAdd    bool
+	failRemove bool
+}
+
+func newFakeWG() *fakeWG {
+	return &fakeWG{installed: map[netip.Prefix]string{}}
+}
+
+func (f *fakeWG) counter() *AllowedIPsRefCounter {
+	return NewAllowedIPs(
+		func(prefix netip.Prefix, peerKey string) (string, error) {
+			if f.failAdd {
+				f.failAdd = false
+				return "", errors.New("add failed")
+			}
+			f.adds++
+			f.installed[prefix] = peerKey
+			return peerKey, nil
+		},
+		func(prefix netip.Prefix, peerKey string) error {
+			if f.failRemove {
+				f.failRemove = false
+				return errors.New("remove failed")
+			}
+			f.removes++
+			// only clear if this peer is the one installed, mirroring wg semantics
+			if f.installed[prefix] == peerKey {
+				delete(f.installed, prefix)
+			}
+			return nil
+		},
+	)
+}
+
+func mustPrefix(t *testing.T, s string) netip.Prefix {
+	t.Helper()
+	p, err := netip.ParsePrefix(s)
+	if err != nil {
+		t.Fatalf("parse prefix %q: %v", s, err)
+	}
+	return p
+}
+
+func mustIncrement(t *testing.T, c *AllowedIPsRefCounter, p netip.Prefix, peer string) Ref[string] {
+	t.Helper()
+	ref, err := c.Increment(p, peer)
+	if err != nil {
+		t.Fatalf("Increment(%v, %s): %v", p, peer, err)
+	}
+	return ref
+}
+
+func mustDecrement(t *testing.T, c *AllowedIPsRefCounter, p netip.Prefix, peer string) Ref[string] {
+	t.Helper()
+	ref, err := c.Decrement(p, peer)
+	if err != nil {
+		t.Fatalf("Decrement(%v, %s): %v", p, peer, err)
+	}
+	return ref
+}
+
+// TestAllowedIPs_SwapOnActivePeerRemoval reproduces the reported bug: two networks with the same
+// prefix routed by different peers. Removing the network whose peer is installed must hand the
+// prefix over to the surviving peer instead of leaving it on the removed one.
+func TestAllowedIPs_SwapOnActivePeerRemoval(t *testing.T) {
+	f := newFakeWG()
+	c := f.counter()
+	p := mustPrefix(t, "10.44.8.0/24")
+
+	mustIncrement(t, c, p, "peerA")
+	mustIncrement(t, c, p, "peerB")
+	// First peer wins while both are present.
+	if got := f.installed[p]; got != "peerA" {
+		t.Fatalf("expected peerA installed, got %q", got)
+	}
+
+	// Remove the active peer's network -> must swap to peerB.
+	mustDecrement(t, c, p, "peerA")
+	if got := f.installed[p]; got != "peerB" {
+		t.Fatalf("BUG: prefix stuck on removed peer, want peerB got %q", got)
+	}
+
+	// Remove the last one -> prefix gone.
+	mustDecrement(t, c, p, "peerB")
+	if _, ok := f.installed[p]; ok {
+		t.Fatalf("expected prefix removed, still installed on %q", f.installed[p])
+	}
+}
+
+// TestAllowedIPs_RemoveNonActivePeer removing a non-installed peer must not touch WireGuard.
+func TestAllowedIPs_RemoveNonActivePeer(t *testing.T) {
+	f := newFakeWG()
+	c := f.counter()
+	p := mustPrefix(t, "10.44.8.0/24")
+
+	mustIncrement(t, c, p, "peerA")
+	mustIncrement(t, c, p, "peerB")
+	removesBefore := f.removes
+
+	mustDecrement(t, c, p, "peerB")
+	if f.installed[p] != "peerA" {
+		t.Fatalf("active peer must stay peerA, got %q", f.installed[p])
+	}
+	if f.removes != removesBefore {
+		t.Fatalf("removing a non-active peer must not call wg remove")
+	}
+}
+
+// TestAllowedIPs_SamePeerMultipleRefs two references via the same peer must keep the prefix until
+// the last reference is released (the reason the per-peer count must be an int, not a set).
+func TestAllowedIPs_SamePeerMultipleRefs(t *testing.T) {
+	f := newFakeWG()
+	c := f.counter()
+	p := mustPrefix(t, "10.44.8.0/24")
+
+	mustIncrement(t, c, p, "peerA")
+	mustIncrement(t, c, p, "peerA")
+	if f.adds != 1 {
+		t.Fatalf("expected a single wg add for the same peer, got %d", f.adds)
+	}
+
+	mustDecrement(t, c, p, "peerA")
+	if f.installed[p] != "peerA" {
+		t.Fatalf("prefix must stay while a reference remains, got %q", f.installed[p])
+	}
+	if f.removes != 0 {
+		t.Fatalf("no wg remove expected while a reference remains, got %d", f.removes)
+	}
+
+	mustDecrement(t, c, p, "peerA")
+	if _, ok := f.installed[p]; ok {
+		t.Fatalf("prefix must be removed after last reference")
+	}
+}
+
+// TestAllowedIPs_RefCountAndActive checks the Ref returned to callers (used for the HA-disabled log).
+func TestAllowedIPs_RefCountAndActive(t *testing.T) {
+	f := newFakeWG()
+	c := f.counter()
+	p := mustPrefix(t, "10.44.8.0/24")
+
+	ref := mustIncrement(t, c, p, "peerA")
+	if ref.Count != 1 || ref.Out != "peerA" {
+		t.Fatalf("want {1, peerA}, got {%d, %q}", ref.Count, ref.Out)
+	}
+	ref = mustIncrement(t, c, p, "peerB")
+	if ref.Count != 2 || ref.Out != "peerA" {
+		t.Fatalf("want {2, peerA}, got {%d, %q}", ref.Count, ref.Out)
+	}
+}
+
+// TestAllowedIPs_Flush removes everything installed and clears the counter.
+func TestAllowedIPs_Flush(t *testing.T) {
+	f := newFakeWG()
+	c := f.counter()
+	p1 := mustPrefix(t, "10.44.8.0/24")
+	p2 := mustPrefix(t, "10.44.9.0/24")
+
+	mustIncrement(t, c, p1, "peerA")
+	mustIncrement(t, c, p2, "peerB")
+
+	if err := c.Flush(); err != nil {
+		t.Fatal(err)
+	}
+	if len(f.installed) != 0 {
+		t.Fatalf("expected all prefixes removed, got %v", f.installed)
+	}
+	// After flush, a fresh increment must add again.
+	mustIncrement(t, c, p1, "peerC")
+	if f.installed[p1] != "peerC" {
+		t.Fatalf("counter not reset after flush")
+	}
+}
+
+// TestAllowedIPs_SelfHealAfterSwapAddError ensures a failed add during a swap does not permanently
+// strand the prefix: the next Decrement (or Increment) must retry and install a surviving peer.
+func TestAllowedIPs_SelfHealAfterSwapAddError(t *testing.T) {
+	f := newFakeWG()
+	c := f.counter()
+	p := mustPrefix(t, "10.44.8.0/24")
+
+	mustIncrement(t, c, p, "peerA")
+	mustIncrement(t, c, p, "peerB")
+	mustIncrement(t, c, p, "peerC")
+
+	// Removing the active peerA triggers a swap to a survivor; make the add fail once.
+	f.failAdd = true
+	if _, err := c.Decrement(p, "peerA"); err == nil {
+		t.Fatalf("expected error from failed swap add")
+	}
+	if _, ok := f.installed[p]; ok {
+		t.Fatalf("nothing should be installed after a failed swap add, got %q", f.installed[p])
+	}
+
+	// A later Decrement of a non-active survivor must retry the hand-off (self-heal), not stay stuck.
+	ref := mustDecrement(t, c, p, "peerC")
+	if got := f.installed[p]; got == "" {
+		t.Fatalf("self-heal failed: prefix left unrouted after add recovered")
+	}
+	if ref.Out == "" {
+		t.Fatalf("expected an active peer after self-heal, got empty")
+	}
+}
+
+// TestAllowedIPs_SelfHealAfterRemoveError ensures a failed remove during a swap is retried instead
+// of leaving e.active stuck on a peer that no longer holds references.
+func TestAllowedIPs_SelfHealAfterRemoveError(t *testing.T) {
+	f := newFakeWG()
+	c := f.counter()
+	p := mustPrefix(t, "10.44.8.0/24")
+
+	mustIncrement(t, c, p, "peerA")
+	mustIncrement(t, c, p, "peerB")
+
+	// Releasing active peerA must detach it (remove) then add peerB; fail the remove once.
+	f.failRemove = true
+	if _, err := c.Decrement(p, "peerA"); err == nil {
+		t.Fatalf("expected error from failed remove")
+	}
+
+	// Next Decrement of the non-active survivor retries: removes stale peerA, installs peerB.
+	mustDecrement(t, c, p, "peerB")
+	// peerB had only one ref, so after retry the prefix is fully released.
+	if _, ok := f.installed[p]; ok {
+		t.Fatalf("expected prefix released after self-heal, still on %q", f.installed[p])
+	}
+}

client/internal/routemanager/refcounter/types.go

@@ -5,5 +5,7 @@ import "net/netip"
 // RouteRefCounter is a Counter for Route, it doesn't take any input on Increment and doesn't use any output on Decrement
 type RouteRefCounter = Counter[netip.Prefix, struct{}, struct{}]
 
-// AllowedIPsRefCounter is a Counter for AllowedIPs, it takes a peer key on Increment and passes it back to Decrement
-type AllowedIPsRefCounter = Counter[netip.Prefix, string, string]
+// AllowedIPsRefCounter tracks WireGuard AllowedIPs per prefix. Unlike the generic Counter it is peer-aware:
+// a prefix can be claimed by several peers at once and WireGuard allows a given prefix on exactly one peer,
+// so the counter records the per-peer reference count and swaps the installed peer when the active one is released.
+// See allowedips.go.

client/internal/routemanager/static/route.go

@@ -15,6 +15,11 @@ type Route struct {
 	route                *route.Route
 	routeRefCounter      *refcounter.RouteRefCounter
 	allowedIPsRefcounter *refcounter.AllowedIPsRefCounter
+	// currentPeerKey is the routing peer this watcher currently has the prefix installed on
+	// (the HA winner elected by the watcher). It can differ from route.Peer and change on
+	// failover, so it is recorded on AddAllowedIPs and used on RemoveAllowedIPs to decrement
+	// the exact peer that was incremented.
+	currentPeerKey string
 }
 
 func NewRoute(params common.HandlerParams) *Route {
@@ -52,12 +57,15 @@ func (r *Route) AddAllowedIPs(peerKey string) error {
 			ref.Out,
 		)
 	}
+	r.currentPeerKey = peerKey
 	return nil
 }
 
 func (r *Route) RemoveAllowedIPs() error {
-	if _, err := r.allowedIPsRefcounter.Decrement(r.route.Network); err != nil {
-		return err
+	var err error
+	if _, decErr := r.allowedIPsRefcounter.Decrement(r.route.Network, r.currentPeerKey); decErr != nil {
+		err = fmt.Errorf("remove allowed IP %s: %w", r.route.Network, decErr)
 	}
-	return nil
+	r.currentPeerKey = ""
+	return err
 }

go.mod

@@ -103,7 +103,6 @@ require (
 	github.com/rs/xid v1.3.0
 	github.com/shirou/gopsutil/v4 v4.25.8
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
-	github.com/soheilhy/cmux v0.1.5
 	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
 	github.com/stretchr/testify v1.11.1
 	github.com/testcontainers/testcontainers-go v0.37.0

go.sum

@@ -603,8 +603,6 @@ github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w
 github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
-github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
-github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
@@ -759,7 +757,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=

management/internals/server/boot.go

@@ -24,13 +24,13 @@ import (
 
 	"github.com/netbirdio/netbird/encryption"
 	"github.com/netbirdio/netbird/formatter/hook"
-	"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
 	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
 	accesslogsmanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs/manager"
 	rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
 	nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
 	"github.com/netbirdio/netbird/management/server/activity"
 	activitystore "github.com/netbirdio/netbird/management/server/activity/store"
+	"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
 	nbcache "github.com/netbirdio/netbird/management/server/cache"
 	nbContext "github.com/netbirdio/netbird/management/server/context"
 	nbhttp "github.com/netbirdio/netbird/management/server/http"
@@ -184,12 +184,7 @@ func (s *BaseServer) GRPCServer() *grpc.Server {
 			grpc.ChainStreamInterceptor(realip.StreamServerInterceptorOpts(realipOpts...), streamInterceptor, proxyStream),
 		}
 
-		// With the native transport enabled, TLS is terminated at the listeners
-		// (cmux-split shared listener and legacy port), so transport credentials
-		// must not be set or the server would attempt a second handshake.
-		if nativeGRPCEnabled() { //nolint:gocritic
-			log.Info("native gRPC transport enabled, TLS is terminated at the listeners")
-		} else if s.Config.HttpConfig.LetsEncryptDomain != "" {
+		if s.Config.HttpConfig.LetsEncryptDomain != "" {
 			certManager, err := encryption.CreateCertManager(s.Config.Datadir, s.Config.HttpConfig.LetsEncryptDomain)
 			if err != nil {
 				log.Fatalf("failed to create certificate service: %v", err)

management/internals/server/server.go

@@ -6,16 +6,12 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"os"
-	"slices"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
-	"github.com/soheilhy/cmux"
 	"go.opentelemetry.io/otel/metric"
 	"golang.org/x/crypto/acme/autocert"
 	"golang.org/x/net/http2"
@@ -40,18 +36,8 @@ const (
 	DefaultSelfHostedDomain = "netbird.selfhosted"
 
 	ContainerKeyBaseServer = "baseServer"
-
-	// NativeGRPCEnvVar enables serving gRPC on the native gRPC transport,
-	// multiplexed with HTTP on the shared listener, instead of through the
-	// net/http ServeHTTP path which costs two extra goroutines per stream.
-	NativeGRPCEnvVar = "NB_MGMT_NATIVE_GRPC"
 )
 
-func nativeGRPCEnabled() bool {
-	enabled, _ := strconv.ParseBool(os.Getenv(NativeGRPCEnvVar))
-	return enabled
-}
-
 type Server interface {
 	Start(ctx context.Context) error
 	Stop() error
@@ -196,22 +182,11 @@ func (s *BaseServer) Start(ctx context.Context) error {
 		}
 	}
 
-	// With the native transport enabled the gRPC server carries no transport
-	// credentials, so TLS must be terminated at each of its listeners.
-	var grpcTLSConfig *tls.Config
-	if nativeGRPCEnabled() {
-		if s.certManager != nil {
-			grpcTLSConfig = s.certManager.TLSConfig()
-		} else {
-			grpcTLSConfig = tlsConfig
-		}
-	}
-
 	var compatListener net.Listener
 	if s.mgmtPort != ManagementLegacyPort && !s.disableLegacyManagementPort {
 		// The Management gRPC server was running on port 33073 previously. Old agents that are already connected to it
 		// are using port 33073. For compatibility purposes we keep running a 2nd gRPC server on port 33073.
-		compatListener, err = s.serveGRPC(srvCtx, s.GRPCServer(), ManagementLegacyPort, grpcTLSConfig)
+		compatListener, err = s.serveGRPC(srvCtx, s.GRPCServer(), ManagementLegacyPort)
 		if err != nil {
 			return err
 		}
@@ -221,38 +196,22 @@ func (s *BaseServer) Start(ctx context.Context) error {
 	rootHandler := s.handlerFunc(srvCtx, s.GRPCServer(), s.APIHandler(), s.IDPHandler(), s.Metrics().GetMeter())
 	switch {
 	case s.certManager != nil:
+		// a call to certManager.Listener() always creates a new listener so we do it once
+		cml := s.certManager.Listener()
 		if s.mgmtPort == 443 {
 			// CertManager, HTTP and gRPC API all on the same port
 			rootHandler = s.certManager.HTTPHandler(rootHandler)
-			if nativeGRPCEnabled() {
-				var tcpListener net.Listener
-				tcpListener, err = net.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort))
-				if err != nil {
-					return fmt.Errorf("failed creating TCP listener on port %d: %v", s.mgmtPort, err)
-				}
-				s.listener = tls.NewListener(tcpListener, preferHTTP1ForDualProtoClients(s.certManager.TLSConfig()))
-			} else {
-				s.listener = s.certManager.Listener()
-			}
+			s.listener = cml
 		} else {
-			mgmtTLSConfig := s.certManager.TLSConfig()
-			if nativeGRPCEnabled() {
-				mgmtTLSConfig = preferHTTP1ForDualProtoClients(mgmtTLSConfig)
-			}
-			s.listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort), mgmtTLSConfig)
+			s.listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort), s.certManager.TLSConfig())
 			if err != nil {
 				return fmt.Errorf("failed creating TLS listener on port %d: %v", s.mgmtPort, err)
 			}
-			cml := s.certManager.Listener()
 			log.WithContext(ctx).Infof("running HTTP server (LetsEncrypt challenge handler): %s", cml.Addr().String())
 			s.serveHTTP(ctx, cml, s.certManager.HTTPHandler(nil))
 		}
 	case tlsConfig != nil:
-		mgmtTLSConfig := tlsConfig
-		if nativeGRPCEnabled() {
-			mgmtTLSConfig = preferHTTP1ForDualProtoClients(mgmtTLSConfig)
-		}
-		s.listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort), mgmtTLSConfig)
+		s.listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort), tlsConfig)
 		if err != nil {
 			return fmt.Errorf("failed creating TLS listener on port %d: %v", s.mgmtPort, err)
 		}
@@ -265,12 +224,7 @@ func (s *BaseServer) Start(ctx context.Context) error {
 
 	log.WithContext(ctx).Infof("management server version %s", version.NetbirdVersion())
 	log.WithContext(ctx).Infof("running HTTP server and gRPC server on the same port: %s", s.listener.Addr().String())
-	if nativeGRPCEnabled() {
-		log.WithContext(ctx).Infof("serving gRPC on the native transport (multiplexed with HTTP)")
-		s.serveMultiplexed(ctx, s.listener, s.GRPCServer(), rootHandler, tlsEnabled)
-	} else {
-		s.serveGRPCWithHTTP(ctx, s.listener, rootHandler, tlsEnabled)
-	}
+	s.serveGRPCWithHTTP(ctx, s.listener, rootHandler, tlsEnabled)
 
 	s.update = version.NewUpdateAndStart("nb/management")
 	s.update.SetDaemonVersion(version.NetbirdVersion())
@@ -377,14 +331,11 @@ func (s *BaseServer) handlerFunc(_ context.Context, gRPCHandler *grpc.Server, ht
 	})
 }
 
-func (s *BaseServer) serveGRPC(ctx context.Context, grpcServer *grpc.Server, port int, tlsConf *tls.Config) (net.Listener, error) {
+func (s *BaseServer) serveGRPC(ctx context.Context, grpcServer *grpc.Server, port int) (net.Listener, error) {
 	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
 	if err != nil {
 		return nil, err
 	}
-	if tlsConf != nil {
-		listener = tls.NewListener(listener, tlsConf)
-	}
 
 	s.wg.Add(1)
 	go func() {
@@ -448,69 +399,6 @@ func (s *BaseServer) serveGRPCWithHTTP(ctx context.Context, listener net.Listene
 	}()
 }
 
-// preferHTTP1ForDualProtoClients steers TLS clients that offer both "h2" and
-// "http/1.1" in ALPN (browsers, REST clients) to HTTP/1.1. gRPC clients offer
-// only "h2", so with this steering every HTTP/2 connection on the shared
-// listener carries gRPC and can be routed to the native transport without
-// inspecting frames. ACME "acme-tls/1" and single-protocol clients keep the
-// base configuration.
-func preferHTTP1ForDualProtoClients(base *tls.Config) *tls.Config {
-	h1Config := base.Clone()
-	h1Config.NextProtos = []string{"http/1.1"}
-	steered := base.Clone()
-	steered.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
-		if slices.Contains(hello.SupportedProtos, "http/1.1") && slices.Contains(hello.SupportedProtos, "h2") {
-			return h1Config, nil
-		}
-		return base, nil
-	}
-	return steered
-}
-
-// serveMultiplexed splits the shared listener by protocol: HTTP/2 connections
-// go to the native gRPC transport (see preferHTTP1ForDualProtoClients for why
-// they are all gRPC), everything else is served by net/http.
-//
-// Content-type based classification cannot be used here: cmux's SendSettings
-// matchers greet non-matching HTTP/2 connections and corrupt them for any
-// subsequent handler, while read-only matchers deadlock grpc-go clients,
-// which do not send HEADERS until they receive the server SETTINGS frame.
-func (s *BaseServer) serveMultiplexed(ctx context.Context, listener net.Listener, grpcServer *grpc.Server, handler http.Handler, tlsEnabled bool) {
-	mux := cmux.New(listener)
-	grpcListener := mux.Match(cmux.HTTP2())
-	httpListener := mux.Match(cmux.Any())
-
-	httpHandler := handler
-	if !tlsEnabled {
-		//nolint:staticcheck // h2c also handles the HTTP/1 Upgrade mechanism, which http.Server's UnencryptedHTTP2 does not
-		httpHandler = h2c.NewHandler(handler, &http2.Server{})
-	}
-
-	s.wg.Add(3)
-	go func() {
-		defer s.wg.Done()
-		s.reportServeError(ctx, grpcServer.Serve(grpcListener))
-	}()
-	go func() {
-		defer s.wg.Done()
-		s.reportServeError(ctx, http.Serve(httpListener, httpHandler))
-	}()
-	go func() {
-		defer s.wg.Done()
-		s.reportServeError(ctx, mux.Serve())
-	}()
-}
-
-func (s *BaseServer) reportServeError(ctx context.Context, err error) {
-	if ctx.Err() != nil || err == nil {
-		return
-	}
-	select {
-	case s.errCh <- err:
-	default:
-	}
-}
-
 // ResolveDomains determines dnsDomain and mgmtSingleAccModeDomain based on store state.
 // Fresh installs use the default self-hosted domain, while existing installs reuse the
 // persisted account domain to keep addressing stable across config changes.

management/internals/server/server_multiplex_test.go

@@ -1,109 +0,0 @@
-package server
-
-import (
-	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"fmt"
-	"io"
-	"math/big"
-	"net"
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/health"
-	healthpb "google.golang.org/grpc/health/grpc_health_v1"
-)
-
-func newSelfSignedCert(t *testing.T) tls.Certificate {
-	t.Helper()
-
-	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	require.NoError(t, err)
-
-	template := x509.Certificate{
-		SerialNumber: big.NewInt(1),
-		Subject:      pkix.Name{CommonName: "127.0.0.1"},
-		NotBefore:    time.Now().Add(-time.Hour),
-		NotAfter:     time.Now().Add(time.Hour),
-		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
-	}
-	der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
-	require.NoError(t, err)
-
-	return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
-}
-
-func TestServeMultiplexedRoutesProtocols(t *testing.T) {
-	tcpListener, err := net.Listen("tcp", "127.0.0.1:0")
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = tcpListener.Close() })
-
-	baseTLSConfig := &tls.Config{
-		Certificates: []tls.Certificate{newSelfSignedCert(t)},
-		NextProtos:   []string{"h2", "http/1.1"},
-	}
-	tlsListener := tls.NewListener(tcpListener, preferHTTP1ForDualProtoClients(baseTLSConfig))
-
-	grpcServer := grpc.NewServer()
-	healthpb.RegisterHealthServer(grpcServer, health.NewServer())
-	t.Cleanup(grpcServer.Stop)
-
-	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_, _ = fmt.Fprintf(w, "proto=%d", r.ProtoMajor)
-	})
-
-	s := &BaseServer{errCh: make(chan error, 4)}
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
-	s.serveMultiplexed(ctx, tlsListener, grpcServer, handler, true)
-
-	addr := tcpListener.Addr().String()
-	url := "https://" + addr + "/"
-
-	grpcConn, err := grpc.NewClient(addr,
-		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = grpcConn.Close() })
-
-	checkCtx, checkCancel := context.WithTimeout(ctx, 5*time.Second)
-	defer checkCancel()
-	resp, err := healthpb.NewHealthClient(grpcConn).Check(checkCtx, &healthpb.HealthCheckRequest{})
-	require.NoError(t, err)
-	require.Equal(t, healthpb.HealthCheckResponse_SERVING, resp.Status)
-
-	get := func(client *http.Client) string {
-		t.Helper()
-		res, err := client.Get(url)
-		require.NoError(t, err)
-		body, err := io.ReadAll(res.Body)
-		require.NoError(t, err)
-		_ = res.Body.Close()
-		return string(body)
-	}
-
-	dualProtoClient := &http.Client{
-		Timeout: 5 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig:   &tls.Config{InsecureSkipVerify: true},
-			ForceAttemptHTTP2: true,
-		},
-	}
-	require.Equal(t, "proto=1", get(dualProtoClient), "dual-ALPN client should be steered to HTTP/1.1")
-
-	h1OnlyClient := &http.Client{
-		Timeout: 5 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true, NextProtos: []string{"http/1.1"}},
-		},
-	}
-	require.Equal(t, "proto=1", get(h1OnlyClient))
-}

management/internals/shared/grpc/server.go

@@ -215,13 +215,12 @@ func (s *Server) Job(srv proto.ManagementService_JobServer) error {
 		return status.Errorf(codes.Unauthenticated, "peer is not registered")
 	}
 
-	stream := s.jobManager.RegisterStream(ctx, accountID, peer.ID, func(event *job.Event) error {
-		return s.sendJob(ctx, peerKey, event, srv)
-	})
-	defer s.jobManager.UnregisterStream(ctx, accountID, peer.ID, stream)
+	s.startResponseReceiver(ctx, srv)
+
+	updates := s.jobManager.CreateJobChannel(ctx, accountID, peer.ID)
 	log.WithContext(ctx).Debugf("Job: took %v", time.Since(reqStart))
 
-	return s.receiveJobResponses(ctx, peerKey, srv)
+	return s.sendJobsLoop(ctx, accountID, peerKey, peer, updates, srv)
 }
 
 // Sync validates the existence of a connecting peer, sends an initial state (all available for the connecting peers) and
@@ -363,26 +362,51 @@ func (s *Server) handleHandshake(ctx context.Context, srv proto.ManagementServic
 	return peerKey, nil
 }
 
-func (s *Server) receiveJobResponses(ctx context.Context, peerKey wgtypes.Key, srv proto.ManagementService_JobServer) error {
-	for {
-		msg, err := srv.Recv()
-		if err != nil {
-			if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || ctx.Err() != nil {
-				log.WithContext(ctx).Debugf("job stream of peer %s has been closed", peerKey.String())
-				return nil //nolint:nilerr
+func (s *Server) startResponseReceiver(ctx context.Context, srv proto.ManagementService_JobServer) {
+	go func() {
+		for {
+			msg, err := srv.Recv()
+			if err != nil {
+				if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
+					return
+				}
+				log.WithContext(ctx).Warnf("recv job response error: %v", err)
+				return
 			}
-			log.WithContext(ctx).Warnf("recv job response error: %v", err)
-			return err
+
+			jobResp := &proto.JobResponse{}
+			if _, err := s.parseRequest(ctx, msg, jobResp); err != nil {
+				log.WithContext(ctx).Warnf("invalid job response: %v", err)
+				continue
+			}
+
+			if err := s.jobManager.HandleResponse(ctx, jobResp, msg.WgPubKey); err != nil {
+				log.WithContext(ctx).Errorf("handle job response failed: %v", err)
+			}
+		}
+	}()
+}
+
+func (s *Server) sendJobsLoop(ctx context.Context, accountID string, peerKey wgtypes.Key, peer *nbpeer.Peer, updates *job.Channel, srv proto.ManagementService_JobServer) error {
+	// todo figure out better error handling strategy
+	defer s.jobManager.CloseChannel(ctx, accountID, peer.ID)
+
+	for {
+		event, err := updates.Event(ctx)
+		if err != nil {
+			if errors.Is(err, job.ErrJobChannelClosed) {
+				log.WithContext(ctx).Debugf("jobs channel for peer %s was closed", peerKey.String())
+				return nil
+			}
+
+			// happens when connection drops, e.g. client disconnects
+			log.WithContext(ctx).Debugf("stream of peer %s has been closed", peerKey.String())
+			return ctx.Err()
 		}
 
-		jobResp := &proto.JobResponse{}
-		if _, err := s.parseRequest(ctx, msg, jobResp); err != nil {
-			log.WithContext(ctx).Warnf("invalid job response: %v", err)
-			continue
-		}
-
-		if err := s.jobManager.HandleResponse(ctx, jobResp, msg.WgPubKey); err != nil {
-			log.WithContext(ctx).Errorf("handle job response failed: %v", err)
+		if err := s.sendJob(ctx, peerKey, event, srv); err != nil {
+			log.WithContext(ctx).Warnf("send job failed: %v", err)
+			return nil
 		}
 	}
 }

management/internals/shared/grpc/token_mgr.go

@@ -43,9 +43,8 @@ type TimeBasedAuthSecretsManager struct {
 	updateManager   network_map.PeersUpdateManager
 	settingsManager settings.Manager
 	groupsManager   groups.Manager
-	scheduler       *refreshScheduler
-	turnJobs        map[string]*refreshJob
-	relayJobs       map[string]*refreshJob
+	turnCancelMap   map[string]chan struct{}
+	relayCancelMap  map[string]chan struct{}
 	wgKey           wgtypes.Key
 }
 
@@ -61,6 +60,8 @@ func NewTimeBasedAuthSecretsManager(updateManager network_map.PeersUpdateManager
 		updateManager:   updateManager,
 		turnCfg:         turnCfg,
 		relayCfg:        relayCfg,
+		turnCancelMap:   make(map[string]chan struct{}),
+		relayCancelMap:  make(map[string]chan struct{}),
 		settingsManager: settingsManager,
 		groupsManager:   groupsManager,
 		wgKey:           key,
@@ -126,16 +127,16 @@ func (m *TimeBasedAuthSecretsManager) GenerateRelayToken() (*Token, error) {
 }
 
 func (m *TimeBasedAuthSecretsManager) cancelTURN(peerID string) {
-	if job, ok := m.turnJobs[peerID]; ok {
-		m.scheduler.cancel(job)
-		delete(m.turnJobs, peerID)
+	if channel, ok := m.turnCancelMap[peerID]; ok {
+		close(channel)
+		delete(m.turnCancelMap, peerID)
 	}
 }
 
 func (m *TimeBasedAuthSecretsManager) cancelRelay(peerID string) {
-	if job, ok := m.relayJobs[peerID]; ok {
-		m.scheduler.cancel(job)
-		delete(m.relayJobs, peerID)
+	if channel, ok := m.relayCancelMap[peerID]; ok {
+		close(channel)
+		delete(m.relayCancelMap, peerID)
 	}
 }
 
@@ -147,31 +148,6 @@ func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) {
 	m.cancelRelay(peerID)
 }
 
-func (m *TimeBasedAuthSecretsManager) ensureScheduler() {
-	if m.scheduler == nil {
-		m.scheduler = newRefreshScheduler(m.runRefreshJob)
-		m.turnJobs = make(map[string]*refreshJob)
-		m.relayJobs = make(map[string]*refreshJob)
-	}
-}
-
-func (m *TimeBasedAuthSecretsManager) runRefreshJob(job *refreshJob) {
-	switch job.kind {
-	case refreshKindTURN:
-		m.pushNewTURNAndRelayTokens(job.ctx, job.accountID, job.peerID)
-	case refreshKindRelay:
-		m.pushNewRelayTokens(job.ctx, job.accountID, job.peerID)
-	}
-}
-
-func refreshInterval(ttl time.Duration) time.Duration {
-	interval := ttl / 4 * 3
-	if interval <= 0 {
-		interval = defaultDuration / 4 * 3
-	}
-	return interval
-}
-
 // SetupRefresh starts peer credentials refresh
 func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, accountID, peerID string) {
 	m.mux.Lock()
@@ -181,38 +157,54 @@ func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, accountI
 	m.cancelRelay(peerID)
 
 	if m.turnCfg != nil && m.turnCfg.TimeBasedCredentials {
-		m.ensureScheduler()
-		job := &refreshJob{
-			ctx:       ctx,
-			accountID: accountID,
-			peerID:    peerID,
-			kind:      refreshKindTURN,
-			interval:  refreshInterval(m.turnCfg.CredentialsTTL.Duration),
-		}
-		m.turnJobs[peerID] = job
-		m.scheduler.schedule(job)
+		turnCancel := make(chan struct{}, 1)
+		m.turnCancelMap[peerID] = turnCancel
+		go m.refreshTURNTokens(ctx, accountID, peerID, turnCancel)
 		log.WithContext(ctx).Debugf("starting TURN refresh for %s", peerID)
 	} else {
 		log.WithContext(ctx).Debugf("no TURN configuration, skipping TURN refresh for %s", peerID)
 	}
 
 	if m.relayCfg != nil {
-		m.ensureScheduler()
-		job := &refreshJob{
-			ctx:       ctx,
-			accountID: accountID,
-			peerID:    peerID,
-			kind:      refreshKindRelay,
-			interval:  refreshInterval(m.relayCfg.CredentialsTTL.Duration),
-		}
-		m.relayJobs[peerID] = job
-		m.scheduler.schedule(job)
+		relayCancel := make(chan struct{}, 1)
+		m.relayCancelMap[peerID] = relayCancel
+		go m.refreshRelayTokens(ctx, accountID, peerID, relayCancel)
 		log.WithContext(ctx).Tracef("starting relay refresh for %s", peerID)
 	} else {
 		log.WithContext(ctx).Tracef("no relay configuration, skipping relay refresh for %s", peerID)
 	}
 }
 
+func (m *TimeBasedAuthSecretsManager) refreshTURNTokens(ctx context.Context, accountID, peerID string, cancel chan struct{}) {
+	ticker := time.NewTicker(m.turnCfg.CredentialsTTL.Duration / 4 * 3)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-cancel:
+			log.WithContext(ctx).Tracef("stopping TURN refresh for %s", peerID)
+			return
+		case <-ticker.C:
+			m.pushNewTURNAndRelayTokens(ctx, accountID, peerID)
+		}
+	}
+}
+
+func (m *TimeBasedAuthSecretsManager) refreshRelayTokens(ctx context.Context, accountID, peerID string, cancel chan struct{}) {
+	ticker := time.NewTicker(m.relayCfg.CredentialsTTL.Duration / 4 * 3)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-cancel:
+			log.WithContext(ctx).Tracef("stopping relay refresh for %s", peerID)
+			return
+		case <-ticker.C:
+			m.pushNewRelayTokens(ctx, accountID, peerID)
+		}
+	}
+}
+
 func (m *TimeBasedAuthSecretsManager) pushNewTURNAndRelayTokens(ctx context.Context, accountID, peerID string) {
 	turnToken, err := m.turnHmacToken.GenerateToken(sha1.New)
 	if err != nil {

management/internals/shared/grpc/token_mgr_test.go

@@ -112,12 +112,12 @@ func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) {
 
 	tested.SetupRefresh(ctx, "someAccountID", peer)
 
-	if _, ok := tested.turnJobs[peer]; !ok {
-		t.Errorf("expecting peer to be present in the turn jobs map, got not present")
+	if _, ok := tested.turnCancelMap[peer]; !ok {
+		t.Errorf("expecting peer to be present in the turn cancel map, got not present")
 	}
 
-	if _, ok := tested.relayJobs[peer]; !ok {
-		t.Errorf("expecting peer to be present in the relay jobs map, got not present")
+	if _, ok := tested.relayCancelMap[peer]; !ok {
+		t.Errorf("expecting peer to be present in the relay cancel map, got not present")
 	}
 
 	var updates []*network_map.UpdateMessage
@@ -212,26 +212,19 @@ func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) {
 	require.NoError(t, err)
 
 	tested.SetupRefresh(context.Background(), "someAccountID", peer)
-	if _, ok := tested.turnJobs[peer]; !ok {
-		t.Errorf("expecting peer to be present in turn jobs map, got not present")
+	if _, ok := tested.turnCancelMap[peer]; !ok {
+		t.Errorf("expecting peer to be present in turn cancel map, got not present")
 	}
-	if _, ok := tested.relayJobs[peer]; !ok {
-		t.Errorf("expecting peer to be present in relay jobs map, got not present")
+	if _, ok := tested.relayCancelMap[peer]; !ok {
+		t.Errorf("expecting peer to be present in relay cancel map, got not present")
 	}
 
 	tested.CancelRefresh(peer)
-	if _, ok := tested.turnJobs[peer]; ok {
-		t.Errorf("expecting peer to be not present in turn jobs map, got present")
+	if _, ok := tested.turnCancelMap[peer]; ok {
+		t.Errorf("expecting peer to be not present in turn cancel map, got present")
 	}
-	if _, ok := tested.relayJobs[peer]; ok {
-		t.Errorf("expecting peer to be not present in relay jobs map, got present")
-	}
-
-	tested.scheduler.mu.Lock()
-	heapLen := len(tested.scheduler.jobs)
-	tested.scheduler.mu.Unlock()
-	if heapLen != 0 {
-		t.Errorf("expecting scheduler heap to be empty after cancel, got %d entries", heapLen)
+	if _, ok := tested.relayCancelMap[peer]; ok {
+		t.Errorf("expecting peer to be not present in relay cancel map, got present")
 	}
 }
 

management/internals/shared/grpc/token_refresh_scheduler.go

@@ -1,158 +0,0 @@
-package grpc
-
-import (
-	"container/heap"
-	"context"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-const (
-	refreshWorkerCount   = 4
-	refreshWorkQueueSize = 1024
-)
-
-type refreshKind int
-
-const (
-	refreshKindTURN refreshKind = iota
-	refreshKindRelay
-)
-
-type refreshJob struct {
-	ctx       context.Context
-	accountID string
-	peerID    string
-	kind      refreshKind
-	interval  time.Duration
-	nextRun   time.Time
-	index     int
-	cancelled atomic.Bool
-}
-
-type refreshJobHeap []*refreshJob
-
-func (h refreshJobHeap) Len() int           { return len(h) }
-func (h refreshJobHeap) Less(i, j int) bool { return h[i].nextRun.Before(h[j].nextRun) }
-
-func (h refreshJobHeap) Swap(i, j int) {
-	h[i], h[j] = h[j], h[i]
-	h[i].index = i
-	h[j].index = j
-}
-
-func (h *refreshJobHeap) Push(x any) {
-	job := x.(*refreshJob)
-	job.index = len(*h)
-	*h = append(*h, job)
-}
-
-func (h *refreshJobHeap) Pop() any {
-	old := *h
-	n := len(old)
-	job := old[n-1]
-	old[n-1] = nil
-	job.index = -1
-	*h = old[:n-1]
-	return job
-}
-
-// refreshScheduler executes periodic credential refresh jobs for all peers
-// from one timer goroutine and a fixed worker pool, instead of two parked
-// goroutines per connected peer.
-type refreshScheduler struct {
-	mu   sync.Mutex
-	jobs refreshJobHeap
-	wake chan struct{}
-	work chan *refreshJob
-	run  func(job *refreshJob)
-}
-
-func newRefreshScheduler(run func(job *refreshJob)) *refreshScheduler {
-	s := &refreshScheduler{
-		wake: make(chan struct{}, 1),
-		work: make(chan *refreshJob, refreshWorkQueueSize),
-		run:  run,
-	}
-	go s.loop()
-	for range refreshWorkerCount {
-		go s.worker()
-	}
-	return s
-}
-
-func (s *refreshScheduler) schedule(job *refreshJob) {
-	s.mu.Lock()
-	job.nextRun = time.Now().Add(job.interval)
-	heap.Push(&s.jobs, job)
-	s.mu.Unlock()
-
-	select {
-	case s.wake <- struct{}{}:
-	default:
-	}
-}
-
-func (s *refreshScheduler) cancel(job *refreshJob) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	job.cancelled.Store(true)
-	if job.index >= 0 {
-		heap.Remove(&s.jobs, job.index)
-	}
-}
-
-func (s *refreshScheduler) loop() {
-	timer := time.NewTimer(time.Hour)
-	if !timer.Stop() {
-		<-timer.C
-	}
-
-	for {
-		s.mu.Lock()
-		now := time.Now()
-		var due []*refreshJob
-		for len(s.jobs) > 0 && !s.jobs[0].nextRun.After(now) {
-			job := s.jobs[0]
-			job.nextRun = job.nextRun.Add(job.interval)
-			if !job.nextRun.After(now) {
-				job.nextRun = now.Add(job.interval)
-			}
-			heap.Fix(&s.jobs, 0)
-			due = append(due, job)
-		}
-		wait := time.Duration(-1)
-		if len(s.jobs) > 0 {
-			wait = time.Until(s.jobs[0].nextRun)
-		}
-		s.mu.Unlock()
-
-		for _, job := range due {
-			s.work <- job
-		}
-
-		if wait < 0 {
-			<-s.wake
-			continue
-		}
-
-		timer.Reset(wait)
-		select {
-		case <-s.wake:
-			if !timer.Stop() {
-				<-timer.C
-			}
-		case <-timer.C:
-		}
-	}
-}
-
-func (s *refreshScheduler) worker() {
-	for job := range s.work {
-		if job.cancelled.Load() {
-			continue
-		}
-		s.run(job)
-	}
-}

management/internals/shared/grpc/token_refresh_scheduler_test.go

@@ -1,56 +0,0 @@
-package grpc
-
-import (
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestRefreshInterval(t *testing.T) {
-	defaultInterval := defaultDuration / 4 * 3
-
-	require.Equal(t, 9*time.Hour, refreshInterval(12*time.Hour))
-	require.Equal(t, defaultInterval, refreshInterval(0))
-	require.Equal(t, defaultInterval, refreshInterval(-time.Second))
-	require.Equal(t, defaultInterval, refreshInterval(3*time.Nanosecond))
-	require.Positive(t, refreshInterval(4*time.Nanosecond))
-}
-
-func TestWorkerSkipsCancelledJob(t *testing.T) {
-	var ran atomic.Int32
-	scheduler := newRefreshScheduler(func(*refreshJob) {
-		ran.Add(1)
-	})
-
-	cancelledJob := &refreshJob{interval: time.Hour}
-	cancelledJob.cancelled.Store(true)
-	liveJob := &refreshJob{interval: time.Hour}
-
-	scheduler.work <- cancelledJob
-	scheduler.work <- liveJob
-
-	require.Eventually(t, func() bool {
-		return ran.Load() == 1
-	}, 2*time.Second, 10*time.Millisecond, "live job should run exactly once, cancelled job never")
-}
-
-func TestCancelBeforeFirePreventsRun(t *testing.T) {
-	var ran atomic.Int32
-	scheduler := newRefreshScheduler(func(*refreshJob) {
-		ran.Add(1)
-	})
-
-	job := &refreshJob{interval: 50 * time.Millisecond}
-	scheduler.schedule(job)
-	scheduler.cancel(job)
-
-	time.Sleep(150 * time.Millisecond)
-	require.Zero(t, ran.Load(), "cancelled job must never fire")
-
-	scheduler.mu.Lock()
-	heapLen := len(scheduler.jobs)
-	scheduler.mu.Unlock()
-	require.Zero(t, heapLen)
-}

management/server/job/manager.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 
@@ -20,17 +21,11 @@ type Event struct {
 	Response *proto.JobResponse
 }
 
-// PeerStream is the send side of a peer's Job stream. Sends are serialized by
-// its mutex; the receive side runs on the stream's gRPC handler goroutine.
-type PeerStream struct {
-	send func(*Event) error
-	mu   sync.Mutex
-}
-
 type Manager struct {
 	mu           *sync.RWMutex
-	streams      map[string]*PeerStream // per-peer job streams
-	pending      map[string]*Event      // jobID → event
+	jobChannels  map[string]*Channel // per-peer job streams
+	pending      map[string]*Event   // jobID → event
+	responseWait time.Duration
 	metrics      telemetry.AppMetrics
 	Store        store.Store
 	peersManager peers.Manager
@@ -39,8 +34,9 @@ type Manager struct {
 func NewJobManager(metrics telemetry.AppMetrics, store store.Store, peersManager peers.Manager) *Manager {
 
 	return &Manager{
-		streams:      make(map[string]*PeerStream),
+		jobChannels:  make(map[string]*Channel),
 		pending:      make(map[string]*Event),
+		responseWait: 5 * time.Minute,
 		metrics:      metrics,
 		mu:           &sync.RWMutex{},
 		Store:        store,
@@ -48,9 +44,8 @@ func NewJobManager(metrics telemetry.AppMetrics, store store.Store, peersManager
 	}
 }
 
-// RegisterStream registers the send side of a peer's Job stream, replacing any
-// previous registration for the peer.
-func (jm *Manager) RegisterStream(ctx context.Context, accountID, peerID string, send func(*Event) error) *PeerStream {
+// CreateJobChannel creates or replaces a channel for a peer
+func (jm *Manager) CreateJobChannel(ctx context.Context, accountID, peerID string) *Channel {
 	// all pending jobs stored in db for this peer should be failed
 	if err := jm.Store.MarkAllPendingJobsAsFailed(ctx, accountID, peerID, "Pending job cleanup: marked as failed automatically due to being stuck too long"); err != nil {
 		log.WithContext(ctx).Error(err.Error())
@@ -59,41 +54,23 @@ func (jm *Manager) RegisterStream(ctx context.Context, accountID, peerID string,
 	jm.mu.Lock()
 	defer jm.mu.Unlock()
 
-	stream := &PeerStream{send: send}
-	jm.streams[peerID] = stream
-	return stream
-}
-
-// UnregisterStream removes a peer's stream registration and fails its pending
-// jobs. It is a no-op if the registration was already replaced by a newer
-// stream of the same peer.
-func (jm *Manager) UnregisterStream(ctx context.Context, accountID, peerID string, stream *PeerStream) {
-	jm.mu.Lock()
-	defer jm.mu.Unlock()
-
-	if jm.streams[peerID] != stream {
-		return
+	if ch, ok := jm.jobChannels[peerID]; ok {
+		ch.Close()
+		delete(jm.jobChannels, peerID)
 	}
-	delete(jm.streams, peerID)
 
-	for jobID, ev := range jm.pending {
-		if ev.PeerID == peerID {
-			// if the client disconnect and there is pending job then mark it as failed
-			if err := jm.Store.MarkPendingJobsAsFailed(ctx, accountID, peerID, jobID, "Time out peer disconnected"); err != nil {
-				log.WithContext(ctx).Errorf("failed to mark pending jobs as failed: %v", err)
-			}
-			delete(jm.pending, jobID)
-		}
-	}
+	ch := NewChannel()
+	jm.jobChannels[peerID] = ch
+	return ch
 }
 
 // SendJob sends a job to a peer and tracks it as pending
 func (jm *Manager) SendJob(ctx context.Context, accountID, peerID string, req *proto.JobRequest) error {
 	jm.mu.RLock()
-	stream, ok := jm.streams[peerID]
+	ch, ok := jm.jobChannels[peerID]
 	jm.mu.RUnlock()
 	if !ok {
-		return fmt.Errorf("peer %s has no stream", peerID)
+		return fmt.Errorf("peer %s has no channel", peerID)
 	}
 
 	event := &Event{
@@ -105,10 +82,7 @@ func (jm *Manager) SendJob(ctx context.Context, accountID, peerID string, req *p
 	jm.pending[string(req.ID)] = event
 	jm.mu.Unlock()
 
-	stream.mu.Lock()
-	err := stream.send(event)
-	stream.mu.Unlock()
-	if err != nil {
+	if err := ch.AddEvent(ctx, jm.responseWait, event); err != nil {
 		jm.cleanup(ctx, accountID, string(req.ID), err.Error())
 		return err
 	}
@@ -153,6 +127,27 @@ func (jm *Manager) HandleResponse(ctx context.Context, resp *proto.JobResponse,
 	return nil
 }
 
+// CloseChannel closes a peer’s channel and cleans up its jobs
+func (jm *Manager) CloseChannel(ctx context.Context, accountID, peerID string) {
+	jm.mu.Lock()
+	defer jm.mu.Unlock()
+
+	if ch, ok := jm.jobChannels[peerID]; ok {
+		ch.Close()
+		delete(jm.jobChannels, peerID)
+	}
+
+	for jobID, ev := range jm.pending {
+		if ev.PeerID == peerID {
+			// if the client disconnect and there is pending job then mark it as failed
+			if err := jm.Store.MarkPendingJobsAsFailed(ctx, accountID, peerID, jobID, "Time out peer disconnected"); err != nil {
+				log.WithContext(ctx).Errorf("failed to mark pending jobs as failed: %v", err)
+			}
+			delete(jm.pending, jobID)
+		}
+	}
+}
+
 // cleanup removes a pending job safely
 func (jm *Manager) cleanup(ctx context.Context, accountID, jobID string, reason string) {
 	jm.mu.Lock()
@@ -170,7 +165,7 @@ func (jm *Manager) IsPeerConnected(peerID string) bool {
 	jm.mu.RLock()
 	defer jm.mu.RUnlock()
 
-	_, ok := jm.streams[peerID]
+	_, ok := jm.jobChannels[peerID]
 	return ok
 }
 

management/server/job/manager_test.go

@@ -1,90 +0,0 @@
-package job
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/golang/mock/gomock"
-	"github.com/stretchr/testify/require"
-
-	"github.com/netbirdio/netbird/management/server/store"
-	"github.com/netbirdio/netbird/shared/management/proto"
-)
-
-func newTestManager(t *testing.T) (*Manager, *store.MockStore) {
-	t.Helper()
-	ctrl := gomock.NewController(t)
-	t.Cleanup(ctrl.Finish)
-	mockStore := store.NewMockStore(ctrl)
-	return NewJobManager(nil, mockStore, nil), mockStore
-}
-
-func TestSendJobDeliversThroughRegisteredStream(t *testing.T) {
-	ctx := context.Background()
-	manager, mockStore := newTestManager(t)
-	mockStore.EXPECT().MarkAllPendingJobsAsFailed(gomock.Any(), "acc", "peer1", gomock.Any()).Return(nil)
-
-	var sent []*Event
-	manager.RegisterStream(ctx, "acc", "peer1", func(event *Event) error {
-		sent = append(sent, event)
-		return nil
-	})
-	require.True(t, manager.IsPeerConnected("peer1"))
-
-	err := manager.SendJob(ctx, "acc", "peer1", &proto.JobRequest{ID: []byte("job1")})
-	require.NoError(t, err)
-	require.Len(t, sent, 1)
-	require.Equal(t, "peer1", sent[0].PeerID)
-	require.True(t, manager.IsPeerHasPendingJobs("peer1"))
-}
-
-func TestSendJobWithoutStream(t *testing.T) {
-	manager, _ := newTestManager(t)
-	err := manager.SendJob(context.Background(), "acc", "peer1", &proto.JobRequest{ID: []byte("job1")})
-	require.Error(t, err)
-}
-
-func TestSendJobFailureCleansPending(t *testing.T) {
-	ctx := context.Background()
-	manager, mockStore := newTestManager(t)
-	mockStore.EXPECT().MarkAllPendingJobsAsFailed(gomock.Any(), "acc", "peer1", gomock.Any()).Return(nil)
-	mockStore.EXPECT().MarkPendingJobsAsFailed(gomock.Any(), "acc", "peer1", "job1", gomock.Any()).Return(nil)
-
-	manager.RegisterStream(ctx, "acc", "peer1", func(*Event) error {
-		return errors.New("stream broken")
-	})
-
-	err := manager.SendJob(ctx, "acc", "peer1", &proto.JobRequest{ID: []byte("job1")})
-	require.Error(t, err)
-	require.False(t, manager.IsPeerHasPendingJobs("peer1"))
-}
-
-func TestUnregisterStreamIgnoresSupersededRegistration(t *testing.T) {
-	ctx := context.Background()
-	manager, mockStore := newTestManager(t)
-	mockStore.EXPECT().MarkAllPendingJobsAsFailed(gomock.Any(), "acc", "peer1", gomock.Any()).Return(nil).Times(2)
-
-	first := manager.RegisterStream(ctx, "acc", "peer1", func(*Event) error { return nil })
-	second := manager.RegisterStream(ctx, "acc", "peer1", func(*Event) error { return nil })
-
-	manager.UnregisterStream(ctx, "acc", "peer1", first)
-	require.True(t, manager.IsPeerConnected("peer1"), "stale unregister must not remove the replacement stream")
-
-	manager.UnregisterStream(ctx, "acc", "peer1", second)
-	require.False(t, manager.IsPeerConnected("peer1"))
-}
-
-func TestUnregisterStreamFailsPendingJobs(t *testing.T) {
-	ctx := context.Background()
-	manager, mockStore := newTestManager(t)
-	mockStore.EXPECT().MarkAllPendingJobsAsFailed(gomock.Any(), "acc", "peer1", gomock.Any()).Return(nil)
-	mockStore.EXPECT().MarkPendingJobsAsFailed(gomock.Any(), "acc", "peer1", "job1", gomock.Any()).Return(nil)
-
-	stream := manager.RegisterStream(ctx, "acc", "peer1", func(*Event) error { return nil })
-	require.NoError(t, manager.SendJob(ctx, "acc", "peer1", &proto.JobRequest{ID: []byte("job1")}))
-	require.True(t, manager.IsPeerHasPendingJobs("peer1"))
-
-	manager.UnregisterStream(ctx, "acc", "peer1", stream)
-	require.False(t, manager.IsPeerHasPendingJobs("peer1"))
-}