mirror of
https://github.com/netbirdio/netbird.git
synced 2026-07-22 08:21:30 +02:00
Compare commits
9 Commits
lazy-conn-
...
feature/na
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8fd098dcc7 | ||
|
|
820303c71d | ||
|
|
ad31406494 | ||
|
|
4d8d0e30db | ||
|
|
5aa2a748d7 | ||
|
|
e1a24376ab | ||
|
|
8f901f8899 | ||
|
|
c6bf5fbbfb | ||
|
|
e70a69bbcf |
@@ -17,7 +17,9 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal"
|
||||
"github.com/netbirdio/netbird/client/internal/auth"
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
nbnet "github.com/netbirdio/netbird/client/net"
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
"github.com/netbirdio/netbird/client/server"
|
||||
"github.com/netbirdio/netbird/client/system"
|
||||
"github.com/netbirdio/netbird/util"
|
||||
)
|
||||
@@ -331,6 +333,14 @@ func doForegroundLogin(ctx context.Context, cmd *cobra.Command, setupKey string,
|
||||
return fmt.Errorf("read config file %s: %v", configFilePath, err)
|
||||
}
|
||||
|
||||
// Mirror runInForegroundMode: recover residual state (DNS, firewall,
|
||||
// ssh config, legacy routing) from a previous unclean shutdown and
|
||||
// enable advanced routing before dialing management.
|
||||
if err := server.RestoreResidualState(ctx, profilemanager.NewServiceManager(configFilePath).GetStatePath()); err != nil {
|
||||
log.Warnf("failed to restore residual state: %v", err)
|
||||
}
|
||||
nbnet.Init()
|
||||
|
||||
err = foregroundLogin(ctx, cmd, config, setupKey, activeProf.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("foreground login failed: %v", err)
|
||||
|
||||
@@ -22,6 +22,8 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal/peer"
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
nbnet "github.com/netbirdio/netbird/client/net"
|
||||
"github.com/netbirdio/netbird/client/server"
|
||||
"github.com/netbirdio/netbird/client/system"
|
||||
"github.com/netbirdio/netbird/shared/management/domain"
|
||||
"github.com/netbirdio/netbird/util"
|
||||
@@ -229,6 +231,24 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command, activeProf *pr
|
||||
|
||||
_, _ = profilemanager.UpdateOldManagementURL(ctx, config, configFilePath)
|
||||
|
||||
// Restore residual state left by a previous run that did not shut down
|
||||
// cleanly, mirroring what the daemon does before connecting: it recovers
|
||||
// DNS config (a stale resolv.conf takeover can make the management
|
||||
// hostname unresolvable), firewall rules, ssh config and legacy routing.
|
||||
// Route cleanup itself happens at engine start; nbnet.Init() below lets
|
||||
// the management dial bypass a leftover fwmark rule until then.
|
||||
// Foreground mode is particularly exposed in containers: a crashed
|
||||
// container restarts inside the same (pod) network namespace, so stale
|
||||
// state survives while the process does not.
|
||||
if err := server.RestoreResidualState(ctx, profilemanager.NewServiceManager(configPath).GetStatePath()); err != nil {
|
||||
log.Warnf("failed to restore residual state: %v", err)
|
||||
}
|
||||
|
||||
// Enable advanced routing (as the daemon does on startup) so the
|
||||
// management dial bypasses a leftover fwmark rule instead of being
|
||||
// shunted into a stale routing table.
|
||||
nbnet.Init()
|
||||
|
||||
err = foregroundLogin(ctx, cmd, config, providedSetupKey, activeProf.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("foreground login failed: %v", err)
|
||||
|
||||
@@ -85,11 +85,6 @@ type Options struct {
|
||||
DisableIPv6 bool
|
||||
// BlockInbound blocks all inbound connections from peers
|
||||
BlockInbound bool
|
||||
// EnableRosenpass enables the Rosenpass post-quantum key exchange.
|
||||
EnableRosenpass bool
|
||||
// RosenpassPermissive lets a Rosenpass-enabled peer still connect to peers
|
||||
// that do not run Rosenpass (falling back to the plain WireGuard PSK).
|
||||
RosenpassPermissive bool
|
||||
// BlockLANAccess blocks the embedded peer from reaching the host's
|
||||
// LAN (RFC 1918, link-local, loopback) when it's used as a routing
|
||||
// peer. Mirrors profilemanager.ConfigInput.BlockLANAccess. Useful
|
||||
@@ -208,8 +203,6 @@ func New(opts Options) (*Client, error) {
|
||||
DisableIPv6: &opts.DisableIPv6,
|
||||
BlockInbound: &opts.BlockInbound,
|
||||
BlockLANAccess: &opts.BlockLANAccess,
|
||||
RosenpassEnabled: &opts.EnableRosenpass,
|
||||
RosenpassPermissive: &opts.RosenpassPermissive,
|
||||
WireguardPort: opts.WireguardPort,
|
||||
MTU: opts.MTU,
|
||||
DNSLabels: parsedLabels,
|
||||
|
||||
@@ -2,7 +2,6 @@ package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"maps"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
@@ -15,7 +14,6 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal/peer"
|
||||
"github.com/netbirdio/netbird/client/internal/peerstore"
|
||||
"github.com/netbirdio/netbird/route"
|
||||
mgmProto "github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
// lazyForce is the resolved local decision for lazy connections, layered above the
|
||||
@@ -37,18 +35,13 @@ const (
|
||||
//
|
||||
// The implementation is not thread-safe; it is protected by engine.syncMsgMux.
|
||||
type ConnMgr struct {
|
||||
peerStore *peerstore.Store
|
||||
statusRecorder *peer.Status
|
||||
iface lazyconn.WGIface
|
||||
force lazyForce
|
||||
// remoteLazyEnabled caches the account-wide lazy feature flag from management.
|
||||
// It is the default for peers that do not carry a per-peer lazy hint.
|
||||
remoteLazyEnabled bool
|
||||
peerStore *peerstore.Store
|
||||
statusRecorder *peer.Status
|
||||
iface lazyconn.WGIface
|
||||
force lazyForce
|
||||
rosenpassEnabled bool
|
||||
|
||||
lazyConnMgr *manager.Manager
|
||||
// appliedExcludeList is the exclude set last handed to the lazy manager, kept so an
|
||||
// unchanged set on the next sync skips the O(n) reconciliation.
|
||||
appliedExcludeList map[string]bool
|
||||
|
||||
wg sync.WaitGroup
|
||||
lazyCtx context.Context
|
||||
@@ -57,59 +50,78 @@ type ConnMgr struct {
|
||||
|
||||
func NewConnMgr(engineConfig *EngineConfig, statusRecorder *peer.Status, peerStore *peerstore.Store, iface lazyconn.WGIface) *ConnMgr {
|
||||
e := &ConnMgr{
|
||||
peerStore: peerStore,
|
||||
statusRecorder: statusRecorder,
|
||||
iface: iface,
|
||||
force: resolveLazyForce(engineConfig.LazyConnection),
|
||||
peerStore: peerStore,
|
||||
statusRecorder: statusRecorder,
|
||||
iface: iface,
|
||||
force: resolveLazyForce(engineConfig.LazyConnection),
|
||||
rosenpassEnabled: engineConfig.RosenpassEnabled,
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// Start initializes the connection manager. The lazy connection manager always runs so that
|
||||
// per-peer lazy defaults (e.g. proxy peers) work even when the account flag is off; the
|
||||
// account flag and the local override decide the default lazy state per peer (see
|
||||
// PeerLazyDefault). Rosenpass peers stay lazy-capable too: their connections just never idle
|
||||
// on their own, since rosenpass rekey traffic keeps them active.
|
||||
// Start initializes the connection manager. It starts the lazy connection manager when a
|
||||
// local override forces it on; with no local override it waits for the management feature flag.
|
||||
func (e *ConnMgr) Start(ctx context.Context) {
|
||||
if e.lazyConnMgr != nil {
|
||||
log.Errorf("lazy connection manager is already started")
|
||||
return
|
||||
}
|
||||
|
||||
e.initLazyManager(ctx)
|
||||
e.statusRecorder.UpdateLazyConnection(e.PeerLazyDefault(mgmProto.LazyState_LazyStateDefault))
|
||||
}
|
||||
|
||||
// UpdatedRemoteFeatureFlag caches the account-wide lazy feature flag. The manager itself is
|
||||
// not started or stopped here; the per-sync exclude-list reconciliation moves normal peers
|
||||
// between the lazy and always-active sets when the flag flips.
|
||||
func (e *ConnMgr) UpdatedRemoteFeatureFlag(_ context.Context, enabled bool) error {
|
||||
e.remoteLazyEnabled = enabled
|
||||
if e.isStartedWithLazyMgr() {
|
||||
e.statusRecorder.UpdateLazyConnection(e.PeerLazyDefault(mgmProto.LazyState_LazyStateDefault))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PeerLazyDefault reports whether a peer should be lazy. The local override
|
||||
// (NB_LAZY_CONN/MDM) wins over everything; without a local override the
|
||||
// management per-peer state applies (LazyStateLazy/Eager force the decision),
|
||||
// and LazyStateDefault follows the account-wide flag.
|
||||
func (e *ConnMgr) PeerLazyDefault(state mgmProto.LazyState) bool {
|
||||
switch e.force {
|
||||
case lazyForceOn:
|
||||
return true
|
||||
case lazyForceOff:
|
||||
return false
|
||||
log.Infof("lazy connection manager is disabled by local override (%s or MDM policy)", lazyconn.EnvLazyConn)
|
||||
e.statusRecorder.UpdateLazyConnection(false)
|
||||
return
|
||||
case lazyForceNone:
|
||||
log.Infof("lazy connection manager is managed by the management feature flag")
|
||||
e.statusRecorder.UpdateLazyConnection(false)
|
||||
return
|
||||
}
|
||||
|
||||
switch state {
|
||||
case mgmProto.LazyState_LazyStateLazy:
|
||||
return true
|
||||
case mgmProto.LazyState_LazyStateEager:
|
||||
return false
|
||||
default:
|
||||
return e.remoteLazyEnabled
|
||||
if e.rosenpassEnabled {
|
||||
log.Warnf("rosenpass connection manager is enabled, lazy connection manager will not be started")
|
||||
e.statusRecorder.UpdateLazyConnection(false)
|
||||
return
|
||||
}
|
||||
|
||||
e.initLazyManager(ctx)
|
||||
e.statusRecorder.UpdateLazyConnection(true)
|
||||
}
|
||||
|
||||
// UpdatedRemoteFeatureFlag is called when the remote feature flag is updated.
|
||||
// If enabled, it initializes the lazy connection manager and start it. Do not need to call Start() again.
|
||||
// If disabled, then it closes the lazy connection manager and open the connections to all peers.
|
||||
func (e *ConnMgr) UpdatedRemoteFeatureFlag(ctx context.Context, enabled bool) error {
|
||||
// a local override (NB_LAZY_CONN or local config) takes precedence over management
|
||||
if e.force != lazyForceNone {
|
||||
return nil
|
||||
}
|
||||
|
||||
if enabled {
|
||||
// if the lazy connection manager is already started, do not start it again
|
||||
if e.lazyConnMgr != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if e.rosenpassEnabled {
|
||||
log.Infof("rosenpass connection manager is enabled, lazy connection manager will not be started")
|
||||
e.statusRecorder.UpdateLazyConnection(false)
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("lazy connection manager is enabled by the management feature flag")
|
||||
e.initLazyManager(ctx)
|
||||
e.statusRecorder.UpdateLazyConnection(true)
|
||||
return e.addPeersToLazyConnManager()
|
||||
} else {
|
||||
if e.lazyConnMgr == nil {
|
||||
e.statusRecorder.UpdateLazyConnection(false)
|
||||
return nil
|
||||
}
|
||||
log.Infof("lazy connection manager is disabled by management feature flag")
|
||||
e.closeManager(ctx)
|
||||
e.statusRecorder.UpdateLazyConnection(false)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,13 +141,6 @@ func (e *ConnMgr) SetExcludeList(ctx context.Context, peerIDs map[string]bool) {
|
||||
return
|
||||
}
|
||||
|
||||
// The exclude set is recomputed every sync but rarely changes; skip the O(n)
|
||||
// store lookups and reconciliation when it matches what was already applied.
|
||||
if maps.Equal(peerIDs, e.appliedExcludeList) {
|
||||
return
|
||||
}
|
||||
e.appliedExcludeList = maps.Clone(peerIDs)
|
||||
|
||||
excludedPeers := make([]lazyconn.PeerConfig, 0, len(peerIDs))
|
||||
|
||||
for peerID := range peerIDs {
|
||||
@@ -171,16 +176,12 @@ func (e *ConnMgr) SetExcludeList(ctx context.Context, peerIDs map[string]bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// AddPeerConn registers a peer connection. permanent requests an always-active connection
|
||||
// (the peer belongs to the exclude set: a forwarder, or a peer that is not lazy by policy).
|
||||
// Non-permanent peers are handed to the lazy manager. The subsequent SetExcludeList call
|
||||
// reconciles membership for existing peers across flag flips.
|
||||
func (e *ConnMgr) AddPeerConn(ctx context.Context, peerKey string, conn *peer.Conn, permanent bool) (exists bool) {
|
||||
func (e *ConnMgr) AddPeerConn(ctx context.Context, peerKey string, conn *peer.Conn) (exists bool) {
|
||||
if success := e.peerStore.AddPeerConn(peerKey, conn); !success {
|
||||
return true
|
||||
}
|
||||
|
||||
if !e.isStartedWithLazyMgr() || permanent {
|
||||
if !e.isStartedWithLazyMgr() {
|
||||
if err := conn.Open(ctx); err != nil {
|
||||
conn.Log.Errorf("failed to open connection: %v", err)
|
||||
}
|
||||
@@ -268,7 +269,6 @@ func (e *ConnMgr) Close() {
|
||||
e.lazyCtxCancel()
|
||||
e.wg.Wait()
|
||||
e.lazyConnMgr = nil
|
||||
e.appliedExcludeList = nil
|
||||
}
|
||||
|
||||
func (e *ConnMgr) initLazyManager(engineCtx context.Context) {
|
||||
@@ -276,7 +276,6 @@ func (e *ConnMgr) initLazyManager(engineCtx context.Context) {
|
||||
InactivityThreshold: inactivityThresholdEnv(),
|
||||
}
|
||||
e.lazyConnMgr = manager.NewManager(cfg, engineCtx, e.peerStore, e.iface)
|
||||
e.appliedExcludeList = nil
|
||||
|
||||
e.lazyCtx, e.lazyCtxCancel = context.WithCancel(engineCtx)
|
||||
|
||||
@@ -287,6 +286,43 @@ func (e *ConnMgr) initLazyManager(engineCtx context.Context) {
|
||||
}()
|
||||
}
|
||||
|
||||
func (e *ConnMgr) addPeersToLazyConnManager() error {
|
||||
peers := e.peerStore.PeersPubKey()
|
||||
lazyPeerCfgs := make([]lazyconn.PeerConfig, 0, len(peers))
|
||||
for _, peerID := range peers {
|
||||
var peerConn *peer.Conn
|
||||
var exists bool
|
||||
if peerConn, exists = e.peerStore.PeerConn(peerID); !exists {
|
||||
log.Warnf("failed to find peer conn for peerID: %s", peerID)
|
||||
continue
|
||||
}
|
||||
|
||||
lazyPeerCfg := lazyconn.PeerConfig{
|
||||
PublicKey: peerID,
|
||||
AllowedIPs: peerConn.WgConfig().AllowedIps,
|
||||
PeerConnID: peerConn.ConnID(),
|
||||
Log: peerConn.Log,
|
||||
}
|
||||
lazyPeerCfgs = append(lazyPeerCfgs, lazyPeerCfg)
|
||||
}
|
||||
|
||||
return e.lazyConnMgr.AddActivePeers(lazyPeerCfgs)
|
||||
}
|
||||
|
||||
func (e *ConnMgr) closeManager(ctx context.Context) {
|
||||
if e.lazyConnMgr == nil {
|
||||
return
|
||||
}
|
||||
|
||||
e.lazyCtxCancel()
|
||||
e.wg.Wait()
|
||||
e.lazyConnMgr = nil
|
||||
|
||||
for _, peerID := range e.peerStore.PeersPubKey() {
|
||||
e.peerStore.PeerConnOpen(ctx, peerID)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *ConnMgr) isStartedWithLazyMgr() bool {
|
||||
return e.lazyConnMgr != nil && e.lazyCtxCancel != nil
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/netbirdio/netbird/client/internal/lazyconn"
|
||||
mgmProto "github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
func TestResolveLazyForce(t *testing.T) {
|
||||
@@ -39,90 +38,3 @@ func TestResolveLazyForce(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeerLazyDefault(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
force lazyForce
|
||||
remoteEnabled bool
|
||||
state mgmProto.LazyState
|
||||
want bool
|
||||
}{
|
||||
{name: "force on wins over eager state", force: lazyForceOn, state: mgmProto.LazyState_LazyStateEager, want: true},
|
||||
{name: "force off wins over lazy state", force: lazyForceOff, remoteEnabled: true, state: mgmProto.LazyState_LazyStateLazy, want: false},
|
||||
{name: "none, default, account off -> active", force: lazyForceNone, state: mgmProto.LazyState_LazyStateDefault, want: false},
|
||||
{name: "none, default, account on -> lazy", force: lazyForceNone, remoteEnabled: true, state: mgmProto.LazyState_LazyStateDefault, want: true},
|
||||
{name: "none, lazy state, account off -> lazy", force: lazyForceNone, state: mgmProto.LazyState_LazyStateLazy, want: true},
|
||||
{name: "none, eager state, account on -> active", force: lazyForceNone, remoteEnabled: true, state: mgmProto.LazyState_LazyStateEager, want: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
e := &ConnMgr{force: tt.force, remoteLazyEnabled: tt.remoteEnabled}
|
||||
if got := e.PeerLazyDefault(tt.state); got != tt.want {
|
||||
t.Fatalf("PeerLazyDefault(%v) = %v, want %v", tt.state, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestToExcludedLazyPeers covers the per-peer lazy classification (proxy vs
|
||||
// normal, across the force/account-flag matrix). Forwarder-target exclusion is
|
||||
// covered by TestToExcludedLazyPeers_ForwardTarget.
|
||||
func TestToExcludedLazyPeers(t *testing.T) {
|
||||
const (
|
||||
normalKey = "normal"
|
||||
lazyKey = "lazy-state"
|
||||
eagerKey = "eager-state"
|
||||
)
|
||||
|
||||
peers := []*mgmProto.RemotePeerConfig{
|
||||
{WgPubKey: normalKey, AllowedIps: []string{"100.64.0.1/32"}},
|
||||
{WgPubKey: lazyKey, AllowedIps: []string{"100.64.0.2/32"}, LazyState: mgmProto.LazyState_LazyStateLazy},
|
||||
{WgPubKey: eagerKey, AllowedIps: []string{"100.64.0.3/32"}, LazyState: mgmProto.LazyState_LazyStateEager},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
force lazyForce
|
||||
remoteEnabled bool
|
||||
want map[string]bool
|
||||
}{
|
||||
{
|
||||
name: "account off: lazy-state peer lazy, normal + eager active",
|
||||
force: lazyForceNone, remoteEnabled: false,
|
||||
want: map[string]bool{normalKey: true, eagerKey: true},
|
||||
},
|
||||
{
|
||||
name: "account on: only eager-state peer active",
|
||||
force: lazyForceNone, remoteEnabled: true,
|
||||
want: map[string]bool{eagerKey: true},
|
||||
},
|
||||
{
|
||||
name: "force off: everything active",
|
||||
force: lazyForceOff, remoteEnabled: true,
|
||||
want: map[string]bool{normalKey: true, lazyKey: true, eagerKey: true},
|
||||
},
|
||||
{
|
||||
name: "force on: nothing active",
|
||||
force: lazyForceOn, remoteEnabled: false,
|
||||
want: map[string]bool{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
e := &Engine{connMgr: &ConnMgr{force: tt.force, remoteLazyEnabled: tt.remoteEnabled}}
|
||||
got := e.toExcludedLazyPeers(peers)
|
||||
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("toExcludedLazyPeers() = %v, want %v", got, tt.want)
|
||||
}
|
||||
for k := range tt.want {
|
||||
if !got[k] {
|
||||
t.Fatalf("expected peer %s excluded, got %v", k, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -842,7 +842,8 @@ func (e *Engine) modifyPeers(peersUpdate []*mgmProto.RemotePeerConfig) error {
|
||||
}
|
||||
// third, add the peer connections again
|
||||
for _, p := range modified {
|
||||
if err := e.addNewPeer(p); err != nil {
|
||||
err := e.addNewPeer(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1402,12 +1403,8 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only update the flag when the sync carries a peer config; a nil peer config
|
||||
// (e.g. a partial update) must not reset the cached flag to false.
|
||||
if peerConfig := networkMap.GetPeerConfig(); peerConfig != nil {
|
||||
if err := e.connMgr.UpdatedRemoteFeatureFlag(e.ctx, peerConfig.GetLazyConnectionEnabled()); err != nil {
|
||||
log.Errorf("failed to update lazy connection feature flag: %v", err)
|
||||
}
|
||||
if err := e.connMgr.UpdatedRemoteFeatureFlag(e.ctx, networkMap.GetPeerConfig().GetLazyConnectionEnabled()); err != nil {
|
||||
log.Errorf("failed to update lazy connection feature flag: %v", err)
|
||||
}
|
||||
|
||||
if e.firewall != nil {
|
||||
@@ -1473,7 +1470,8 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error {
|
||||
|
||||
// Ingress forward rules
|
||||
done = e.phase("forward_rules")
|
||||
if _, err := e.updateForwardRules(networkMap.GetForwardingRules()); err != nil {
|
||||
forwardingRules, err := e.updateForwardRules(networkMap.GetForwardingRules())
|
||||
if err != nil {
|
||||
log.Errorf("failed to update forward rules, err: %v", err)
|
||||
}
|
||||
done()
|
||||
@@ -1491,7 +1489,8 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error {
|
||||
|
||||
// must set the exclude list after the peers are added. Without it the manager can not figure out the peers parameters from the store
|
||||
done = e.phase("lazy_exclude")
|
||||
e.connMgr.SetExcludeList(e.ctx, e.toExcludedLazyPeers(remotePeers))
|
||||
excludedLazyPeers := e.toExcludedLazyPeers(forwardingRules, remotePeers)
|
||||
e.connMgr.SetExcludeList(e.ctx, excludedLazyPeers)
|
||||
done()
|
||||
|
||||
e.networkSerial = serial
|
||||
@@ -1735,15 +1734,15 @@ func addrToString(addr netip.Addr) string {
|
||||
// addNewPeers adds peers that were not know before but arrived from the Management service with the update
|
||||
func (e *Engine) addNewPeers(peersUpdate []*mgmProto.RemotePeerConfig) error {
|
||||
for _, p := range peersUpdate {
|
||||
if err := e.addNewPeer(p); err != nil {
|
||||
err := e.addNewPeer(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// addNewPeer add peer if connection doesn't exist. A peer that is not lazy by
|
||||
// policy gets an always-active connection instead.
|
||||
// addNewPeer add peer if connection doesn't exist
|
||||
func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error {
|
||||
peerKey := peerConfig.GetWgPubKey()
|
||||
peerIPs := make([]netip.Prefix, 0, len(peerConfig.GetAllowedIps()))
|
||||
@@ -1778,8 +1777,7 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error {
|
||||
log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err)
|
||||
}
|
||||
|
||||
permanent := !e.connMgr.PeerLazyDefault(peerConfig.GetLazyState())
|
||||
if exists := e.connMgr.AddPeerConn(e.ctx, peerKey, conn, permanent); exists {
|
||||
if exists := e.connMgr.AddPeerConn(e.ctx, peerKey, conn); exists {
|
||||
conn.Close(false)
|
||||
return fmt.Errorf("peer already exists: %s", peerKey)
|
||||
}
|
||||
@@ -2605,19 +2603,46 @@ func (e *Engine) updateForwardRules(rules []*mgmProto.ForwardingRule) ([]firewal
|
||||
return forwardingRules, nberrors.FormatErrorOrNil(merr)
|
||||
}
|
||||
|
||||
// toExcludedLazyPeers returns the peers that must have an always-active
|
||||
// connection: those that are not lazy by policy (the per-peer lazy state or the
|
||||
// account flag, subject to the local override).
|
||||
func (e *Engine) toExcludedLazyPeers(peers []*mgmProto.RemotePeerConfig) map[string]bool {
|
||||
func (e *Engine) toExcludedLazyPeers(rules []firewallManager.ForwardRule, peers []*mgmProto.RemotePeerConfig) map[string]bool {
|
||||
excludedPeers := make(map[string]bool)
|
||||
for _, p := range peers {
|
||||
if !e.connMgr.PeerLazyDefault(p.GetLazyState()) {
|
||||
excludedPeers[p.GetWgPubKey()] = true
|
||||
|
||||
// Ingress forward targets: inbound forwarded traffic is initiated remotely and
|
||||
// cannot wake a lazy connection, so the peer routing the target must stay
|
||||
// permanently connected. AllowedIPs are already parsed on the peer conn, so
|
||||
// reuse those typed prefixes instead of re-parsing the network map strings.
|
||||
for _, r := range rules {
|
||||
for _, p := range peers {
|
||||
if e.peerRoutesAddr(p, r.TranslatedAddress) {
|
||||
log.Infof("exclude forwarder peer from lazy connection: %s", p.GetWgPubKey())
|
||||
excludedPeers[p.GetWgPubKey()] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return excludedPeers
|
||||
}
|
||||
|
||||
// peerRoutesAddr reports whether the peer is a router for addr, matched against
|
||||
// the peer's already-parsed AllowedIPs from the store (the same typed value the
|
||||
// lazy manager consumes) rather than re-parsing the network map strings.
|
||||
func (e *Engine) peerRoutesAddr(p *mgmProto.RemotePeerConfig, addr netip.Addr) bool {
|
||||
prefixes, ok := e.peerStore.AllowedIPs(p.GetWgPubKey())
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return prefixesContain(prefixes, addr)
|
||||
}
|
||||
|
||||
// prefixesContain reports whether addr falls within any of the prefixes.
|
||||
func prefixesContain(prefixes []netip.Prefix, addr netip.Addr) bool {
|
||||
for _, prefix := range prefixes {
|
||||
if prefix.Contains(addr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isChecksEqual checks if two slices of checks are equal.
|
||||
func isChecksEqual(checks1, checks2 []*mgmProto.Checks) bool {
|
||||
normalize := func(checks []*mgmProto.Checks) []string {
|
||||
|
||||
87
client/internal/engine_lazy_exclude_test.go
Normal file
87
client/internal/engine_lazy_exclude_test.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
firewallManager "github.com/netbirdio/netbird/client/firewall/manager"
|
||||
"github.com/netbirdio/netbird/client/internal/peer"
|
||||
"github.com/netbirdio/netbird/client/internal/peerstore"
|
||||
mgmProto "github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
func TestPrefixesContain(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefixes []string
|
||||
addr string
|
||||
want bool
|
||||
}{
|
||||
{name: "own overlay /32 matches", prefixes: []string{"100.110.8.145/32"}, addr: "100.110.8.145", want: true},
|
||||
{name: "addr inside routed subnet", prefixes: []string{"10.121.0.0/16"}, addr: "10.121.208.4", want: true},
|
||||
{name: "addr outside subnet", prefixes: []string{"10.121.0.0/16"}, addr: "10.122.0.1", want: false},
|
||||
{name: "different /32", prefixes: []string{"100.110.8.145/32"}, addr: "100.110.8.146", want: false},
|
||||
{name: "ipv6 /128 matches", prefixes: []string{"fd00::1/128"}, addr: "fd00::1", want: true},
|
||||
{name: "no prefixes", prefixes: nil, addr: "10.121.208.4", want: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
prefixes := make([]netip.Prefix, 0, len(tt.prefixes))
|
||||
for _, p := range tt.prefixes {
|
||||
prefixes = append(prefixes, netip.MustParsePrefix(p))
|
||||
}
|
||||
require.Equal(t, tt.want, prefixesContain(prefixes, netip.MustParseAddr(tt.addr)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestToExcludedLazyPeers_ForwardTarget guards a regression: the forward-target
|
||||
// peer (the peer routing a ForwardRule.TranslatedAddress) must be excluded from
|
||||
// lazy connections, matched via the peer's already-parsed AllowedIPs.
|
||||
func TestToExcludedLazyPeers_ForwardTarget(t *testing.T) {
|
||||
const targetPeerKey = "cccccccccccccccccccccccccccccccccccccccccc0="
|
||||
const otherPeerKey = "dddddddddddddddddddddddddddddddddddddddddd0="
|
||||
|
||||
store := peerstore.NewConnStore()
|
||||
store.AddPeerConn(targetPeerKey, newTestConn(t, targetPeerKey, "100.110.8.145/32"))
|
||||
store.AddPeerConn(otherPeerKey, newTestConn(t, otherPeerKey, "100.110.9.10/32"))
|
||||
|
||||
e := &Engine{peerStore: store}
|
||||
|
||||
peers := []*mgmProto.RemotePeerConfig{
|
||||
{WgPubKey: targetPeerKey, AllowedIps: []string{"100.110.8.145/32"}},
|
||||
{WgPubKey: otherPeerKey, AllowedIps: []string{"100.110.9.10/32"}},
|
||||
}
|
||||
rules := []firewallManager.ForwardRule{
|
||||
{TranslatedAddress: netip.MustParseAddr("100.110.8.145")},
|
||||
}
|
||||
|
||||
excluded := e.toExcludedLazyPeers(rules, peers)
|
||||
|
||||
require.True(t, excluded[targetPeerKey], "forward-target peer must be excluded from lazy connections")
|
||||
require.False(t, excluded[otherPeerKey], "non-target peer must not be excluded")
|
||||
require.Len(t, excluded, 1)
|
||||
}
|
||||
|
||||
func TestToExcludedLazyPeers_NoRules(t *testing.T) {
|
||||
e := &Engine{peerStore: peerstore.NewConnStore()}
|
||||
|
||||
peers := []*mgmProto.RemotePeerConfig{
|
||||
{WgPubKey: "peer-a", AllowedIps: []string{"100.110.8.145/32"}},
|
||||
}
|
||||
|
||||
require.Empty(t, e.toExcludedLazyPeers(nil, peers))
|
||||
}
|
||||
|
||||
func newTestConn(t *testing.T, key, allowedIP string) *peer.Conn {
|
||||
t.Helper()
|
||||
conn, err := peer.NewConn(peer.ConnConfig{
|
||||
Key: key,
|
||||
WgConfig: peer.WgConfig{AllowedIps: []netip.Prefix{netip.MustParsePrefix(allowedIP)}},
|
||||
}, peer.ServiceDependencies{})
|
||||
require.NoError(t, err)
|
||||
return conn
|
||||
}
|
||||
@@ -279,8 +279,7 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
|
||||
}, MobileDependency{})
|
||||
|
||||
wgIface := &MockWGIface{
|
||||
NameFunc: func() string { return "utun102" },
|
||||
IsUserspaceBindFunc: func() bool { return true },
|
||||
NameFunc: func() string { return "utun102" },
|
||||
RemovePeerFunc: func(peerKey string) error {
|
||||
return nil
|
||||
},
|
||||
|
||||
@@ -181,7 +181,7 @@ func (s *Server) Start() error {
|
||||
log.Warnf("failed to redirect stderr: %v", err)
|
||||
}
|
||||
|
||||
if err := restoreResidualState(s.rootCtx, s.profileManager.GetStatePath()); err != nil {
|
||||
if err := RestoreResidualState(s.rootCtx, s.profileManager.GetStatePath()); err != nil {
|
||||
log.Warnf(errRestoreResidualState, err)
|
||||
}
|
||||
|
||||
@@ -551,7 +551,7 @@ func (s *Server) Login(callerCtx context.Context, msg *proto.LoginRequest) (*pro
|
||||
s.actCancel = cancel
|
||||
s.mutex.Unlock()
|
||||
|
||||
if err := restoreResidualState(s.rootCtx, s.profileManager.GetStatePath()); err != nil {
|
||||
if err := RestoreResidualState(s.rootCtx, s.profileManager.GetStatePath()); err != nil {
|
||||
log.Warnf(errRestoreResidualState, err)
|
||||
}
|
||||
|
||||
@@ -858,7 +858,7 @@ func (s *Server) Up(callerCtx context.Context, msg *proto.UpRequest) (*proto.UpR
|
||||
|
||||
return s.waitForUp(callerCtx)
|
||||
}
|
||||
if err := restoreResidualState(callerCtx, s.profileManager.GetStatePath()); err != nil {
|
||||
if err := RestoreResidualState(callerCtx, s.profileManager.GetStatePath()); err != nil {
|
||||
log.Warnf(errRestoreResidualState, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ func (s *Server) CleanState(ctx context.Context, req *proto.CleanStateRequest) (
|
||||
|
||||
if req.All {
|
||||
// Reuse existing cleanup logic for all states
|
||||
if err := restoreResidualState(ctx, statePath); err != nil {
|
||||
if err := RestoreResidualState(ctx, statePath); err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to clean all states: %v", err)
|
||||
}
|
||||
|
||||
@@ -113,9 +113,9 @@ func (s *Server) DeleteState(ctx context.Context, req *proto.DeleteStateRequest)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// restoreResidualState checks if the client was not shut down in a clean way and restores residual if required.
|
||||
// RestoreResidualState checks if the client was not shut down in a clean way and restores residual if required.
|
||||
// Otherwise, we might not be able to connect to the management server to retrieve new config.
|
||||
func restoreResidualState(ctx context.Context, statePath string) error {
|
||||
func RestoreResidualState(ctx context.Context, statePath string) error {
|
||||
if statePath == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ func availableProviders() []providerCase {
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
ps = append(ps, providerCase{name: "bedrock", catalogID: "bedrock_api", upstream: "https://bedrock-runtime." + region + ".amazonaws.com", apiKey: k, model: "us.anthropic.claude-haiku-4-5", kind: harness.WireMessages})
|
||||
ps = append(ps, providerCase{name: "bedrock", catalogID: "bedrock_api", upstream: "https://bedrock-runtime." + region + ".amazonaws.com", apiKey: k, model: "us.anthropic.claude-haiku-4-5", kind: harness.WireBedrock})
|
||||
}
|
||||
return ps
|
||||
}
|
||||
@@ -224,9 +224,12 @@ func TestProvidersMatrix(t *testing.T) {
|
||||
var c int
|
||||
var b string
|
||||
var cerr error
|
||||
if pc.kind == harness.WireVertex {
|
||||
switch pc.kind {
|
||||
case harness.WireVertex:
|
||||
c, b, cerr = cl.Vertex(ctx, settings.Endpoint, proxyIP, pc.project, pc.region, pc.model, "Reply with exactly: pong", sessionID)
|
||||
} else {
|
||||
case harness.WireBedrock:
|
||||
c, b, cerr = cl.Bedrock(ctx, settings.Endpoint, proxyIP, pc.model, "Reply with exactly: pong", sessionID)
|
||||
default:
|
||||
c, b, cerr = cl.Chat(ctx, settings.Endpoint, proxyIP, pc.kind, pc.model, "Reply with exactly: pong", sessionID)
|
||||
}
|
||||
if cerr == nil {
|
||||
|
||||
168
e2e/agentnetwork/guardrail_test.go
Normal file
168
e2e/agentnetwork/guardrail_test.go
Normal file
@@ -0,0 +1,168 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// catalogModel returns the normalized catalog id the proxy stamps for a
|
||||
// path-routed provider's configured model — the form the guardrail allowlist is
|
||||
// compared against (region prefix / @version stripped).
|
||||
func catalogModel(pc providerCase) string {
|
||||
switch pc.kind {
|
||||
case harness.WireBedrock:
|
||||
return strings.TrimPrefix(pc.model, "us.")
|
||||
case harness.WireVertex:
|
||||
return strings.SplitN(pc.model, "@", 2)[0]
|
||||
default:
|
||||
return pc.model
|
||||
}
|
||||
}
|
||||
|
||||
// disallowedModel returns a valid-shaped model id for the provider that is NOT
|
||||
// the configured/allowed one, so the guardrail must reject it before the
|
||||
// request ever reaches the upstream.
|
||||
func disallowedModel(pc providerCase) string {
|
||||
switch pc.kind {
|
||||
case harness.WireBedrock:
|
||||
return "us.anthropic.claude-opus-4-8"
|
||||
case harness.WireVertex:
|
||||
return "claude-opus-4-8@20250101"
|
||||
default:
|
||||
return "unlisted-model"
|
||||
}
|
||||
}
|
||||
|
||||
// sendModel drives one request for the given model through the provider's native
|
||||
// wire shape and returns the HTTP status.
|
||||
func sendModel(ctx context.Context, t *testing.T, cl *harness.Client, endpoint, proxyIP string, pc providerCase, model string) int {
|
||||
t.Helper()
|
||||
var code int
|
||||
var err error
|
||||
switch pc.kind {
|
||||
case harness.WireBedrock:
|
||||
code, _, err = cl.Bedrock(ctx, endpoint, proxyIP, model, "Reply with exactly: pong", "")
|
||||
case harness.WireVertex:
|
||||
code, _, err = cl.Vertex(ctx, endpoint, proxyIP, pc.project, pc.region, model, "Reply with exactly: pong", "")
|
||||
default:
|
||||
code, _, err = cl.Chat(ctx, endpoint, proxyIP, pc.kind, model, "Reply with exactly: pong", "")
|
||||
}
|
||||
require.NoError(t, err, "request must reach the proxy for %s", pc.name)
|
||||
return code
|
||||
}
|
||||
|
||||
// TestModelAllowlistEnforced provisions a Model Allowlist guardrail limiting each
|
||||
// path-routed provider (Bedrock, Vertex) to its configured model, then drives
|
||||
// requests over the tunnel: the allowed model returns 200 while a model outside
|
||||
// the allowlist is denied 403 by the guardrail before it reaches the upstream.
|
||||
// This is the coverage missing for #6751 — the model for these providers travels
|
||||
// in the URL path, and the allowlist must be enforced there.
|
||||
func TestModelAllowlistEnforced(t *testing.T) {
|
||||
var providers []providerCase
|
||||
for _, pc := range availableProviders() {
|
||||
if pc.kind == harness.WireBedrock || pc.kind == harness.WireVertex {
|
||||
providers = append(providers, pc)
|
||||
}
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
t.Skip("no path-routed provider keys set (AWS_BEARER_TOKEN_BEDROCK / GOOGLE_VERTEX_*); source ~/.llm-keys")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-allowlist"})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-allowlist-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
|
||||
// Providers with their configured (allowed) models; the first bootstraps the cluster.
|
||||
ids := make([]string, 0, len(providers))
|
||||
allowed := make([]string, 0, len(providers))
|
||||
for i, pc := range providers {
|
||||
req := providerRequest(pc)
|
||||
if i == 0 {
|
||||
req.BootstrapCluster = ptr(harness.AgentNetworkCluster)
|
||||
}
|
||||
prov, perr := srv.CreateProvider(ctx, req)
|
||||
require.NoError(t, perr, "create provider %s", pc.name)
|
||||
id := prov.Id
|
||||
ids = append(ids, id)
|
||||
allowed = append(allowed, catalogModel(pc))
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), id) })
|
||||
}
|
||||
|
||||
// Guardrail allowlisting exactly the configured models.
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-allowlist"
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = allowed
|
||||
guard, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail")
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-allowlist",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: ids,
|
||||
GuardrailIds: &[]string{guard.Id},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings for endpoint")
|
||||
require.NotEmpty(t, settings.Endpoint, "agent-network endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-proxy-allowlist")
|
||||
require.NoError(t, err, "mint proxy token via CLI")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken)
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, sk.Key)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve agent-network endpoint to proxy IP")
|
||||
|
||||
for _, pc := range providers {
|
||||
pc := pc
|
||||
t.Run(pc.name, func(t *testing.T) {
|
||||
// The admin's allowlisted model is served end to end.
|
||||
assert.Equal(t, 200, sendModel(ctx, t, cl, settings.Endpoint, proxyIP, pc, pc.model),
|
||||
"allowlisted model must be permitted for %s", pc.name)
|
||||
// A model outside the allowlist is rejected by the guardrail (before
|
||||
// the upstream), regardless of whether it is a real catalog model.
|
||||
assert.Equal(t, 403, sendModel(ctx, t, cl, settings.Endpoint, proxyIP, pc, disallowedModel(pc)),
|
||||
"model outside the allowlist must be denied for %s", pc.name)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -107,6 +107,17 @@ func (c *Combined) DeletePolicy(ctx context.Context, id string) error {
|
||||
return anDelete(ctx, c, "/api/agent-network/policies/"+id)
|
||||
}
|
||||
|
||||
// CreateGuardrail creates an agent-network guardrail (e.g. a model allowlist)
|
||||
// that can then be attached to a policy via its GuardrailIds.
|
||||
func (c *Combined) CreateGuardrail(ctx context.Context, req api.AgentNetworkGuardrailRequest) (api.AgentNetworkGuardrail, error) {
|
||||
return anRequest[api.AgentNetworkGuardrail](ctx, c, http.MethodPost, "/api/agent-network/guardrails", req)
|
||||
}
|
||||
|
||||
// DeleteGuardrail removes a guardrail by id.
|
||||
func (c *Combined) DeleteGuardrail(ctx context.Context, id string) error {
|
||||
return anDelete(ctx, c, "/api/agent-network/guardrails/"+id)
|
||||
}
|
||||
|
||||
// GetSettings returns the account's agent-network settings row. It exists only
|
||||
// after the first provider create bootstraps it.
|
||||
func (c *Combined) GetSettings(ctx context.Context) (api.AgentNetworkSettings, error) {
|
||||
|
||||
@@ -194,6 +194,11 @@ const (
|
||||
// WireVertex is the Anthropic-on-Vertex rawPredict shape: the client posts
|
||||
// the full Vertex model path and the proxy mints the SA OAuth token.
|
||||
WireVertex = "vertex"
|
||||
// WireBedrock is the native AWS Bedrock InvokeModel shape: the model id
|
||||
// travels in the URL path (/model/{id}/invoke), not the body, so the proxy
|
||||
// routes by path. This is what a Bedrock SDK client sends and the shape the
|
||||
// model-allowlist guardrail must enforce.
|
||||
WireBedrock = "bedrock"
|
||||
)
|
||||
|
||||
// Chat issues a chat-completion POST to the agent-network endpoint over the
|
||||
@@ -226,6 +231,17 @@ func (cl *Client) Vertex(ctx context.Context, endpoint, proxyIP, project, region
|
||||
return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(nil, sessionID))
|
||||
}
|
||||
|
||||
// Bedrock issues a native AWS Bedrock InvokeModel POST over the tunnel. The
|
||||
// model id is carried in the request path (/model/{id}/invoke), so the proxy
|
||||
// routes by path; the body uses the bedrock anthropic_version rather than a
|
||||
// model field. A non-empty sessionID is sent as the universal x-session-id
|
||||
// header the proxy records.
|
||||
func (cl *Client) Bedrock(ctx context.Context, endpoint, proxyIP, model, prompt, sessionID string) (int, string, error) {
|
||||
path := "/model/" + model + "/invoke"
|
||||
body := fmt.Sprintf(`{"anthropic_version":"bedrock-2023-05-31","max_tokens":64,"messages":[{"role":"user","content":%q}]}`, prompt)
|
||||
return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(nil, sessionID))
|
||||
}
|
||||
|
||||
// withSessionID appends the x-session-id header when sessionID is non-empty.
|
||||
func withSessionID(headers []string, sessionID string) []string {
|
||||
if sessionID == "" {
|
||||
|
||||
1
go.mod
1
go.mod
@@ -103,6 +103,7 @@ require (
|
||||
github.com/rs/xid v1.3.0
|
||||
github.com/shirou/gopsutil/v4 v4.25.8
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
|
||||
github.com/soheilhy/cmux v0.1.5
|
||||
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/testcontainers/testcontainers-go v0.37.0
|
||||
|
||||
3
go.sum
3
go.sum
@@ -603,6 +603,8 @@ github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
|
||||
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
|
||||
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
|
||||
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
|
||||
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
@@ -757,6 +759,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
|
||||
@@ -226,30 +226,6 @@ func (m *managerImpl) CreateProxyPeer(ctx context.Context, accountID string, pee
|
||||
return nil
|
||||
}
|
||||
|
||||
// Dedupe stale embedded peer records for the same (account, cluster).
|
||||
// The proxy generates a fresh WireGuard keypair on every startup
|
||||
// (proxy/internal/roundtrip/netbird.go), so without this sweep the
|
||||
// prior embedded peer would linger forever — holding its CGNAT IP
|
||||
// allocation, polluting other peers' rosters, and (most visibly)
|
||||
// leaving the synth DNS pointing at the dead address. The
|
||||
// (account, cluster) tuple identifies "the embedded peer for this
|
||||
// proxy instance at this cluster"; any record matching that tuple
|
||||
// with a different pubkey is by definition stale and must go.
|
||||
staleIDs, err := m.findStaleEmbeddedProxyPeers(ctx, accountID, cluster, peerKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scan for stale embedded proxy peers: %w", err)
|
||||
}
|
||||
if len(staleIDs) > 0 {
|
||||
// userID="" + checkConnected=false: the deletion is initiated
|
||||
// by management itself on behalf of the freshly-registering
|
||||
// proxy, not by an end user; the stale peer may still be
|
||||
// marked Connected from its prior session, but its session is
|
||||
// dead by definition (its key no longer exists).
|
||||
if err := m.DeletePeers(ctx, accountID, staleIDs, "", false); err != nil {
|
||||
return fmt.Errorf("delete stale embedded proxy peers %v: %w", staleIDs, err)
|
||||
}
|
||||
}
|
||||
|
||||
name := fmt.Sprintf("proxy-%s", xid.New().String())
|
||||
newPeer := &peer.Peer{
|
||||
Ephemeral: true,
|
||||
@@ -275,29 +251,3 @@ func (m *managerImpl) CreateProxyPeer(ctx context.Context, accountID string, pee
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// findStaleEmbeddedProxyPeers returns the peer IDs of embedded proxy peer
|
||||
// records in accountID that target the same cluster but carry a different
|
||||
// WireGuard pubkey than the freshly-registering one. Used by CreateProxyPeer
|
||||
// to garbage-collect stale records left behind when the proxy restarts with a
|
||||
// regenerated keypair.
|
||||
func (m *managerImpl) findStaleEmbeddedProxyPeers(ctx context.Context, accountID, cluster, newKey string) ([]string, error) {
|
||||
account, err := m.store.GetAccount(ctx, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var stale []string
|
||||
for _, p := range account.Peers {
|
||||
if p == nil || !p.ProxyMeta.Embedded {
|
||||
continue
|
||||
}
|
||||
if p.ProxyMeta.Cluster != cluster {
|
||||
continue
|
||||
}
|
||||
if p.Key == newKey {
|
||||
continue
|
||||
}
|
||||
stale = append(stale, p.ID)
|
||||
}
|
||||
return stale, nil
|
||||
}
|
||||
|
||||
@@ -24,13 +24,13 @@ import (
|
||||
|
||||
"github.com/netbirdio/netbird/encryption"
|
||||
"github.com/netbirdio/netbird/formatter/hook"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
|
||||
accesslogsmanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs/manager"
|
||||
rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
|
||||
nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
|
||||
"github.com/netbirdio/netbird/management/server/activity"
|
||||
activitystore "github.com/netbirdio/netbird/management/server/activity/store"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
nbcache "github.com/netbirdio/netbird/management/server/cache"
|
||||
nbContext "github.com/netbirdio/netbird/management/server/context"
|
||||
nbhttp "github.com/netbirdio/netbird/management/server/http"
|
||||
@@ -184,7 +184,12 @@ func (s *BaseServer) GRPCServer() *grpc.Server {
|
||||
grpc.ChainStreamInterceptor(realip.StreamServerInterceptorOpts(realipOpts...), streamInterceptor, proxyStream),
|
||||
}
|
||||
|
||||
if s.Config.HttpConfig.LetsEncryptDomain != "" {
|
||||
// With the native transport enabled, TLS is terminated at the listeners
|
||||
// (cmux-split shared listener and legacy port), so transport credentials
|
||||
// must not be set or the server would attempt a second handshake.
|
||||
if nativeGRPCEnabled() { //nolint:gocritic
|
||||
log.Info("native gRPC transport enabled, TLS is terminated at the listeners")
|
||||
} else if s.Config.HttpConfig.LetsEncryptDomain != "" {
|
||||
certManager, err := encryption.CreateCertManager(s.Config.Datadir, s.Config.HttpConfig.LetsEncryptDomain)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to create certificate service: %v", err)
|
||||
|
||||
@@ -6,12 +6,16 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/soheilhy/cmux"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"golang.org/x/crypto/acme/autocert"
|
||||
"golang.org/x/net/http2"
|
||||
@@ -36,8 +40,18 @@ const (
|
||||
DefaultSelfHostedDomain = "netbird.selfhosted"
|
||||
|
||||
ContainerKeyBaseServer = "baseServer"
|
||||
|
||||
// NativeGRPCEnvVar enables serving gRPC on the native gRPC transport,
|
||||
// multiplexed with HTTP on the shared listener, instead of through the
|
||||
// net/http ServeHTTP path which costs two extra goroutines per stream.
|
||||
NativeGRPCEnvVar = "NB_MGMT_NATIVE_GRPC"
|
||||
)
|
||||
|
||||
func nativeGRPCEnabled() bool {
|
||||
enabled, _ := strconv.ParseBool(os.Getenv(NativeGRPCEnvVar))
|
||||
return enabled
|
||||
}
|
||||
|
||||
type Server interface {
|
||||
Start(ctx context.Context) error
|
||||
Stop() error
|
||||
@@ -182,11 +196,22 @@ func (s *BaseServer) Start(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// With the native transport enabled the gRPC server carries no transport
|
||||
// credentials, so TLS must be terminated at each of its listeners.
|
||||
var grpcTLSConfig *tls.Config
|
||||
if nativeGRPCEnabled() {
|
||||
if s.certManager != nil {
|
||||
grpcTLSConfig = s.certManager.TLSConfig()
|
||||
} else {
|
||||
grpcTLSConfig = tlsConfig
|
||||
}
|
||||
}
|
||||
|
||||
var compatListener net.Listener
|
||||
if s.mgmtPort != ManagementLegacyPort && !s.disableLegacyManagementPort {
|
||||
// The Management gRPC server was running on port 33073 previously. Old agents that are already connected to it
|
||||
// are using port 33073. For compatibility purposes we keep running a 2nd gRPC server on port 33073.
|
||||
compatListener, err = s.serveGRPC(srvCtx, s.GRPCServer(), ManagementLegacyPort)
|
||||
compatListener, err = s.serveGRPC(srvCtx, s.GRPCServer(), ManagementLegacyPort, grpcTLSConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -196,22 +221,38 @@ func (s *BaseServer) Start(ctx context.Context) error {
|
||||
rootHandler := s.handlerFunc(srvCtx, s.GRPCServer(), s.APIHandler(), s.IDPHandler(), s.Metrics().GetMeter())
|
||||
switch {
|
||||
case s.certManager != nil:
|
||||
// a call to certManager.Listener() always creates a new listener so we do it once
|
||||
cml := s.certManager.Listener()
|
||||
if s.mgmtPort == 443 {
|
||||
// CertManager, HTTP and gRPC API all on the same port
|
||||
rootHandler = s.certManager.HTTPHandler(rootHandler)
|
||||
s.listener = cml
|
||||
if nativeGRPCEnabled() {
|
||||
var tcpListener net.Listener
|
||||
tcpListener, err = net.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed creating TCP listener on port %d: %v", s.mgmtPort, err)
|
||||
}
|
||||
s.listener = tls.NewListener(tcpListener, preferHTTP1ForDualProtoClients(s.certManager.TLSConfig()))
|
||||
} else {
|
||||
s.listener = s.certManager.Listener()
|
||||
}
|
||||
} else {
|
||||
s.listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort), s.certManager.TLSConfig())
|
||||
mgmtTLSConfig := s.certManager.TLSConfig()
|
||||
if nativeGRPCEnabled() {
|
||||
mgmtTLSConfig = preferHTTP1ForDualProtoClients(mgmtTLSConfig)
|
||||
}
|
||||
s.listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort), mgmtTLSConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed creating TLS listener on port %d: %v", s.mgmtPort, err)
|
||||
}
|
||||
cml := s.certManager.Listener()
|
||||
log.WithContext(ctx).Infof("running HTTP server (LetsEncrypt challenge handler): %s", cml.Addr().String())
|
||||
s.serveHTTP(ctx, cml, s.certManager.HTTPHandler(nil))
|
||||
}
|
||||
case tlsConfig != nil:
|
||||
s.listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort), tlsConfig)
|
||||
mgmtTLSConfig := tlsConfig
|
||||
if nativeGRPCEnabled() {
|
||||
mgmtTLSConfig = preferHTTP1ForDualProtoClients(mgmtTLSConfig)
|
||||
}
|
||||
s.listener, err = tls.Listen("tcp", fmt.Sprintf(":%d", s.mgmtPort), mgmtTLSConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed creating TLS listener on port %d: %v", s.mgmtPort, err)
|
||||
}
|
||||
@@ -224,7 +265,12 @@ func (s *BaseServer) Start(ctx context.Context) error {
|
||||
|
||||
log.WithContext(ctx).Infof("management server version %s", version.NetbirdVersion())
|
||||
log.WithContext(ctx).Infof("running HTTP server and gRPC server on the same port: %s", s.listener.Addr().String())
|
||||
s.serveGRPCWithHTTP(ctx, s.listener, rootHandler, tlsEnabled)
|
||||
if nativeGRPCEnabled() {
|
||||
log.WithContext(ctx).Infof("serving gRPC on the native transport (multiplexed with HTTP)")
|
||||
s.serveMultiplexed(ctx, s.listener, s.GRPCServer(), rootHandler, tlsEnabled)
|
||||
} else {
|
||||
s.serveGRPCWithHTTP(ctx, s.listener, rootHandler, tlsEnabled)
|
||||
}
|
||||
|
||||
s.update = version.NewUpdateAndStart("nb/management")
|
||||
s.update.SetDaemonVersion(version.NetbirdVersion())
|
||||
@@ -331,11 +377,14 @@ func (s *BaseServer) handlerFunc(_ context.Context, gRPCHandler *grpc.Server, ht
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BaseServer) serveGRPC(ctx context.Context, grpcServer *grpc.Server, port int) (net.Listener, error) {
|
||||
func (s *BaseServer) serveGRPC(ctx context.Context, grpcServer *grpc.Server, port int, tlsConf *tls.Config) (net.Listener, error) {
|
||||
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tlsConf != nil {
|
||||
listener = tls.NewListener(listener, tlsConf)
|
||||
}
|
||||
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
@@ -399,6 +448,69 @@ func (s *BaseServer) serveGRPCWithHTTP(ctx context.Context, listener net.Listene
|
||||
}()
|
||||
}
|
||||
|
||||
// preferHTTP1ForDualProtoClients steers TLS clients that offer both "h2" and
|
||||
// "http/1.1" in ALPN (browsers, REST clients) to HTTP/1.1. gRPC clients offer
|
||||
// only "h2", so with this steering every HTTP/2 connection on the shared
|
||||
// listener carries gRPC and can be routed to the native transport without
|
||||
// inspecting frames. ACME "acme-tls/1" and single-protocol clients keep the
|
||||
// base configuration.
|
||||
func preferHTTP1ForDualProtoClients(base *tls.Config) *tls.Config {
|
||||
h1Config := base.Clone()
|
||||
h1Config.NextProtos = []string{"http/1.1"}
|
||||
steered := base.Clone()
|
||||
steered.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
|
||||
if slices.Contains(hello.SupportedProtos, "http/1.1") && slices.Contains(hello.SupportedProtos, "h2") {
|
||||
return h1Config, nil
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
return steered
|
||||
}
|
||||
|
||||
// serveMultiplexed splits the shared listener by protocol: HTTP/2 connections
|
||||
// go to the native gRPC transport (see preferHTTP1ForDualProtoClients for why
|
||||
// they are all gRPC), everything else is served by net/http.
|
||||
//
|
||||
// Content-type based classification cannot be used here: cmux's SendSettings
|
||||
// matchers greet non-matching HTTP/2 connections and corrupt them for any
|
||||
// subsequent handler, while read-only matchers deadlock grpc-go clients,
|
||||
// which do not send HEADERS until they receive the server SETTINGS frame.
|
||||
func (s *BaseServer) serveMultiplexed(ctx context.Context, listener net.Listener, grpcServer *grpc.Server, handler http.Handler, tlsEnabled bool) {
|
||||
mux := cmux.New(listener)
|
||||
grpcListener := mux.Match(cmux.HTTP2())
|
||||
httpListener := mux.Match(cmux.Any())
|
||||
|
||||
httpHandler := handler
|
||||
if !tlsEnabled {
|
||||
//nolint:staticcheck // h2c also handles the HTTP/1 Upgrade mechanism, which http.Server's UnencryptedHTTP2 does not
|
||||
httpHandler = h2c.NewHandler(handler, &http2.Server{})
|
||||
}
|
||||
|
||||
s.wg.Add(3)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
s.reportServeError(ctx, grpcServer.Serve(grpcListener))
|
||||
}()
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
s.reportServeError(ctx, http.Serve(httpListener, httpHandler))
|
||||
}()
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
s.reportServeError(ctx, mux.Serve())
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *BaseServer) reportServeError(ctx context.Context, err error) {
|
||||
if ctx.Err() != nil || err == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.errCh <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveDomains determines dnsDomain and mgmtSingleAccModeDomain based on store state.
|
||||
// Fresh installs use the default self-hosted domain, while existing installs reuse the
|
||||
// persisted account domain to keep addressing stable across config changes.
|
||||
|
||||
109
management/internals/server/server_multiplex_test.go
Normal file
109
management/internals/server/server_multiplex_test.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/health"
|
||||
healthpb "google.golang.org/grpc/health/grpc_health_v1"
|
||||
)
|
||||
|
||||
func newSelfSignedCert(t *testing.T) tls.Certificate {
|
||||
t.Helper()
|
||||
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
|
||||
template := x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "127.0.0.1"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
|
||||
require.NoError(t, err)
|
||||
|
||||
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
|
||||
}
|
||||
|
||||
func TestServeMultiplexedRoutesProtocols(t *testing.T) {
|
||||
tcpListener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = tcpListener.Close() })
|
||||
|
||||
baseTLSConfig := &tls.Config{
|
||||
Certificates: []tls.Certificate{newSelfSignedCert(t)},
|
||||
NextProtos: []string{"h2", "http/1.1"},
|
||||
}
|
||||
tlsListener := tls.NewListener(tcpListener, preferHTTP1ForDualProtoClients(baseTLSConfig))
|
||||
|
||||
grpcServer := grpc.NewServer()
|
||||
healthpb.RegisterHealthServer(grpcServer, health.NewServer())
|
||||
t.Cleanup(grpcServer.Stop)
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintf(w, "proto=%d", r.ProtoMajor)
|
||||
})
|
||||
|
||||
s := &BaseServer{errCh: make(chan error, 4)}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
s.serveMultiplexed(ctx, tlsListener, grpcServer, handler, true)
|
||||
|
||||
addr := tcpListener.Addr().String()
|
||||
url := "https://" + addr + "/"
|
||||
|
||||
grpcConn, err := grpc.NewClient(addr,
|
||||
grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = grpcConn.Close() })
|
||||
|
||||
checkCtx, checkCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer checkCancel()
|
||||
resp, err := healthpb.NewHealthClient(grpcConn).Check(checkCtx, &healthpb.HealthCheckRequest{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, healthpb.HealthCheckResponse_SERVING, resp.Status)
|
||||
|
||||
get := func(client *http.Client) string {
|
||||
t.Helper()
|
||||
res, err := client.Get(url)
|
||||
require.NoError(t, err)
|
||||
body, err := io.ReadAll(res.Body)
|
||||
require.NoError(t, err)
|
||||
_ = res.Body.Close()
|
||||
return string(body)
|
||||
}
|
||||
|
||||
dualProtoClient := &http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
ForceAttemptHTTP2: true,
|
||||
},
|
||||
}
|
||||
require.Equal(t, "proto=1", get(dualProtoClient), "dual-ALPN client should be steered to HTTP/1.1")
|
||||
|
||||
h1OnlyClient := &http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true, NextProtos: []string{"http/1.1"}},
|
||||
},
|
||||
}
|
||||
require.Equal(t, "proto=1", get(h1OnlyClient))
|
||||
}
|
||||
@@ -164,7 +164,6 @@ func ToSyncResponse(ctx context.Context, config *nbconfig.Config, httpConfig *nb
|
||||
// filtered at the source (network map builder).
|
||||
includeIPv6 := peer.SupportsIPv6() && peer.IPv6.IsValid()
|
||||
useSourcePrefixes := peer.SupportsSourcePrefixes()
|
||||
localIsProxy := peer.ProxyMeta.Embedded
|
||||
|
||||
response := &proto.SyncResponse{
|
||||
PeerConfig: toPeerConfig(peer, networkMap.Network, dnsName, settings, httpConfig, deviceFlowConfig, networkMap.EnableSSH),
|
||||
@@ -184,7 +183,7 @@ func ToSyncResponse(ctx context.Context, config *nbconfig.Config, httpConfig *nb
|
||||
response.NetworkMap.PeerConfig = response.PeerConfig
|
||||
|
||||
remotePeers := make([]*proto.RemotePeerConfig, 0, len(networkMap.Peers)+len(networkMap.OfflinePeers))
|
||||
remotePeers = appendRemotePeerConfig(remotePeers, networkMap.Peers, dnsName, includeIPv6, localIsProxy)
|
||||
remotePeers = appendRemotePeerConfig(remotePeers, networkMap.Peers, dnsName, includeIPv6)
|
||||
|
||||
if !shouldSkipSendingDeprecatedRemotePeers(peer.Meta.WtVersion) {
|
||||
response.RemotePeers = remotePeers
|
||||
@@ -194,7 +193,7 @@ func ToSyncResponse(ctx context.Context, config *nbconfig.Config, httpConfig *nb
|
||||
response.RemotePeersIsEmpty = len(remotePeers) == 0
|
||||
response.NetworkMap.RemotePeersIsEmpty = response.RemotePeersIsEmpty
|
||||
|
||||
response.NetworkMap.OfflinePeers = appendRemotePeerConfig(nil, networkMap.OfflinePeers, dnsName, includeIPv6, localIsProxy)
|
||||
response.NetworkMap.OfflinePeers = appendRemotePeerConfig(nil, networkMap.OfflinePeers, dnsName, includeIPv6)
|
||||
|
||||
firewallRules := toProtocolFirewallRules(networkMap.FirewallRules, includeIPv6, useSourcePrefixes)
|
||||
response.NetworkMap.FirewallRules = firewallRules
|
||||
@@ -293,7 +292,7 @@ func shouldSkipSendingDeprecatedRemotePeers(peerVersion string) bool {
|
||||
return precomputedDeprecatedRemotePeersConstraint.Check(peerNBVersion)
|
||||
}
|
||||
|
||||
func appendRemotePeerConfig(dst []*proto.RemotePeerConfig, peers []*nbpeer.Peer, dnsName string, includeIPv6 bool, localIsProxy bool) []*proto.RemotePeerConfig {
|
||||
func appendRemotePeerConfig(dst []*proto.RemotePeerConfig, peers []*nbpeer.Peer, dnsName string, includeIPv6 bool) []*proto.RemotePeerConfig {
|
||||
for _, rPeer := range peers {
|
||||
allowedIPs := []string{rPeer.IP.String() + "/32"}
|
||||
if includeIPv6 && rPeer.IPv6.IsValid() {
|
||||
@@ -305,24 +304,11 @@ func appendRemotePeerConfig(dst []*proto.RemotePeerConfig, peers []*nbpeer.Peer,
|
||||
SshConfig: &proto.SSHConfig{SshPubKey: []byte(rPeer.SSHKey)},
|
||||
Fqdn: rPeer.FQDN(dnsName),
|
||||
AgentVersion: rPeer.Meta.WtVersion,
|
||||
LazyState: lazyStateFor(localIsProxy, rPeer),
|
||||
})
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// lazyStateFor returns the per-peer lazy override for a remote peer. Connections
|
||||
// involving an ephemeral proxy peer on either endpoint default to lazy so shared
|
||||
// proxy infrastructure is not kept permanently connected to every peer. All
|
||||
// other peers follow the account-wide flag. A future admin-facing per-peer
|
||||
// setting can return LazyStateEager here to force a peer always-active.
|
||||
func lazyStateFor(localIsProxy bool, rPeer *nbpeer.Peer) proto.LazyState {
|
||||
if localIsProxy || rPeer.ProxyMeta.Embedded {
|
||||
return proto.LazyState_LazyStateLazy
|
||||
}
|
||||
return proto.LazyState_LazyStateDefault
|
||||
}
|
||||
|
||||
// toProtocolDNSConfig converts nbdns.Config to proto.DNSConfig using the cache
|
||||
func toProtocolDNSConfig(update nbdns.Config, cache *cache.DNSConfigCache, forwardPort int64) *proto.DNSConfig {
|
||||
protoUpdate := &proto.DNSConfig{
|
||||
|
||||
@@ -215,12 +215,13 @@ func (s *Server) Job(srv proto.ManagementService_JobServer) error {
|
||||
return status.Errorf(codes.Unauthenticated, "peer is not registered")
|
||||
}
|
||||
|
||||
s.startResponseReceiver(ctx, srv)
|
||||
|
||||
updates := s.jobManager.CreateJobChannel(ctx, accountID, peer.ID)
|
||||
stream := s.jobManager.RegisterStream(ctx, accountID, peer.ID, func(event *job.Event) error {
|
||||
return s.sendJob(ctx, peerKey, event, srv)
|
||||
})
|
||||
defer s.jobManager.UnregisterStream(ctx, accountID, peer.ID, stream)
|
||||
log.WithContext(ctx).Debugf("Job: took %v", time.Since(reqStart))
|
||||
|
||||
return s.sendJobsLoop(ctx, accountID, peerKey, peer, updates, srv)
|
||||
return s.receiveJobResponses(ctx, peerKey, srv)
|
||||
}
|
||||
|
||||
// Sync validates the existence of a connecting peer, sends an initial state (all available for the connecting peers) and
|
||||
@@ -362,51 +363,26 @@ func (s *Server) handleHandshake(ctx context.Context, srv proto.ManagementServic
|
||||
return peerKey, nil
|
||||
}
|
||||
|
||||
func (s *Server) startResponseReceiver(ctx context.Context, srv proto.ManagementService_JobServer) {
|
||||
go func() {
|
||||
for {
|
||||
msg, err := srv.Recv()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
|
||||
return
|
||||
}
|
||||
log.WithContext(ctx).Warnf("recv job response error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
jobResp := &proto.JobResponse{}
|
||||
if _, err := s.parseRequest(ctx, msg, jobResp); err != nil {
|
||||
log.WithContext(ctx).Warnf("invalid job response: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.jobManager.HandleResponse(ctx, jobResp, msg.WgPubKey); err != nil {
|
||||
log.WithContext(ctx).Errorf("handle job response failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Server) sendJobsLoop(ctx context.Context, accountID string, peerKey wgtypes.Key, peer *nbpeer.Peer, updates *job.Channel, srv proto.ManagementService_JobServer) error {
|
||||
// todo figure out better error handling strategy
|
||||
defer s.jobManager.CloseChannel(ctx, accountID, peer.ID)
|
||||
|
||||
func (s *Server) receiveJobResponses(ctx context.Context, peerKey wgtypes.Key, srv proto.ManagementService_JobServer) error {
|
||||
for {
|
||||
event, err := updates.Event(ctx)
|
||||
msg, err := srv.Recv()
|
||||
if err != nil {
|
||||
if errors.Is(err, job.ErrJobChannelClosed) {
|
||||
log.WithContext(ctx).Debugf("jobs channel for peer %s was closed", peerKey.String())
|
||||
return nil
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || ctx.Err() != nil {
|
||||
log.WithContext(ctx).Debugf("job stream of peer %s has been closed", peerKey.String())
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
|
||||
// happens when connection drops, e.g. client disconnects
|
||||
log.WithContext(ctx).Debugf("stream of peer %s has been closed", peerKey.String())
|
||||
return ctx.Err()
|
||||
log.WithContext(ctx).Warnf("recv job response error: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.sendJob(ctx, peerKey, event, srv); err != nil {
|
||||
log.WithContext(ctx).Warnf("send job failed: %v", err)
|
||||
return nil
|
||||
jobResp := &proto.JobResponse{}
|
||||
if _, err := s.parseRequest(ctx, msg, jobResp); err != nil {
|
||||
log.WithContext(ctx).Warnf("invalid job response: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.jobManager.HandleResponse(ctx, jobResp, msg.WgPubKey); err != nil {
|
||||
log.WithContext(ctx).Errorf("handle job response failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,8 +43,9 @@ type TimeBasedAuthSecretsManager struct {
|
||||
updateManager network_map.PeersUpdateManager
|
||||
settingsManager settings.Manager
|
||||
groupsManager groups.Manager
|
||||
turnCancelMap map[string]chan struct{}
|
||||
relayCancelMap map[string]chan struct{}
|
||||
scheduler *refreshScheduler
|
||||
turnJobs map[string]*refreshJob
|
||||
relayJobs map[string]*refreshJob
|
||||
wgKey wgtypes.Key
|
||||
}
|
||||
|
||||
@@ -60,8 +61,6 @@ func NewTimeBasedAuthSecretsManager(updateManager network_map.PeersUpdateManager
|
||||
updateManager: updateManager,
|
||||
turnCfg: turnCfg,
|
||||
relayCfg: relayCfg,
|
||||
turnCancelMap: make(map[string]chan struct{}),
|
||||
relayCancelMap: make(map[string]chan struct{}),
|
||||
settingsManager: settingsManager,
|
||||
groupsManager: groupsManager,
|
||||
wgKey: key,
|
||||
@@ -127,16 +126,16 @@ func (m *TimeBasedAuthSecretsManager) GenerateRelayToken() (*Token, error) {
|
||||
}
|
||||
|
||||
func (m *TimeBasedAuthSecretsManager) cancelTURN(peerID string) {
|
||||
if channel, ok := m.turnCancelMap[peerID]; ok {
|
||||
close(channel)
|
||||
delete(m.turnCancelMap, peerID)
|
||||
if job, ok := m.turnJobs[peerID]; ok {
|
||||
m.scheduler.cancel(job)
|
||||
delete(m.turnJobs, peerID)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *TimeBasedAuthSecretsManager) cancelRelay(peerID string) {
|
||||
if channel, ok := m.relayCancelMap[peerID]; ok {
|
||||
close(channel)
|
||||
delete(m.relayCancelMap, peerID)
|
||||
if job, ok := m.relayJobs[peerID]; ok {
|
||||
m.scheduler.cancel(job)
|
||||
delete(m.relayJobs, peerID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,6 +147,31 @@ func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) {
|
||||
m.cancelRelay(peerID)
|
||||
}
|
||||
|
||||
func (m *TimeBasedAuthSecretsManager) ensureScheduler() {
|
||||
if m.scheduler == nil {
|
||||
m.scheduler = newRefreshScheduler(m.runRefreshJob)
|
||||
m.turnJobs = make(map[string]*refreshJob)
|
||||
m.relayJobs = make(map[string]*refreshJob)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *TimeBasedAuthSecretsManager) runRefreshJob(job *refreshJob) {
|
||||
switch job.kind {
|
||||
case refreshKindTURN:
|
||||
m.pushNewTURNAndRelayTokens(job.ctx, job.accountID, job.peerID)
|
||||
case refreshKindRelay:
|
||||
m.pushNewRelayTokens(job.ctx, job.accountID, job.peerID)
|
||||
}
|
||||
}
|
||||
|
||||
func refreshInterval(ttl time.Duration) time.Duration {
|
||||
interval := ttl / 4 * 3
|
||||
if interval <= 0 {
|
||||
interval = defaultDuration / 4 * 3
|
||||
}
|
||||
return interval
|
||||
}
|
||||
|
||||
// SetupRefresh starts peer credentials refresh
|
||||
func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, accountID, peerID string) {
|
||||
m.mux.Lock()
|
||||
@@ -157,54 +181,38 @@ func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, accountI
|
||||
m.cancelRelay(peerID)
|
||||
|
||||
if m.turnCfg != nil && m.turnCfg.TimeBasedCredentials {
|
||||
turnCancel := make(chan struct{}, 1)
|
||||
m.turnCancelMap[peerID] = turnCancel
|
||||
go m.refreshTURNTokens(ctx, accountID, peerID, turnCancel)
|
||||
m.ensureScheduler()
|
||||
job := &refreshJob{
|
||||
ctx: ctx,
|
||||
accountID: accountID,
|
||||
peerID: peerID,
|
||||
kind: refreshKindTURN,
|
||||
interval: refreshInterval(m.turnCfg.CredentialsTTL.Duration),
|
||||
}
|
||||
m.turnJobs[peerID] = job
|
||||
m.scheduler.schedule(job)
|
||||
log.WithContext(ctx).Debugf("starting TURN refresh for %s", peerID)
|
||||
} else {
|
||||
log.WithContext(ctx).Debugf("no TURN configuration, skipping TURN refresh for %s", peerID)
|
||||
}
|
||||
|
||||
if m.relayCfg != nil {
|
||||
relayCancel := make(chan struct{}, 1)
|
||||
m.relayCancelMap[peerID] = relayCancel
|
||||
go m.refreshRelayTokens(ctx, accountID, peerID, relayCancel)
|
||||
m.ensureScheduler()
|
||||
job := &refreshJob{
|
||||
ctx: ctx,
|
||||
accountID: accountID,
|
||||
peerID: peerID,
|
||||
kind: refreshKindRelay,
|
||||
interval: refreshInterval(m.relayCfg.CredentialsTTL.Duration),
|
||||
}
|
||||
m.relayJobs[peerID] = job
|
||||
m.scheduler.schedule(job)
|
||||
log.WithContext(ctx).Tracef("starting relay refresh for %s", peerID)
|
||||
} else {
|
||||
log.WithContext(ctx).Tracef("no relay configuration, skipping relay refresh for %s", peerID)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *TimeBasedAuthSecretsManager) refreshTURNTokens(ctx context.Context, accountID, peerID string, cancel chan struct{}) {
|
||||
ticker := time.NewTicker(m.turnCfg.CredentialsTTL.Duration / 4 * 3)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-cancel:
|
||||
log.WithContext(ctx).Tracef("stopping TURN refresh for %s", peerID)
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.pushNewTURNAndRelayTokens(ctx, accountID, peerID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *TimeBasedAuthSecretsManager) refreshRelayTokens(ctx context.Context, accountID, peerID string, cancel chan struct{}) {
|
||||
ticker := time.NewTicker(m.relayCfg.CredentialsTTL.Duration / 4 * 3)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-cancel:
|
||||
log.WithContext(ctx).Tracef("stopping relay refresh for %s", peerID)
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.pushNewRelayTokens(ctx, accountID, peerID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *TimeBasedAuthSecretsManager) pushNewTURNAndRelayTokens(ctx context.Context, accountID, peerID string) {
|
||||
turnToken, err := m.turnHmacToken.GenerateToken(sha1.New)
|
||||
if err != nil {
|
||||
|
||||
@@ -112,12 +112,12 @@ func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) {
|
||||
|
||||
tested.SetupRefresh(ctx, "someAccountID", peer)
|
||||
|
||||
if _, ok := tested.turnCancelMap[peer]; !ok {
|
||||
t.Errorf("expecting peer to be present in the turn cancel map, got not present")
|
||||
if _, ok := tested.turnJobs[peer]; !ok {
|
||||
t.Errorf("expecting peer to be present in the turn jobs map, got not present")
|
||||
}
|
||||
|
||||
if _, ok := tested.relayCancelMap[peer]; !ok {
|
||||
t.Errorf("expecting peer to be present in the relay cancel map, got not present")
|
||||
if _, ok := tested.relayJobs[peer]; !ok {
|
||||
t.Errorf("expecting peer to be present in the relay jobs map, got not present")
|
||||
}
|
||||
|
||||
var updates []*network_map.UpdateMessage
|
||||
@@ -212,19 +212,26 @@ func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
tested.SetupRefresh(context.Background(), "someAccountID", peer)
|
||||
if _, ok := tested.turnCancelMap[peer]; !ok {
|
||||
t.Errorf("expecting peer to be present in turn cancel map, got not present")
|
||||
if _, ok := tested.turnJobs[peer]; !ok {
|
||||
t.Errorf("expecting peer to be present in turn jobs map, got not present")
|
||||
}
|
||||
if _, ok := tested.relayCancelMap[peer]; !ok {
|
||||
t.Errorf("expecting peer to be present in relay cancel map, got not present")
|
||||
if _, ok := tested.relayJobs[peer]; !ok {
|
||||
t.Errorf("expecting peer to be present in relay jobs map, got not present")
|
||||
}
|
||||
|
||||
tested.CancelRefresh(peer)
|
||||
if _, ok := tested.turnCancelMap[peer]; ok {
|
||||
t.Errorf("expecting peer to be not present in turn cancel map, got present")
|
||||
if _, ok := tested.turnJobs[peer]; ok {
|
||||
t.Errorf("expecting peer to be not present in turn jobs map, got present")
|
||||
}
|
||||
if _, ok := tested.relayCancelMap[peer]; ok {
|
||||
t.Errorf("expecting peer to be not present in relay cancel map, got present")
|
||||
if _, ok := tested.relayJobs[peer]; ok {
|
||||
t.Errorf("expecting peer to be not present in relay jobs map, got present")
|
||||
}
|
||||
|
||||
tested.scheduler.mu.Lock()
|
||||
heapLen := len(tested.scheduler.jobs)
|
||||
tested.scheduler.mu.Unlock()
|
||||
if heapLen != 0 {
|
||||
t.Errorf("expecting scheduler heap to be empty after cancel, got %d entries", heapLen)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
158
management/internals/shared/grpc/token_refresh_scheduler.go
Normal file
158
management/internals/shared/grpc/token_refresh_scheduler.go
Normal file
@@ -0,0 +1,158 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
refreshWorkerCount = 4
|
||||
refreshWorkQueueSize = 1024
|
||||
)
|
||||
|
||||
type refreshKind int
|
||||
|
||||
const (
|
||||
refreshKindTURN refreshKind = iota
|
||||
refreshKindRelay
|
||||
)
|
||||
|
||||
type refreshJob struct {
|
||||
ctx context.Context
|
||||
accountID string
|
||||
peerID string
|
||||
kind refreshKind
|
||||
interval time.Duration
|
||||
nextRun time.Time
|
||||
index int
|
||||
cancelled atomic.Bool
|
||||
}
|
||||
|
||||
type refreshJobHeap []*refreshJob
|
||||
|
||||
func (h refreshJobHeap) Len() int { return len(h) }
|
||||
func (h refreshJobHeap) Less(i, j int) bool { return h[i].nextRun.Before(h[j].nextRun) }
|
||||
|
||||
func (h refreshJobHeap) Swap(i, j int) {
|
||||
h[i], h[j] = h[j], h[i]
|
||||
h[i].index = i
|
||||
h[j].index = j
|
||||
}
|
||||
|
||||
func (h *refreshJobHeap) Push(x any) {
|
||||
job := x.(*refreshJob)
|
||||
job.index = len(*h)
|
||||
*h = append(*h, job)
|
||||
}
|
||||
|
||||
func (h *refreshJobHeap) Pop() any {
|
||||
old := *h
|
||||
n := len(old)
|
||||
job := old[n-1]
|
||||
old[n-1] = nil
|
||||
job.index = -1
|
||||
*h = old[:n-1]
|
||||
return job
|
||||
}
|
||||
|
||||
// refreshScheduler executes periodic credential refresh jobs for all peers
|
||||
// from one timer goroutine and a fixed worker pool, instead of two parked
|
||||
// goroutines per connected peer.
|
||||
type refreshScheduler struct {
|
||||
mu sync.Mutex
|
||||
jobs refreshJobHeap
|
||||
wake chan struct{}
|
||||
work chan *refreshJob
|
||||
run func(job *refreshJob)
|
||||
}
|
||||
|
||||
func newRefreshScheduler(run func(job *refreshJob)) *refreshScheduler {
|
||||
s := &refreshScheduler{
|
||||
wake: make(chan struct{}, 1),
|
||||
work: make(chan *refreshJob, refreshWorkQueueSize),
|
||||
run: run,
|
||||
}
|
||||
go s.loop()
|
||||
for range refreshWorkerCount {
|
||||
go s.worker()
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *refreshScheduler) schedule(job *refreshJob) {
|
||||
s.mu.Lock()
|
||||
job.nextRun = time.Now().Add(job.interval)
|
||||
heap.Push(&s.jobs, job)
|
||||
s.mu.Unlock()
|
||||
|
||||
select {
|
||||
case s.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *refreshScheduler) cancel(job *refreshJob) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
job.cancelled.Store(true)
|
||||
if job.index >= 0 {
|
||||
heap.Remove(&s.jobs, job.index)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *refreshScheduler) loop() {
|
||||
timer := time.NewTimer(time.Hour)
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
|
||||
for {
|
||||
s.mu.Lock()
|
||||
now := time.Now()
|
||||
var due []*refreshJob
|
||||
for len(s.jobs) > 0 && !s.jobs[0].nextRun.After(now) {
|
||||
job := s.jobs[0]
|
||||
job.nextRun = job.nextRun.Add(job.interval)
|
||||
if !job.nextRun.After(now) {
|
||||
job.nextRun = now.Add(job.interval)
|
||||
}
|
||||
heap.Fix(&s.jobs, 0)
|
||||
due = append(due, job)
|
||||
}
|
||||
wait := time.Duration(-1)
|
||||
if len(s.jobs) > 0 {
|
||||
wait = time.Until(s.jobs[0].nextRun)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
for _, job := range due {
|
||||
s.work <- job
|
||||
}
|
||||
|
||||
if wait < 0 {
|
||||
<-s.wake
|
||||
continue
|
||||
}
|
||||
|
||||
timer.Reset(wait)
|
||||
select {
|
||||
case <-s.wake:
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *refreshScheduler) worker() {
|
||||
for job := range s.work {
|
||||
if job.cancelled.Load() {
|
||||
continue
|
||||
}
|
||||
s.run(job)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRefreshInterval(t *testing.T) {
|
||||
defaultInterval := defaultDuration / 4 * 3
|
||||
|
||||
require.Equal(t, 9*time.Hour, refreshInterval(12*time.Hour))
|
||||
require.Equal(t, defaultInterval, refreshInterval(0))
|
||||
require.Equal(t, defaultInterval, refreshInterval(-time.Second))
|
||||
require.Equal(t, defaultInterval, refreshInterval(3*time.Nanosecond))
|
||||
require.Positive(t, refreshInterval(4*time.Nanosecond))
|
||||
}
|
||||
|
||||
func TestWorkerSkipsCancelledJob(t *testing.T) {
|
||||
var ran atomic.Int32
|
||||
scheduler := newRefreshScheduler(func(*refreshJob) {
|
||||
ran.Add(1)
|
||||
})
|
||||
|
||||
cancelledJob := &refreshJob{interval: time.Hour}
|
||||
cancelledJob.cancelled.Store(true)
|
||||
liveJob := &refreshJob{interval: time.Hour}
|
||||
|
||||
scheduler.work <- cancelledJob
|
||||
scheduler.work <- liveJob
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
return ran.Load() == 1
|
||||
}, 2*time.Second, 10*time.Millisecond, "live job should run exactly once, cancelled job never")
|
||||
}
|
||||
|
||||
func TestCancelBeforeFirePreventsRun(t *testing.T) {
|
||||
var ran atomic.Int32
|
||||
scheduler := newRefreshScheduler(func(*refreshJob) {
|
||||
ran.Add(1)
|
||||
})
|
||||
|
||||
job := &refreshJob{interval: 50 * time.Millisecond}
|
||||
scheduler.schedule(job)
|
||||
scheduler.cancel(job)
|
||||
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
require.Zero(t, ran.Load(), "cancelled job must never fire")
|
||||
|
||||
scheduler.mu.Lock()
|
||||
heapLen := len(scheduler.jobs)
|
||||
scheduler.mu.Unlock()
|
||||
require.Zero(t, heapLen)
|
||||
}
|
||||
@@ -1,19 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
// nolint:gosec
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/management/cmd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
go func() {
|
||||
log.Println(http.ListenAndServe("localhost:6060", nil))
|
||||
}()
|
||||
if pprofAddr := os.Getenv("NB_PPROF_ADDR"); pprofAddr != "" {
|
||||
log.Infof("pprof enabled, listening on: %s", pprofAddr)
|
||||
go func() {
|
||||
log.Println(http.ListenAndServe(pprofAddr, nil))
|
||||
}()
|
||||
}
|
||||
|
||||
if err := cmd.Execute(); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/peers"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
agenttypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
nbpeer "github.com/netbirdio/netbird/management/server/peer"
|
||||
"github.com/netbirdio/netbird/management/server/permissions"
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
"github.com/netbirdio/netbird/management/server/types"
|
||||
)
|
||||
|
||||
// TestAgentNetwork_ProxyRestart_PropagatesNewPeerAndDropsStale is the no-mock
|
||||
// regression guard for the bug the user reported: restarting the proxy creates
|
||||
// a fresh embedded peer with a NEW WireGuard public key (the proxy generates
|
||||
// the keypair on every startup at proxy/internal/roundtrip/netbird.go:312).
|
||||
// The PRIOR embedded peer record is never deleted on management, so the
|
||||
// account accumulates a stale peer holding a stale CGNAT IP. Other peers
|
||||
// in the account either keep routing to the dead IP, or — if synth DNS
|
||||
// picks the wrong record — never see the new IP at all.
|
||||
//
|
||||
// What this test exercises (no mocks):
|
||||
// - real SQLite test store
|
||||
// - real DefaultAccountManager, network-map controller, peer-update channels
|
||||
// - real peers.Manager.CreateProxyPeer path (the very method the proxy
|
||||
// invokes over gRPC on every startup)
|
||||
// - real agentnetwork.Manager + synth chain so the client receives a
|
||||
// concrete DNS record that must point at the LATEST proxy peer.
|
||||
//
|
||||
// Pre-fix expected behavior (red): two embedded peers exist after the
|
||||
// "restart"; the synth DNS record points at the stale one; the client
|
||||
// receives an update reflecting the new peer but the old one lingers.
|
||||
// Post-fix expected behavior (green): exactly one embedded peer exists
|
||||
// after restart (with the new key) AND the client's network map carries
|
||||
// the synth DNS pointing at that new peer's CGNAT IP.
|
||||
func TestAgentNetwork_ProxyRestart_PropagatesNewPeerAndDropsStale(t *testing.T) {
|
||||
am, updateManager, err := createManager(t)
|
||||
require.NoError(t, err, "createManager must succeed")
|
||||
ctx := context.Background()
|
||||
|
||||
const (
|
||||
accountID = "an-restart-acct"
|
||||
adminUserID = "an-restart-admin"
|
||||
groupAID = "an-restart-grp-A"
|
||||
clusterAddr = "eu.proxy.netbird.io"
|
||||
clientKey = "BhRPtynAAYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8="
|
||||
// Two different proxy pubkeys — the "before" and "after" of a
|
||||
// proxy-process restart with fresh-keypair generation.
|
||||
proxyKey1 = "Aaaaa1aaaaYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8="
|
||||
proxyKey2 = "Bbbbb2bbbbYRDy08+q4HTMsos8fs4plTP4NOSh7C1ry8="
|
||||
)
|
||||
|
||||
// --- Account scaffold ---
|
||||
account := newAccountWithId(ctx, accountID, adminUserID, "an-restart.test", "", "", false)
|
||||
require.NoError(t, am.Store.SaveAccount(ctx, account))
|
||||
|
||||
clientPeer := &nbpeer.Peer{
|
||||
Key: clientKey,
|
||||
Name: "an-restart-client",
|
||||
DNSLabel: "an-restart-client",
|
||||
Meta: nbpeer.PeerSystemMeta{Hostname: "an-restart-client", GoOS: "linux", WtVersion: "development"},
|
||||
}
|
||||
addedClient, _, _, _, err := am.AddPeer(ctx, "", "", adminUserID, clientPeer, false)
|
||||
require.NoError(t, err, "AddPeer for client must succeed")
|
||||
require.NoError(t, am.MarkPeerConnected(ctx, clientKey, accountID, time.Now().UnixNano(), &types.NetworkMap{}),
|
||||
"MarkPeerConnected for the client peer must succeed (affected-peer fan-out skips disconnected peers)")
|
||||
|
||||
// Place the client in group A so the synth policy reaches it.
|
||||
account, err = am.Store.GetAccount(ctx, accountID)
|
||||
require.NoError(t, err)
|
||||
account.Groups[groupAID] = &types.Group{ID: groupAID, Name: "groupA", Peers: []string{addedClient.ID}}
|
||||
require.NoError(t, am.Store.SaveAccount(ctx, account), "SaveAccount must persist group A")
|
||||
|
||||
// --- Real peers + agent-network managers ---
|
||||
permMgr := permissions.NewManager(am.Store)
|
||||
peersMgr := peers.NewManager(am.Store, permMgr)
|
||||
peersMgr.SetAccountManager(am)
|
||||
peersMgr.SetNetworkMapController(am.networkMapController)
|
||||
agentMgr := agentnetwork.NewManager(am.Store, permMgr, am, nil)
|
||||
|
||||
// Subscribe BEFORE any state-mutating call so we don't lose the update
|
||||
// that contains the synth DNS record.
|
||||
clientCh := updateManager.CreateChannel(ctx, addedClient.ID)
|
||||
t.Cleanup(func() { updateManager.CloseChannel(ctx, addedClient.ID) })
|
||||
drain(clientCh)
|
||||
|
||||
// --- First proxy startup: register peer key K1, then mark it
|
||||
// connected. In production the proxy follows CreateProxyPeer with the
|
||||
// regular sync stream which lands on MarkPeerConnected; the synth DNS
|
||||
// path filters out peers that aren't Connected (types/account.go:323),
|
||||
// so without this step no DNS record would be emitted.
|
||||
require.NoError(t, peersMgr.CreateProxyPeer(ctx, accountID, proxyKey1, clusterAddr),
|
||||
"first CreateProxyPeer (proxy startup) must succeed")
|
||||
|
||||
peer1ID, err := am.Store.GetPeerIDByKey(ctx, store.LockingStrengthNone, proxyKey1)
|
||||
require.NoError(t, err, "proxy peer for K1 must be persisted after CreateProxyPeer")
|
||||
require.NotEmpty(t, peer1ID)
|
||||
|
||||
require.NoError(t, am.MarkPeerConnected(ctx, proxyKey1, accountID, time.Now().UnixNano(), &types.NetworkMap{}),
|
||||
"MarkPeerConnected for K1 must succeed")
|
||||
|
||||
account, err = am.Store.GetAccount(ctx, accountID)
|
||||
require.NoError(t, err)
|
||||
proxyIP1 := account.Peers[peer1ID].IP.String()
|
||||
require.NotEmpty(t, proxyIP1, "K1 must have an assigned overlay IP")
|
||||
|
||||
// --- Provider + policy. CreateProvider / CreatePolicy trigger the
|
||||
// agentnetwork reconcile which runs UpdateAccountPeers; the resulting
|
||||
// NetworkMap delivered to the client carries the synth DNS record
|
||||
// pointing at K1's IP. ---
|
||||
provider, err := agentMgr.CreateProvider(ctx, adminUserID, &agenttypes.Provider{
|
||||
AccountID: accountID,
|
||||
ProviderID: "openai_api",
|
||||
Name: "openai-test",
|
||||
UpstreamURL: "https://api.openai.com",
|
||||
APIKey: "sk-test-key",
|
||||
Enabled: true,
|
||||
Models: []agenttypes.ProviderModel{{ID: "gpt-5.4"}},
|
||||
}, clusterAddr)
|
||||
require.NoError(t, err, "CreateProvider must succeed")
|
||||
|
||||
_, err = agentMgr.CreatePolicy(ctx, adminUserID, &agenttypes.Policy{
|
||||
AccountID: accountID,
|
||||
Name: "p1",
|
||||
Enabled: true,
|
||||
SourceGroups: []string{groupAID},
|
||||
DestinationProviderIDs: []string{provider.ID},
|
||||
})
|
||||
require.NoError(t, err, "CreatePolicy must succeed")
|
||||
|
||||
settings, err := am.Store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID)
|
||||
require.NoError(t, err)
|
||||
fqdn := settings.Endpoint()
|
||||
|
||||
rdata1 := awaitZoneRData(clientCh, clusterAddr, fqdn, true)
|
||||
require.Equal(t, proxyIP1, rdata1,
|
||||
"client must receive a synth DNS record pointing at K1's overlay IP after the synth path runs")
|
||||
drain(clientCh)
|
||||
|
||||
// --- Proxy restart: NEW keypair K2, same account, same cluster ---
|
||||
require.NoError(t, peersMgr.CreateProxyPeer(ctx, accountID, proxyKey2, clusterAddr),
|
||||
"second CreateProxyPeer (proxy restart with fresh keypair) must succeed")
|
||||
|
||||
peer2ID, err := am.Store.GetPeerIDByKey(ctx, store.LockingStrengthNone, proxyKey2)
|
||||
require.NoError(t, err, "proxy peer for K2 must be persisted after restart")
|
||||
require.NotEmpty(t, peer2ID)
|
||||
|
||||
require.NoError(t, am.MarkPeerConnected(ctx, proxyKey2, accountID, time.Now().UnixNano(), &types.NetworkMap{}),
|
||||
"MarkPeerConnected for K2 must succeed")
|
||||
|
||||
// In production the agent's sync stream pulls a fresh NetworkMap as
|
||||
// part of its normal reconcile cadence; in this isolated test
|
||||
// MarkPeerConnected's affected-peer fan-out can race the channel-side
|
||||
// buffer in a way that swallows the synth-DNS-bearing update before
|
||||
// our await reads it. Trigger an explicit account-wide fan-out so the
|
||||
// assertion below tests what production actually delivers, not the
|
||||
// in-test buffer race.
|
||||
am.UpdateAccountPeers(ctx, accountID, types.UpdateReason{Resource: types.UpdateResourcePeer, Operation: types.UpdateOperationUpdate})
|
||||
|
||||
account, err = am.Store.GetAccount(ctx, accountID)
|
||||
require.NoError(t, err)
|
||||
proxyIP2 := account.Peers[peer2ID].IP.String()
|
||||
require.NotEmpty(t, proxyIP2, "K2 must have an assigned overlay IP")
|
||||
require.NotEqual(t, proxyIP1, proxyIP2, "K2 must get a different overlay IP than K1 (sanity)")
|
||||
|
||||
// CRITICAL ASSERTION 1: K1 must no longer be in the store. The SqlStore
|
||||
// returns ("", nil) for a missing key rather than NotFound, so assert
|
||||
// on the returned ID being empty.
|
||||
staleID, err := am.Store.GetPeerIDByKey(ctx, store.LockingStrengthNone, proxyKey1)
|
||||
require.NoError(t, err, "GetPeerIDByKey for a missing peer must not error")
|
||||
assert.Empty(t, staleID,
|
||||
"stale embedded proxy peer K1 must be removed when a new embedded peer registers for the same (account, cluster); pre-fix this assertion fails because management never cleans up the prior peer record")
|
||||
|
||||
// CRITICAL ASSERTION 2: exactly one embedded proxy peer remains, and it
|
||||
// is K2.
|
||||
account, err = am.Store.GetAccount(ctx, accountID)
|
||||
require.NoError(t, err)
|
||||
embeddedKeys := []string{}
|
||||
for _, p := range account.Peers {
|
||||
if p.ProxyMeta.Embedded {
|
||||
embeddedKeys = append(embeddedKeys, p.Key)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, []string{proxyKey2}, embeddedKeys,
|
||||
"after a proxy restart exactly one embedded proxy peer should remain — the one with the new key K2")
|
||||
|
||||
// CRITICAL ASSERTION 3: the synth DNS record the client receives now
|
||||
// points at K2's IP, not K1's.
|
||||
rdata2 := awaitZoneRData(clientCh, clusterAddr, fqdn, true)
|
||||
assert.Equal(t, proxyIP2, rdata2,
|
||||
"after proxy restart, the client's synth DNS record must point at the NEW embedded peer's IP, not the stale K1 IP")
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
@@ -21,11 +20,17 @@ type Event struct {
|
||||
Response *proto.JobResponse
|
||||
}
|
||||
|
||||
// PeerStream is the send side of a peer's Job stream. Sends are serialized by
|
||||
// its mutex; the receive side runs on the stream's gRPC handler goroutine.
|
||||
type PeerStream struct {
|
||||
send func(*Event) error
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
mu *sync.RWMutex
|
||||
jobChannels map[string]*Channel // per-peer job streams
|
||||
pending map[string]*Event // jobID → event
|
||||
responseWait time.Duration
|
||||
streams map[string]*PeerStream // per-peer job streams
|
||||
pending map[string]*Event // jobID → event
|
||||
metrics telemetry.AppMetrics
|
||||
Store store.Store
|
||||
peersManager peers.Manager
|
||||
@@ -34,9 +39,8 @@ type Manager struct {
|
||||
func NewJobManager(metrics telemetry.AppMetrics, store store.Store, peersManager peers.Manager) *Manager {
|
||||
|
||||
return &Manager{
|
||||
jobChannels: make(map[string]*Channel),
|
||||
streams: make(map[string]*PeerStream),
|
||||
pending: make(map[string]*Event),
|
||||
responseWait: 5 * time.Minute,
|
||||
metrics: metrics,
|
||||
mu: &sync.RWMutex{},
|
||||
Store: store,
|
||||
@@ -44,8 +48,9 @@ func NewJobManager(metrics telemetry.AppMetrics, store store.Store, peersManager
|
||||
}
|
||||
}
|
||||
|
||||
// CreateJobChannel creates or replaces a channel for a peer
|
||||
func (jm *Manager) CreateJobChannel(ctx context.Context, accountID, peerID string) *Channel {
|
||||
// RegisterStream registers the send side of a peer's Job stream, replacing any
|
||||
// previous registration for the peer.
|
||||
func (jm *Manager) RegisterStream(ctx context.Context, accountID, peerID string, send func(*Event) error) *PeerStream {
|
||||
// all pending jobs stored in db for this peer should be failed
|
||||
if err := jm.Store.MarkAllPendingJobsAsFailed(ctx, accountID, peerID, "Pending job cleanup: marked as failed automatically due to being stuck too long"); err != nil {
|
||||
log.WithContext(ctx).Error(err.Error())
|
||||
@@ -54,23 +59,41 @@ func (jm *Manager) CreateJobChannel(ctx context.Context, accountID, peerID strin
|
||||
jm.mu.Lock()
|
||||
defer jm.mu.Unlock()
|
||||
|
||||
if ch, ok := jm.jobChannels[peerID]; ok {
|
||||
ch.Close()
|
||||
delete(jm.jobChannels, peerID)
|
||||
}
|
||||
stream := &PeerStream{send: send}
|
||||
jm.streams[peerID] = stream
|
||||
return stream
|
||||
}
|
||||
|
||||
ch := NewChannel()
|
||||
jm.jobChannels[peerID] = ch
|
||||
return ch
|
||||
// UnregisterStream removes a peer's stream registration and fails its pending
|
||||
// jobs. It is a no-op if the registration was already replaced by a newer
|
||||
// stream of the same peer.
|
||||
func (jm *Manager) UnregisterStream(ctx context.Context, accountID, peerID string, stream *PeerStream) {
|
||||
jm.mu.Lock()
|
||||
defer jm.mu.Unlock()
|
||||
|
||||
if jm.streams[peerID] != stream {
|
||||
return
|
||||
}
|
||||
delete(jm.streams, peerID)
|
||||
|
||||
for jobID, ev := range jm.pending {
|
||||
if ev.PeerID == peerID {
|
||||
// if the client disconnect and there is pending job then mark it as failed
|
||||
if err := jm.Store.MarkPendingJobsAsFailed(ctx, accountID, peerID, jobID, "Time out peer disconnected"); err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to mark pending jobs as failed: %v", err)
|
||||
}
|
||||
delete(jm.pending, jobID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SendJob sends a job to a peer and tracks it as pending
|
||||
func (jm *Manager) SendJob(ctx context.Context, accountID, peerID string, req *proto.JobRequest) error {
|
||||
jm.mu.RLock()
|
||||
ch, ok := jm.jobChannels[peerID]
|
||||
stream, ok := jm.streams[peerID]
|
||||
jm.mu.RUnlock()
|
||||
if !ok {
|
||||
return fmt.Errorf("peer %s has no channel", peerID)
|
||||
return fmt.Errorf("peer %s has no stream", peerID)
|
||||
}
|
||||
|
||||
event := &Event{
|
||||
@@ -82,7 +105,10 @@ func (jm *Manager) SendJob(ctx context.Context, accountID, peerID string, req *p
|
||||
jm.pending[string(req.ID)] = event
|
||||
jm.mu.Unlock()
|
||||
|
||||
if err := ch.AddEvent(ctx, jm.responseWait, event); err != nil {
|
||||
stream.mu.Lock()
|
||||
err := stream.send(event)
|
||||
stream.mu.Unlock()
|
||||
if err != nil {
|
||||
jm.cleanup(ctx, accountID, string(req.ID), err.Error())
|
||||
return err
|
||||
}
|
||||
@@ -127,27 +153,6 @@ func (jm *Manager) HandleResponse(ctx context.Context, resp *proto.JobResponse,
|
||||
return nil
|
||||
}
|
||||
|
||||
// CloseChannel closes a peer’s channel and cleans up its jobs
|
||||
func (jm *Manager) CloseChannel(ctx context.Context, accountID, peerID string) {
|
||||
jm.mu.Lock()
|
||||
defer jm.mu.Unlock()
|
||||
|
||||
if ch, ok := jm.jobChannels[peerID]; ok {
|
||||
ch.Close()
|
||||
delete(jm.jobChannels, peerID)
|
||||
}
|
||||
|
||||
for jobID, ev := range jm.pending {
|
||||
if ev.PeerID == peerID {
|
||||
// if the client disconnect and there is pending job then mark it as failed
|
||||
if err := jm.Store.MarkPendingJobsAsFailed(ctx, accountID, peerID, jobID, "Time out peer disconnected"); err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to mark pending jobs as failed: %v", err)
|
||||
}
|
||||
delete(jm.pending, jobID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup removes a pending job safely
|
||||
func (jm *Manager) cleanup(ctx context.Context, accountID, jobID string, reason string) {
|
||||
jm.mu.Lock()
|
||||
@@ -165,7 +170,7 @@ func (jm *Manager) IsPeerConnected(peerID string) bool {
|
||||
jm.mu.RLock()
|
||||
defer jm.mu.RUnlock()
|
||||
|
||||
_, ok := jm.jobChannels[peerID]
|
||||
_, ok := jm.streams[peerID]
|
||||
return ok
|
||||
}
|
||||
|
||||
|
||||
90
management/server/job/manager_test.go
Normal file
90
management/server/job/manager_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
func newTestManager(t *testing.T) (*Manager, *store.MockStore) {
|
||||
t.Helper()
|
||||
ctrl := gomock.NewController(t)
|
||||
t.Cleanup(ctrl.Finish)
|
||||
mockStore := store.NewMockStore(ctrl)
|
||||
return NewJobManager(nil, mockStore, nil), mockStore
|
||||
}
|
||||
|
||||
func TestSendJobDeliversThroughRegisteredStream(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
manager, mockStore := newTestManager(t)
|
||||
mockStore.EXPECT().MarkAllPendingJobsAsFailed(gomock.Any(), "acc", "peer1", gomock.Any()).Return(nil)
|
||||
|
||||
var sent []*Event
|
||||
manager.RegisterStream(ctx, "acc", "peer1", func(event *Event) error {
|
||||
sent = append(sent, event)
|
||||
return nil
|
||||
})
|
||||
require.True(t, manager.IsPeerConnected("peer1"))
|
||||
|
||||
err := manager.SendJob(ctx, "acc", "peer1", &proto.JobRequest{ID: []byte("job1")})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, sent, 1)
|
||||
require.Equal(t, "peer1", sent[0].PeerID)
|
||||
require.True(t, manager.IsPeerHasPendingJobs("peer1"))
|
||||
}
|
||||
|
||||
func TestSendJobWithoutStream(t *testing.T) {
|
||||
manager, _ := newTestManager(t)
|
||||
err := manager.SendJob(context.Background(), "acc", "peer1", &proto.JobRequest{ID: []byte("job1")})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSendJobFailureCleansPending(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
manager, mockStore := newTestManager(t)
|
||||
mockStore.EXPECT().MarkAllPendingJobsAsFailed(gomock.Any(), "acc", "peer1", gomock.Any()).Return(nil)
|
||||
mockStore.EXPECT().MarkPendingJobsAsFailed(gomock.Any(), "acc", "peer1", "job1", gomock.Any()).Return(nil)
|
||||
|
||||
manager.RegisterStream(ctx, "acc", "peer1", func(*Event) error {
|
||||
return errors.New("stream broken")
|
||||
})
|
||||
|
||||
err := manager.SendJob(ctx, "acc", "peer1", &proto.JobRequest{ID: []byte("job1")})
|
||||
require.Error(t, err)
|
||||
require.False(t, manager.IsPeerHasPendingJobs("peer1"))
|
||||
}
|
||||
|
||||
func TestUnregisterStreamIgnoresSupersededRegistration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
manager, mockStore := newTestManager(t)
|
||||
mockStore.EXPECT().MarkAllPendingJobsAsFailed(gomock.Any(), "acc", "peer1", gomock.Any()).Return(nil).Times(2)
|
||||
|
||||
first := manager.RegisterStream(ctx, "acc", "peer1", func(*Event) error { return nil })
|
||||
second := manager.RegisterStream(ctx, "acc", "peer1", func(*Event) error { return nil })
|
||||
|
||||
manager.UnregisterStream(ctx, "acc", "peer1", first)
|
||||
require.True(t, manager.IsPeerConnected("peer1"), "stale unregister must not remove the replacement stream")
|
||||
|
||||
manager.UnregisterStream(ctx, "acc", "peer1", second)
|
||||
require.False(t, manager.IsPeerConnected("peer1"))
|
||||
}
|
||||
|
||||
func TestUnregisterStreamFailsPendingJobs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
manager, mockStore := newTestManager(t)
|
||||
mockStore.EXPECT().MarkAllPendingJobsAsFailed(gomock.Any(), "acc", "peer1", gomock.Any()).Return(nil)
|
||||
mockStore.EXPECT().MarkPendingJobsAsFailed(gomock.Any(), "acc", "peer1", "job1", gomock.Any()).Return(nil)
|
||||
|
||||
stream := manager.RegisterStream(ctx, "acc", "peer1", func(*Event) error { return nil })
|
||||
require.NoError(t, manager.SendJob(ctx, "acc", "peer1", &proto.JobRequest{ID: []byte("job1")}))
|
||||
require.True(t, manager.IsPeerHasPendingJobs("peer1"))
|
||||
|
||||
manager.UnregisterStream(ctx, "acc", "peer1", stream)
|
||||
require.False(t, manager.IsPeerHasPendingJobs("peer1"))
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/netbirdio/netbird/client/ssh/auth"
|
||||
@@ -42,6 +43,14 @@ type NetworkMapComponents struct {
|
||||
PostureFailedPeers map[string]map[string]struct{}
|
||||
|
||||
RouterPeers map[string]*nbpeer.Peer
|
||||
|
||||
routesByPeerOnce sync.Once
|
||||
routesByPeerIdx map[string][]routeIndexEntry
|
||||
}
|
||||
|
||||
type routeIndexEntry struct {
|
||||
route *route.Route
|
||||
viaGroup bool
|
||||
}
|
||||
|
||||
type AccountSettingsInfo struct {
|
||||
@@ -530,33 +539,43 @@ func (c *NetworkMapComponents) getRoutingPeerRoutes(peerID string) (enabledRoute
|
||||
disabledRoutes = append(disabledRoutes, r)
|
||||
}
|
||||
|
||||
for _, r := range c.Routes {
|
||||
for _, groupID := range r.PeerGroups {
|
||||
group := c.GetGroupInfo(groupID)
|
||||
if group == nil {
|
||||
continue
|
||||
}
|
||||
for _, id := range group.Peers {
|
||||
if id != peerID {
|
||||
continue
|
||||
}
|
||||
|
||||
newPeerRoute := r.Copy()
|
||||
newPeerRoute.Peer = id
|
||||
newPeerRoute.PeerGroups = nil
|
||||
newPeerRoute.ID = route.ID(string(r.ID) + ":" + id)
|
||||
takeRoute(newPeerRoute)
|
||||
break
|
||||
}
|
||||
}
|
||||
if r.Peer == peerID {
|
||||
takeRoute(r.Copy())
|
||||
for _, entry := range c.routesByPeer()[peerID] {
|
||||
if entry.viaGroup {
|
||||
newPeerRoute := entry.route.Copy()
|
||||
newPeerRoute.PeerGroups = nil
|
||||
newPeerRoute.ID = route.ID(string(entry.route.ID) + ":" + peerID)
|
||||
takeRoute(newPeerRoute)
|
||||
continue
|
||||
}
|
||||
takeRoute(entry.route.Copy())
|
||||
}
|
||||
|
||||
return enabledRoutes, disabledRoutes
|
||||
}
|
||||
|
||||
func (c *NetworkMapComponents) routesByPeer() map[string][]routeIndexEntry {
|
||||
c.routesByPeerOnce.Do(func() {
|
||||
idx := make(map[string][]routeIndexEntry)
|
||||
for _, r := range c.Routes {
|
||||
for _, groupID := range r.PeerGroups {
|
||||
group := c.GetGroupInfo(groupID)
|
||||
if group == nil {
|
||||
continue
|
||||
}
|
||||
for _, id := range group.Peers {
|
||||
idx[id] = append(idx[id], routeIndexEntry{route: r, viaGroup: true})
|
||||
}
|
||||
}
|
||||
if r.Peer != "" {
|
||||
idx[r.Peer] = append(idx[r.Peer], routeIndexEntry{route: r})
|
||||
}
|
||||
}
|
||||
c.routesByPeerIdx = idx
|
||||
})
|
||||
|
||||
return c.routesByPeerIdx
|
||||
}
|
||||
|
||||
func (c *NetworkMapComponents) filterRoutesByGroups(routes []*route.Route, groupListMap LookupMap) []*route.Route {
|
||||
var filteredRoutes []*route.Route
|
||||
for _, r := range routes {
|
||||
|
||||
@@ -25,6 +25,14 @@ const (
|
||||
denyCodeModel = "llm_policy.model_blocked"
|
||||
denyReasonModel = "model_blocked"
|
||||
denyMessageModel = "model is not in the policy allowlist"
|
||||
// Deny reason used when an allowlist is configured but the request model
|
||||
// could not be determined. URL/path-routed providers (AWS Bedrock, Google
|
||||
// Vertex, ...) carry the model outside the JSON body, so a request shape the
|
||||
// parser does not recognise reaches the guardrail with no model. Such a
|
||||
// request must be denied (fail closed), never waved through.
|
||||
denyCodeModelUnknown = "llm_policy.model_unknown"
|
||||
denyReasonModelUnknown = "model_unknown"
|
||||
denyMessageModelUnknown = "request model could not be determined for the policy allowlist"
|
||||
)
|
||||
|
||||
// Middleware enforces the model allowlist and optionally captures the
|
||||
@@ -108,23 +116,37 @@ func (m *Middleware) evaluateAllowlist(model string, modelPresent bool) *middlew
|
||||
if len(m.cfg.ModelAllowlist) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !modelPresent {
|
||||
return nil
|
||||
// Fail closed: with an allowlist configured, a request whose model the
|
||||
// upstream parser could not extract (absent or empty) must be denied rather
|
||||
// than allowed. This is what enforces the allowlist for URL/path-routed
|
||||
// providers (Bedrock, Vertex, ...) whose model lives outside the JSON body.
|
||||
if !modelPresent || normaliseModel(model) == "" {
|
||||
return denyModel("", denyCodeModelUnknown, denyMessageModelUnknown, denyReasonModelUnknown)
|
||||
}
|
||||
if m.modelInAllowlist(model) {
|
||||
return nil
|
||||
}
|
||||
return denyModel(model, denyCodeModel, denyMessageModel, denyReasonModel)
|
||||
}
|
||||
|
||||
// denyModel builds a 403 deny Output for a model-allowlist rejection. model is
|
||||
// included in the details only when non-empty.
|
||||
func denyModel(model, code, message, reason string) *middleware.Output {
|
||||
details := map[string]string{}
|
||||
if model != "" {
|
||||
details["model"] = model
|
||||
}
|
||||
return &middleware.Output{
|
||||
Decision: middleware.DecisionDeny,
|
||||
DenyStatus: 403,
|
||||
DenyReason: &middleware.DenyReason{
|
||||
Code: denyCodeModel,
|
||||
Message: denyMessageModel,
|
||||
Details: map[string]string{"model": model},
|
||||
Code: code,
|
||||
Message: message,
|
||||
Details: details,
|
||||
},
|
||||
Metadata: []middleware.KV{
|
||||
{Key: middleware.KeyLLMPolicyDecision, Value: "deny"},
|
||||
{Key: middleware.KeyLLMPolicyReason, Value: denyReasonModel},
|
||||
{Key: middleware.KeyLLMPolicyReason, Value: reason},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,13 +102,44 @@ func TestAllowlistCaseInsensitive(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowlistMissingModelKeyAllows(t *testing.T) {
|
||||
func TestAllowlistMissingModelKeyDenies(t *testing.T) {
|
||||
// Fail closed: with an allowlist configured, a request whose model the
|
||||
// parser could not extract (URL/path-routed providers such as Bedrock or
|
||||
// Vertex whose shape wasn't recognised) must be denied, not allowed.
|
||||
mw := New(Config{ModelAllowlist: []string{"gpt-4o"}})
|
||||
out, err := mw.Invoke(context.Background(), newInput())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "missing model key must allow even with non-empty allowlist")
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "absent model must be denied when an allowlist is set")
|
||||
assert.Equal(t, 403, out.DenyStatus, "deny status must be 403")
|
||||
require.NotNil(t, out.DenyReason, "deny reason must be populated")
|
||||
assert.Equal(t, "llm_policy.model_unknown", out.DenyReason.Code, "deny code must be model_unknown")
|
||||
dec, _ := metaValue(t, out.Metadata, middleware.KeyLLMPolicyDecision)
|
||||
assert.Equal(t, "allow", dec, "decision must be allow when model key is absent")
|
||||
assert.Equal(t, "deny", dec, "decision must be deny when model key is absent")
|
||||
reason, _ := metaValue(t, out.Metadata, middleware.KeyLLMPolicyReason)
|
||||
assert.Equal(t, "model_unknown", reason, "reason metadata must be model_unknown")
|
||||
}
|
||||
|
||||
func TestAllowlistEmptyModelValueDenies(t *testing.T) {
|
||||
// A present-but-empty model is as undeterminable as an absent one.
|
||||
mw := New(Config{ModelAllowlist: []string{"gpt-4o"}})
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: " "},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "empty model must be denied when an allowlist is set")
|
||||
require.NotNil(t, out.DenyReason, "deny reason must be populated")
|
||||
assert.Equal(t, "llm_policy.model_unknown", out.DenyReason.Code, "deny code must be model_unknown")
|
||||
}
|
||||
|
||||
func TestAllowlistEmptyListAllowsMissingModel(t *testing.T) {
|
||||
// Without an allowlist there is nothing to enforce, so a missing model is
|
||||
// still allowed — the fail-closed rule only applies when a list is set.
|
||||
mw := New(Config{})
|
||||
out, err := mw.Invoke(context.Background(), newInput())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "no allowlist must allow even without a model")
|
||||
}
|
||||
|
||||
func TestPromptCaptureDisabledEmitsNoPrompt(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package llm_request_parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/proxy/internal/middleware"
|
||||
"github.com/netbirdio/netbird/proxy/internal/middleware/builtin/llm_guardrail"
|
||||
)
|
||||
|
||||
// runParserGuardrail runs the request parser then the model-allowlist guardrail
|
||||
// in SlotOnRequest order, threading the parser's metadata into the guardrail the
|
||||
// same way the real chain does. It returns the guardrail decision so tests can
|
||||
// assert allowlist enforcement for URL/path-routed providers end to end.
|
||||
func runParserGuardrail(t *testing.T, url string, body []byte, allowlist []string) *middleware.Output {
|
||||
t.Helper()
|
||||
parser := newMiddleware(t)
|
||||
parsed, err := parser.Invoke(context.Background(), &middleware.Input{
|
||||
Slot: middleware.SlotOnRequest,
|
||||
URL: url,
|
||||
Body: body,
|
||||
})
|
||||
require.NoError(t, err, "parser must not error")
|
||||
|
||||
guard := llm_guardrail.New(llm_guardrail.Config{ModelAllowlist: allowlist})
|
||||
out, err := guard.Invoke(context.Background(), &middleware.Input{
|
||||
Slot: middleware.SlotOnRequest,
|
||||
Metadata: parsed.Metadata,
|
||||
})
|
||||
require.NoError(t, err, "guardrail must not error")
|
||||
require.NotNil(t, out, "guardrail must return an output")
|
||||
return out
|
||||
}
|
||||
|
||||
// TestModelAllowlist_URLRoutedProviders validates that the model allowlist is
|
||||
// enforced for providers whose model travels in the URL path (AWS Bedrock,
|
||||
// Google Vertex) rather than the JSON body. The "unknown action" case is the
|
||||
// regression guard for #6751: a Bedrock request shape the parser cannot map to a
|
||||
// model must fail closed under an allowlist instead of bypassing it.
|
||||
func TestModelAllowlist_URLRoutedProviders(t *testing.T) {
|
||||
const bedrockBody = `{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"hi"}]}`
|
||||
const vertexBody = `{"anthropic_version":"vertex-2023-10-16","messages":[{"role":"user","content":"hi"}]}`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
body string
|
||||
allowlist []string
|
||||
decision middleware.Decision
|
||||
denyCode string
|
||||
}{
|
||||
{
|
||||
name: "bedrock allowed model passes",
|
||||
url: "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-haiku-4-5-v1:0/invoke",
|
||||
body: bedrockBody,
|
||||
allowlist: []string{"anthropic.claude-haiku-4-5"},
|
||||
decision: middleware.DecisionAllow,
|
||||
},
|
||||
{
|
||||
name: "bedrock disallowed model denied",
|
||||
url: "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-opus-4-8-v1:0/invoke",
|
||||
body: bedrockBody,
|
||||
allowlist: []string{"anthropic.claude-haiku-4-5"},
|
||||
decision: middleware.DecisionDeny,
|
||||
denyCode: "llm_policy.model_blocked",
|
||||
},
|
||||
{
|
||||
name: "bedrock unknown action fails closed",
|
||||
url: "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-opus-4-8-v1:0/some-future-action",
|
||||
body: bedrockBody,
|
||||
allowlist: []string{"anthropic.claude-haiku-4-5"},
|
||||
decision: middleware.DecisionDeny,
|
||||
denyCode: "llm_policy.model_unknown",
|
||||
},
|
||||
{
|
||||
name: "vertex disallowed model denied",
|
||||
url: "/v1/projects/p/locations/global/publishers/anthropic/models/claude-opus-4-8@20250101:rawPredict",
|
||||
body: vertexBody,
|
||||
allowlist: []string{"claude-haiku-4-5"},
|
||||
decision: middleware.DecisionDeny,
|
||||
denyCode: "llm_policy.model_blocked",
|
||||
},
|
||||
{
|
||||
name: "vertex allowed model passes",
|
||||
url: "/v1/projects/p/locations/global/publishers/anthropic/models/claude-haiku-4-5@20250101:rawPredict",
|
||||
body: vertexBody,
|
||||
allowlist: []string{"claude-haiku-4-5"},
|
||||
decision: middleware.DecisionAllow,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
out := runParserGuardrail(t, tt.url, []byte(tt.body), tt.allowlist)
|
||||
assert.Equal(t, tt.decision, out.Decision, "unexpected decision for %s", tt.name)
|
||||
if tt.decision == middleware.DecisionDeny {
|
||||
require.NotNil(t, out.DenyReason, "deny reason must be set for %s", tt.name)
|
||||
assert.Equal(t, 403, out.DenyStatus, "deny status must be 403 for %s", tt.name)
|
||||
assert.Equal(t, tt.denyCode, out.DenyReason.Code, "deny code for %s", tt.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -30,12 +30,6 @@ import (
|
||||
|
||||
const deviceNamePrefix = "ingress-proxy-"
|
||||
|
||||
// envProxyRosenpass toggles Rosenpass (permissive) on the embedded proxy client. Defaults to on.
|
||||
const envProxyRosenpass = "NB_PROXY_ROSENPASS" //nolint:gosec // env var name, not a credential
|
||||
|
||||
// envProxyClientLogLevel sets the embedded NetBird client's log level.
|
||||
const envProxyClientLogLevel = "NB_PROXY_CLIENT_LOG_LEVEL"
|
||||
|
||||
const clientStopTimeout = 30 * time.Second
|
||||
|
||||
const createProxyPeerTimeout = 30 * time.Second
|
||||
@@ -359,11 +353,11 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account
|
||||
// NB_PROXY_CLIENT_LOG_LEVEL (e.g. "trace") to surface the embedded NetBird
|
||||
// client's relay / signal / handshake detail for local debugging.
|
||||
clientLogLevel := log.WarnLevel.String()
|
||||
if v := strings.TrimSpace(os.Getenv(envProxyClientLogLevel)); v != "" {
|
||||
if v := strings.TrimSpace(os.Getenv("NB_PROXY_CLIENT_LOG_LEVEL")); v != "" {
|
||||
if lvl, err := log.ParseLevel(v); err == nil {
|
||||
clientLogLevel = lvl.String()
|
||||
} else {
|
||||
n.logger.Warnf("invalid %s %q, using %q: %v", envProxyClientLogLevel, v, clientLogLevel, err)
|
||||
n.logger.Warnf("invalid NB_PROXY_CLIENT_LOG_LEVEL %q, using %q: %v", v, clientLogLevel, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -373,26 +367,15 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account
|
||||
}
|
||||
})
|
||||
|
||||
// Rosenpass runs in permissive mode by default so the embedded proxy can
|
||||
// establish connections with Rosenpass-enabled peers (which otherwise fail
|
||||
// on a PSK mismatch) while still falling back to plain WireGuard for peers
|
||||
// that do not run Rosenpass. Set NB_PROXY_ROSENPASS=false to disable it.
|
||||
rosenpassEnabled := true
|
||||
if v, ok := envBool(envProxyRosenpass, n.logger); ok {
|
||||
rosenpassEnabled = v
|
||||
}
|
||||
|
||||
// Create embedded NetBird client with the generated private key.
|
||||
// The peer has already been created via CreateProxyPeer RPC with the public key.
|
||||
wgPort := int(n.clientCfg.WGPort)
|
||||
embedOpts := embed.Options{
|
||||
DeviceName: deviceNamePrefix + n.proxyID,
|
||||
ManagementURL: n.clientCfg.MgmtAddr,
|
||||
PrivateKey: privateKey.String(),
|
||||
LogLevel: clientLogLevel,
|
||||
BlockInbound: n.clientCfg.BlockInbound,
|
||||
EnableRosenpass: rosenpassEnabled,
|
||||
RosenpassPermissive: rosenpassEnabled,
|
||||
DeviceName: deviceNamePrefix + n.proxyID,
|
||||
ManagementURL: n.clientCfg.MgmtAddr,
|
||||
PrivateKey: privateKey.String(),
|
||||
LogLevel: clientLogLevel,
|
||||
BlockInbound: n.clientCfg.BlockInbound,
|
||||
// The embedded proxy peer must never be a stepping stone into
|
||||
// the proxy host's LAN: it only exists to reach NetBird mesh
|
||||
// targets or, when direct_upstream is set, the host network
|
||||
@@ -916,8 +899,6 @@ func logEmbedOptions(logger *log.Logger, accountID types.AccountID, serviceID ty
|
||||
"mtu": mtu,
|
||||
"block_inbound": opts.BlockInbound,
|
||||
"block_lan_access": opts.BlockLANAccess,
|
||||
"rosenpass_enabled": opts.EnableRosenpass,
|
||||
"rosenpass_permissive": opts.RosenpassPermissive,
|
||||
"disable_ipv6": opts.DisableIPv6,
|
||||
"disable_client_routes": opts.DisableClientRoutes,
|
||||
"no_userspace": opts.NoUserspace,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -486,22 +486,6 @@ message RemotePeerConfig {
|
||||
string fqdn = 4;
|
||||
|
||||
string agentVersion = 5;
|
||||
|
||||
// lazyState is the management per-peer override for lazy (on-demand)
|
||||
// connections to this remote peer. LazyStateDefault follows the account-wide
|
||||
// flag; LazyStateLazy forces lazy; LazyStateEager forces an always-active
|
||||
// connection. A local NB_LAZY_CONN/MDM override still wins over this.
|
||||
LazyState lazyState = 6;
|
||||
}
|
||||
|
||||
// LazyState is the management per-peer override for lazy connections.
|
||||
enum LazyState {
|
||||
// Follow the account-wide lazy connection flag.
|
||||
LazyStateDefault = 0;
|
||||
// Force a lazy (on-demand) connection regardless of the account flag.
|
||||
LazyStateLazy = 1;
|
||||
// Force an always-active connection regardless of the account flag.
|
||||
LazyStateEager = 2;
|
||||
}
|
||||
|
||||
// SSHConfig represents SSH configurations of a peer.
|
||||
|
||||
Reference in New Issue
Block a user