From 58114f98fb253ea3e4abf6eefe07d3d57b43e5d5 Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Fri, 11 Sep 2026 15:41:52 +0300 Subject: [PATCH 01/75] [management] Only trust forwarded-IP headers from configured trusted peers (#7454) --- .../getting-started-enterprise.sh | 5 +- infrastructure_files/getting-started.sh | 29 +++ infrastructure_files/management.json.tmpl | 4 +- management/internals/server/boot.go | 56 ++++-- management/internals/server/realip_test.go | 171 ++++++++++++++++++ 5 files changed, 240 insertions(+), 25 deletions(-) create mode 100644 management/internals/server/realip_test.go diff --git a/infrastructure_files/getting-started-enterprise.sh b/infrastructure_files/getting-started-enterprise.sh index 3f7cf6357..701598a60 100755 --- a/infrastructure_files/getting-started-enterprise.sh +++ b/infrastructure_files/getting-started-enterprise.sh @@ -808,8 +808,9 @@ server: # Trust X-Forwarded-* only from the Traefik container's static address. Both # keys must stay in step with the ipv4_address pinned in docker-compose.yml: - # trustedPeers decides whether forwarded headers are read at all, and leaving - # it unset falls back to 0.0.0.0/0. + # trustedPeers decides whether forwarded headers are read at all. Leaving it + # unset trusts nothing and records Traefik's own address as every peer's + # connection IP. reverseProxy: trustedPeers: - "${TRAEFIK_IP}/32" diff --git a/infrastructure_files/getting-started.sh b/infrastructure_files/getting-started.sh index 5efc0181e..afbc5c282 100755 --- a/infrastructure_files/getting-started.sh +++ b/infrastructure_files/getting-started.sh @@ -153,6 +153,7 @@ check_domain_resolves() { # NETBIRD_TRAEFIK_CERTRESOLVER external-Traefik cert resolver (type 1) # NETBIRD_BIND_LOCALHOST_ONLY true/false (default true, types 2-5) # NETBIRD_EXTERNAL_PROXY_NETWORK docker network to join (types 2-4) +# NETBIRD_TRUSTED_PEERS reverse proxy address management sees (default: built-in Traefik's IP, empty for types 1-5) # NETBIRD_NON_INTERACTIVE true forces unattended mode even with a TTY # tty_available succeeds only when we may prompt: never when the operator has @@ -459,6 +460,8 @@ initialize_default_values() { MANAGEMENT_HOST_PORT="8081" # Combined server port (management + signal + relay) BIND_LOCALHOST_ONLY="true" EXTERNAL_PROXY_NETWORK="" + TRUSTED_PEERS="" # Address the reverse proxy connects to management from + # Traefik static IP within the internal bridge network TRAEFIK_IP="172.30.0.10" @@ -519,6 +522,7 @@ apply_agent_network_preset() { REVERSE_PROXY_TYPE="0" ENABLE_PROXY="true" ENABLE_CROWDSEC="false" + TRUSTED_PEERS="${NETBIRD_TRUSTED_PEERS:-$TRAEFIK_IP/32}" TRAEFIK_ACME_EMAIL=$(resolve NETBIRD_LETSENCRYPT_EMAIL required read_traefik_acme_email) @@ -573,6 +577,21 @@ configure_reverse_proxy() { 4) EXTERNAL_PROXY_NETWORK=$(resolve NETBIRD_EXTERNAL_PROXY_NETWORK "" read_proxy_docker_network "Caddy") ;; *) ;; # No network prompt for other options esac + + # Only the bundled Traefik has an address we know at render time. External proxies + # must supply the address their proxy reaches management from. + if [[ "$REVERSE_PROXY_TYPE" == "0" ]]; then + TRUSTED_PEERS="${NETBIRD_TRUSTED_PEERS:-$TRAEFIK_IP/32}" + else + TRUSTED_PEERS="${NETBIRD_TRUSTED_PEERS:-}" + if [[ -z "$TRUSTED_PEERS" ]]; then + echo "" > /dev/stderr + echo "Note: reverseProxy.trustedPeers is unset, so NetBird will use the address your" > /dev/stderr + echo "proxy connects from as each peer's connection IP. To record real client IPs," > /dev/stderr + echo "set NETBIRD_TRUSTED_PEERS to your proxy's address (e.g. 172.20.0.5/32) and re-run." > /dev/stderr + echo "" > /dev/stderr + fi + fi return 0 } @@ -1033,6 +1052,7 @@ server: reverseProxy: trustedHTTPProxies: - "$TRAEFIK_IP/32" +$(render_trusted_peers) store: engine: "sqlite" @@ -1041,6 +1061,12 @@ EOF return 0 } +render_trusted_peers() { + if [[ -n "$TRUSTED_PEERS" ]]; then + printf ' trustedPeers:\n - "%s"' "$TRUSTED_PEERS" + fi +} + render_dashboard_env() { cat < 0 && trustedProxiesCount > 0 { - log.WithContext(context.Background()).Warn("TrustedHTTPProxies and TrustedHTTPProxiesCount both are configured. " + - "This is not recommended way to extract X-Forwarded-For. Consider using one of these options.") - } - realipOpts := []realip.Option{ - realip.WithTrustedPeers(trustedPeers), - realip.WithTrustedProxies(trustedHTTPProxies), - realip.WithTrustedProxiesCount(trustedProxiesCount), - realip.WithHeaders([]string{realip.XForwardedFor, realip.XRealIp}), - } + realipOpts := realIPOptions(s.Config.ReverseProxy) proxyUnary, proxyStream, proxyAuthClose := nbgrpc.NewProxyAuthInterceptors(s.Store()) s.proxyAuthClose = proxyAuthClose gRPCOpts := []grpc.ServerOption{ @@ -333,7 +318,7 @@ func (s *BaseServer) AccessLogsManager() accesslogs.Manager { }) } -func loadTLSConfig(certFile string, certKey string) (*tls.Config, error) { +func loadTLSConfig(certFile, certKey string) (*tls.Config, error) { // Load server's certificate and private key serverCert, err := tls.LoadX509KeyPair(certFile, certKey) if err != nil { @@ -380,3 +365,34 @@ func streamInterceptor( wrapped.WrappedContext = context.WithValue(ctx, nbContext.RequestIDKey, reqID) return handler(srv, wrapped) } + +// realIPOptions builds the real-IP middleware options from the reverse proxy config. +// +// TrustedPeers controls which transport peers are allowed to supply forwarded-IP +// headers. If empty, forwarded headers are ignored and the transport peer address +// is used directly. Operators terminating connections at a reverse proxy should +// configure TrustedPeers with that proxy's address or network. +// +// Only X-Forwarded-For is trusted. X-Real-IP contains a single client-supplied +// address with no proxy chain to validate, and none of the reverse proxies we ship +// use it on the gRPC path. +func realIPOptions(cfg nbconfig.ReverseProxy) []realip.Option { + if idx := slices.IndexFunc(cfg.TrustedPeers, func(p netip.Prefix) bool { return p.Bits() == 0 }); idx >= 0 { + log.WithContext(context.Background()).Warnf("TrustedPeers contains the default route %s, which trusts "+ + "X-Forwarded-For from every client and allows connection IP spoofing. Set TrustedPeers to the address "+ + "of your reverse proxy, or leave it empty to use the connection's source address.", cfg.TrustedPeers[idx]) + } + if cfg.TrustedHTTPProxiesCount > 0 { + log.WithContext(context.Background()).Warn( + "TrustedHTTPProxiesCount skips X-Forwarded-For entries by position before TrustedHTTPProxies filters by address. " + + "An incorrect count may skip the real client IP and produce an incorrect source address.", + ) + } + + return []realip.Option{ + realip.WithTrustedPeers(cfg.TrustedPeers), + realip.WithTrustedProxies(cfg.TrustedHTTPProxies), + realip.WithTrustedProxiesCount(cfg.TrustedHTTPProxiesCount), + realip.WithHeaders([]string{realip.XForwardedFor}), + } +} diff --git a/management/internals/server/realip_test.go b/management/internals/server/realip_test.go new file mode 100644 index 000000000..89ac02730 --- /dev/null +++ b/management/internals/server/realip_test.go @@ -0,0 +1,171 @@ +package server + +import ( + "context" + "io" + "net" + "net/netip" + "testing" + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/realip" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/types/known/emptypb" + + nbconfig "github.com/netbirdio/netbird/management/internals/server/config" +) + +const ( + realIPProbeMethod = "/netbird.test.RealIPProbe/Probe" + realIPProbeStreamMethod = "/netbird.test.RealIPProbe/ProbeStream" +) + +// realIPProbe records the real IP the middleware derived for each call. +type realIPProbe struct { + got chan string +} + +func (p *realIPProbe) record(ctx context.Context) { + addr, _ := realip.FromContext(ctx) + p.got <- addr.String() +} + +func (p *realIPProbe) wait(t *testing.T) string { + t.Helper() + + select { + case got := <-p.got: + return got + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for probe") + return "" + } +} + +func startProbeServer(t *testing.T, cfg nbconfig.ReverseProxy) (*grpc.ClientConn, *realIPProbe) { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + probe := &realIPProbe{got: make(chan string, 1)} + opts := realIPOptions(cfg) + srv := grpc.NewServer( + grpc.ChainUnaryInterceptor(realip.UnaryServerInterceptorOpts(opts...)), + grpc.ChainStreamInterceptor(realip.StreamServerInterceptorOpts(opts...)), + ) + srv.RegisterService(&grpc.ServiceDesc{ + ServiceName: "netbird.test.RealIPProbe", + HandlerType: (*any)(nil), + Methods: []grpc.MethodDesc{{ + MethodName: "Probe", + Handler: func(_ any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor) (any, error) { + req := new(emptypb.Empty) + if err := dec(req); err != nil { + return nil, err + } + handler := func(ctx context.Context, _ any) (any, error) { + probe.record(ctx) + return &emptypb.Empty{}, nil + } + if interceptor == nil { + return handler(ctx, req) + } + return interceptor(ctx, req, &grpc.UnaryServerInfo{FullMethod: realIPProbeMethod}, handler) + }, + }}, + Streams: []grpc.StreamDesc{{ + StreamName: "ProbeStream", + ServerStreams: true, + Handler: func(_ any, stream grpc.ServerStream) error { + probe.record(stream.Context()) + return nil + }, + }}, + }, probe) + + go func() { _ = srv.Serve(listener) }() + t.Cleanup(srv.Stop) + + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + return conn, probe +} + +func callUnary(t *testing.T, conn *grpc.ClientConn, probe *realIPProbe, kv ...string) string { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, kv...) + require.NoError(t, conn.Invoke(ctx, realIPProbeMethod, &emptypb.Empty{}, &emptypb.Empty{})) + + return probe.wait(t) +} + +func callStream(t *testing.T, conn *grpc.ClientConn, probe *realIPProbe, kv ...string) string { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = metadata.AppendToOutgoingContext(ctx, kv...) + desc := &grpc.StreamDesc{StreamName: "ProbeStream", ServerStreams: true} + stream, err := conn.NewStream(ctx, desc, realIPProbeStreamMethod) + require.NoError(t, err) + require.NoError(t, stream.CloseSend()) + require.ErrorIs(t, stream.RecvMsg(&emptypb.Empty{}), io.EOF) + + return probe.wait(t) +} + +func assertRealIP(t *testing.T, cfg nbconfig.ReverseProxy, want string, kv ...string) { + t.Helper() + + conn, probe := startProbeServer(t, cfg) + t.Run("unary", func(t *testing.T) { + assert.Equal(t, want, callUnary(t, conn, probe, kv...)) + }) + t.Run("stream", func(t *testing.T) { + assert.Equal(t, want, callStream(t, conn, probe, kv...)) + }) +} + +func TestRealIPDefaultIgnoresClientForwardedHeaders(t *testing.T) { + assertRealIP(t, nbconfig.ReverseProxy{}, "127.0.0.1", + realip.XForwardedFor, "203.0.113.44", + realip.XRealIp, "203.0.113.44", + ) +} + +func TestRealIPUntrustedPeerIgnoresForwardedHeaders(t *testing.T) { + cfg := nbconfig.ReverseProxy{TrustedPeers: []netip.Prefix{netip.MustParsePrefix("10.9.8.7/32")}} + + assertRealIP(t, cfg, "127.0.0.1", + realip.XForwardedFor, "203.0.113.44", + realip.XRealIp, "203.0.113.44", + ) +} + +func TestRealIPTrustedPeerHonoursForwardedHeaders(t *testing.T) { + cfg := nbconfig.ReverseProxy{TrustedPeers: []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}} + + assertRealIP(t, cfg, "203.0.113.44", + realip.XForwardedFor, "203.0.113.44", + realip.XRealIp, "203.0.113.44", + ) +} + +func TestRealIPIgnoresXRealIPWhenProxyCountIsSet(t *testing.T) { + cfg := nbconfig.ReverseProxy{ + TrustedPeers: []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}, + TrustedHTTPProxiesCount: 1, + } + + assertRealIP(t, cfg, "127.0.0.1", realip.XRealIp, "203.0.113.44") +} From 794956a7a313c8919fd92123fb06ff30b7b0032a Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 11 Sep 2026 16:21:10 +0200 Subject: [PATCH 02/75] [client] Fix relay instance address race (#7498) Read the relay instance URL and IP atomically to prevent reconnects from mixing values from different connections. Extend existing connection and offer/answer logs with relay URLs and IPs to help trace mismatched advertisements. --- client/internal/peer/handshaker.go | 8 +- shared/relay/client/client.go | 41 ++++---- shared/relay/client/client_serverip_test.go | 45 ++++----- shared/relay/client/manager.go | 6 +- shared/relay/client/manager_address_test.go | 103 ++++++++++++++++++++ shared/relay/client/picker.go | 7 +- 6 files changed, 157 insertions(+), 53 deletions(-) create mode 100644 shared/relay/client/manager_address_test.go diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 6ecb2a947..654e32158 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -116,7 +116,7 @@ func (h *Handshaker) Listen(ctx context.Context) { for { select { case remoteOfferAnswer := <-h.remoteOffersCh: - h.log.Infof("received offer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials()) + h.log.Infof("received offer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t, relay server: %s, relay IP: %s", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials(), remoteOfferAnswer.RelaySrvAddress, remoteOfferAnswer.RelaySrvIP) // Record signaling received for reconnection attempts if h.metricsStages != nil { @@ -138,7 +138,7 @@ func (h *Handshaker) Listen(ctx context.Context) { continue } case remoteOfferAnswer := <-h.remoteAnswerCh: - h.log.Infof("received answer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials()) + h.log.Infof("received answer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t, relay server: %s, relay IP: %s", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials(), remoteOfferAnswer.RelaySrvAddress, remoteOfferAnswer.RelaySrvIP) // Record signaling received for reconnection attempts if h.metricsStages != nil { @@ -209,14 +209,14 @@ func (h *Handshaker) sendOffer() error { } offer := h.buildOfferAnswer() - h.log.Debugf("sending offer with serial: %s", offer.SessionIDString()) + h.log.Debugf("sending offer with serial: %s, relay server: %s, relay IP: %s", offer.SessionIDString(), offer.RelaySrvAddress, offer.RelaySrvIP) return h.signaler.SignalOffer(offer, h.config.Key) } func (h *Handshaker) sendAnswer() error { answer := h.buildOfferAnswer() - h.log.Debugf("sending answer with serial: %s", answer.SessionIDString()) + h.log.Debugf("sending answer with serial: %s, relay server: %s, relay IP: %s", answer.SessionIDString(), answer.RelaySrvAddress, answer.RelaySrvIP) return h.signaler.SignalAnswer(answer, h.config.Key) } diff --git a/shared/relay/client/client.go b/shared/relay/client/client.go index 38c9c7375..7171b40ad 100644 --- a/shared/relay/client/client.go +++ b/shared/relay/client/client.go @@ -279,7 +279,7 @@ func (c *Client) Connect(ctx context.Context) error { c.stateSubscription = NewPeersStateSubscription(c.log, c.relayConn, c.closeConnsByPeerID) c.log = c.log.WithField("relay", instanceURL.String()) - c.log.Infof("relay connection established") + c.log.Infof("relay connection established, server IP: %s", connectedIP(c.relayConn)) c.serviceIsRunning = true @@ -364,23 +364,6 @@ func (c *Client) ServerInstanceURL() (string, error) { return c.instanceURL.String(), nil } -// ConnectedIP returns the IP address of the live relay-server connection, -// extracted from the underlying socket's RemoteAddr. Zero value if not -// connected or if the address is not an IP literal. -func (c *Client) ConnectedIP() netip.Addr { - c.mu.Lock() - conn := c.relayConn - c.mu.Unlock() - if conn == nil { - return netip.Addr{} - } - addr := conn.RemoteAddr() - if addr == nil { - return netip.Addr{} - } - return extractIPLiteral(addr.String()) -} - // SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed. func (c *Client) SetOnDisconnectListener(fn func(string)) { c.listenerMutex.Lock() @@ -777,6 +760,17 @@ func (c *Client) listenForStopEvents(ctx context.Context, hc *healthcheck.Receiv } } +func (c *Client) serverInstanceAddress() (string, netip.Addr, error) { + c.mu.Lock() + defer c.mu.Unlock() + + addr, err := c.ServerInstanceURL() + if err != nil { + return "", netip.Addr{}, err + } + return addr, connectedIP(c.relayConn), nil +} + func (c *Client) closeAllConns() { for _, container := range c.conns { container.close() @@ -923,6 +917,17 @@ func (c *Client) handlePeersWentOfflineMsg(buf []byte) { c.stateSubscription.OnPeersWentOffline(peersID) } +func connectedIP(conn net.Conn) netip.Addr { + if conn == nil { + return netip.Addr{} + } + addr := conn.RemoteAddr() + if addr == nil { + return netip.Addr{} + } + return extractIPLiteral(addr.String()) +} + // extractIPLiteral returns the IP from address forms produced by the relay // dialers (URL or host:port). Zero value if the host is not an IP. func extractIPLiteral(s string) netip.Addr { diff --git a/shared/relay/client/client_serverip_test.go b/shared/relay/client/client_serverip_test.go index 7e699e37d..a52d434f7 100644 --- a/shared/relay/client/client_serverip_test.go +++ b/shared/relay/client/client_serverip_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" "github.com/netbirdio/netbird/client/iface" @@ -68,18 +70,17 @@ func TestClient_ServerIPRecoversFromUnresolvableFQDN(t *testing.T) { if !c.Ready() { t.Fatalf("client not ready after connect") } - if got := c.ConnectedIP(); got.String() != "127.0.0.1" { - t.Fatalf("ConnectedIP = %q, want 127.0.0.1", got) - } + url, ip, err := c.serverInstanceAddress() + require.NoError(t, err) + assert.Equal(t, srvCfg.ExposedAddress, url, "relay URL must come from the handshake") + assert.Equal(t, netip.MustParseAddr("127.0.0.1"), ip, "relay IP must come from the connection") }) } -// TestClient_ConnectedIPAfterFQDNDial verifies ConnectedIP returns the -// resolved IP after a successful FQDN-based dial. The underlying socket's -// RemoteAddr must be exposed through the dialer wrappers; if it returns -// the dial-time URL instead, ConnectedIP returns empty and the dial -// IP we advertise to peers is empty too. -func TestClient_ConnectedIPAfterFQDNDial(t *testing.T) { +// TestClient_ServerInstanceAddressAfterFQDNDial verifies the relay address +// includes the resolved IP after an FQDN dial. The dialer wrappers must expose +// the socket's RemoteAddr; returning the dial-time URL would lose the IP. +func TestClient_ServerInstanceAddressAfterFQDNDial(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() @@ -111,10 +112,10 @@ func TestClient_ConnectedIPAfterFQDNDial(t *testing.T) { } t.Cleanup(func() { _ = c.Close() }) - got := c.ConnectedIP().String() - if got != "127.0.0.1" && got != "::1" { - t.Fatalf("ConnectedIP after FQDN dial = %q, want 127.0.0.1 or ::1", got) - } + url, ip, err := c.serverInstanceAddress() + require.NoError(t, err) + assert.Equal(t, srvCfg.ExposedAddress, url, "relay URL must come from the handshake") + assert.Contains(t, []string{"127.0.0.1", "::1"}, ip.String(), "relay IP must resolve to localhost") } func TestSubstituteHost(t *testing.T) { @@ -214,15 +215,12 @@ func TestSubstituteHost(t *testing.T) { } } -func TestClient_ConnectedIPEmptyWhenNotConnected(t *testing.T) { - c := NewClient("rel://example.invalid:80", hmacTokenStore, "x", iface.DefaultMTU) - if got := c.ConnectedIP(); got.IsValid() { - t.Fatalf("ConnectedIP on disconnected client = %q, want zero", got) - } +func TestConnectedIPNilConnection(t *testing.T) { + assert.False(t, connectedIP(nil).IsValid(), "missing connection must not provide an IP") } // staticAddr is a net.Addr that returns a fixed string. Used to verify -// ConnectedIP parses RemoteAddr correctly. +// connectedIP parses RemoteAddr correctly. type staticAddr struct{ s string } func (a staticAddr) Network() string { return "tcp" } @@ -235,7 +233,7 @@ type stubConn struct { func (s stubConn) RemoteAddr() net.Addr { return s.remote } -func TestClient_ConnectedIPParsesRemoteAddr(t *testing.T) { +func TestConnectedIPParsesRemoteAddr(t *testing.T) { tests := []struct { name string s string @@ -252,15 +250,12 @@ func TestClient_ConnectedIPParsesRemoteAddr(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &Client{relayConn: stubConn{remote: staticAddr{s: tt.s}}} - got := c.ConnectedIP() + got := connectedIP(stubConn{remote: staticAddr{s: tt.s}}) var gotStr string if got.IsValid() { gotStr = got.String() } - if gotStr != tt.want { - t.Errorf("ConnectedIP(%q) = %q, want %q", tt.s, gotStr, tt.want) - } + assert.Equal(t, tt.want, gotStr, "IP extracted from RemoteAddr %q", tt.s) }) } } diff --git a/shared/relay/client/manager.go b/shared/relay/client/manager.go index 50fcc0b8f..367c6dfc5 100644 --- a/shared/relay/client/manager.go +++ b/shared/relay/client/manager.go @@ -256,11 +256,7 @@ func (m *Manager) RelayInstanceAddress() (string, netip.Addr, error) { if m.relayClient == nil { return "", netip.Addr{}, ErrRelayClientNotConnected } - addr, err := m.relayClient.ServerInstanceURL() - if err != nil { - return "", netip.Addr{}, err - } - return addr, m.relayClient.ConnectedIP(), nil + return m.relayClient.serverInstanceAddress() } // ServerURLs returns the addresses of the relay servers. diff --git a/shared/relay/client/manager_address_test.go b/shared/relay/client/manager_address_test.go new file mode 100644 index 000000000..4f669e60d --- /dev/null +++ b/shared/relay/client/manager_address_test.go @@ -0,0 +1,103 @@ +package client + +import ( + "net/netip" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestManager_RelayInstanceAddressAcrossReconnect(t *testing.T) { + relays := []struct { + url *RelayAddr + conn stubConn + ip netip.Addr + }{ + { + url: &RelayAddr{addr: "rels://relay-a.example:443"}, + conn: stubConn{remote: staticAddr{s: "192.0.2.1:443"}}, + ip: netip.MustParseAddr("192.0.2.1"), + }, + { + url: &RelayAddr{addr: "rels://relay-b.example:443"}, + conn: stubConn{remote: staticAddr{s: "192.0.2.2:443"}}, + ip: netip.MustParseAddr("192.0.2.2"), + }, + } + c := &Client{ + instanceURL: relays[0].url, + relayConn: relays[0].conn, + serviceIsRunning: true, + } + m := &Manager{relayClient: c} + started := make(chan struct{}) + stop := make(chan struct{}) + done := make(chan struct{}) + t.Cleanup(func() { + close(stop) + <-done + }) + go func() { + defer close(done) + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + // Publish successive connection states using the lifecycle locks. + // Yield before publication so a getter using only muInstanceURL + // can read the old URL while waiting for the new connection's IP. + c.mu.Lock() + runtime.Gosched() + relay := relays[i%len(relays)] + c.muInstanceURL.Lock() + c.instanceURL = relay.url + c.muInstanceURL.Unlock() + c.relayConn = relay.conn + c.mu.Unlock() + if i == 0 { + close(started) + } + } + }() + <-started + + for range 1000 { + url, ip, err := m.RelayInstanceAddress() + require.NoError(t, err) + wantIP := relays[0].ip + if url == relays[1].url.String() { + wantIP = relays[1].ip + } + if !assert.Equal(t, wantIP, ip, "advertised IP must belong to relay %s", url) { + return + } + } +} + +func TestManager_RelayInstanceAddressDisconnected(t *testing.T) { + for _, tt := range []struct { + name string + client *Client + }{ + {name: "no client"}, + {name: "not connected", client: &Client{}}, + { + name: "closed connection", + client: &Client{ + relayConn: stubConn{remote: staticAddr{s: "192.0.2.1:443"}}, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + m := &Manager{relayClient: tt.client} + url, ip, err := m.RelayInstanceAddress() + assert.Error(t, err) + assert.Empty(t, url, "disconnected relay must not advertise a URL") + assert.False(t, ip.IsValid(), "disconnected relay must not advertise a stale IP") + }) + } +} diff --git a/shared/relay/client/picker.go b/shared/relay/client/picker.go index 17b1390b1..fc1d8c1cb 100644 --- a/shared/relay/client/picker.go +++ b/shared/relay/client/picker.go @@ -63,7 +63,12 @@ func (sp *ServerPicker) PickServer(parentCtx context.Context) (*Client, error) { if !ok { return nil, <-errChan } - log.Infof("chosen home Relay server: %s", cr.Url) + instanceURL, serverIP, err := cr.RelayClient.serverInstanceAddress() + if err != nil { + log.Infof("chosen home Relay server: %s, instance address unavailable: %v", cr.Url, err) + return cr.RelayClient, nil + } + log.Infof("chosen home Relay server: %s, instance URL: %s, server IP: %s", cr.Url, instanceURL, serverIP) return cr.RelayClient, nil case <-ctx.Done(): return nil, fmt.Errorf("connect to relay server: %w", ctx.Err()) From f422c4165437679a1470893e7917a8ef597a2277 Mon Sep 17 00:00:00 2001 From: dmitri-netbird Date: Fri, 11 Sep 2026 17:07:26 +0200 Subject: [PATCH 03/75] [management] extract peer update logic and wrap it in tests (#7338) * extract peer update loop into a dedicated struct and wrap it in tests Signed-off-by: Dmitri Dolguikh * make linter happy Signed-off-by: Dmitri Dolguikh --------- Signed-off-by: Dmitri Dolguikh --- encryption/message.go | 10 ++ .../shared/grpc/peer_update_handler.go | 135 +++++++++++++++ .../shared/grpc/peer_update_handler_test.go | 155 ++++++++++++++++++ management/internals/shared/grpc/server.go | 88 +--------- .../internals/shared/grpc/sync_sender_mock.go | 70 ++++++++ management/internals/shared/grpc/token_mgr.go | 2 + .../internals/shared/grpc/token_mgr_mock.go | 111 +++++++++++++ .../internals/shared/grpc/update_debouncer.go | 8 + .../shared/grpc/update_debouncer_mock.go | 96 +++++++++++ 9 files changed, 589 insertions(+), 86 deletions(-) create mode 100644 management/internals/shared/grpc/peer_update_handler.go create mode 100644 management/internals/shared/grpc/peer_update_handler_test.go create mode 100644 management/internals/shared/grpc/sync_sender_mock.go create mode 100644 management/internals/shared/grpc/token_mgr_mock.go create mode 100644 management/internals/shared/grpc/update_debouncer_mock.go diff --git a/encryption/message.go b/encryption/message.go index 6e4cd7391..2bf2c59dc 100644 --- a/encryption/message.go +++ b/encryption/message.go @@ -6,6 +6,16 @@ import ( "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) +type Encrypter interface { + EncryptMessage(remotePubKey wgtypes.Key, ourPrivateKey wgtypes.Key, message pb.Message) ([]byte, error) +} + +type DefaultEncrypter struct{} + +func (e DefaultEncrypter) EncryptMessage(remotePubKey wgtypes.Key, ourPrivateKey wgtypes.Key, message pb.Message) ([]byte, error) { + return EncryptMessage(remotePubKey, ourPrivateKey, message) +} + // EncryptMessage encrypts a body of the given protobuf Message func EncryptMessage(remotePubKey wgtypes.Key, ourPrivateKey wgtypes.Key, message pb.Message) ([]byte, error) { byteResp, err := pb.Marshal(message) diff --git a/management/internals/shared/grpc/peer_update_handler.go b/management/internals/shared/grpc/peer_update_handler.go new file mode 100644 index 000000000..d2f403841 --- /dev/null +++ b/management/internals/shared/grpc/peer_update_handler.go @@ -0,0 +1,135 @@ +package grpc + +import ( + "context" + "time" + + "github.com/netbirdio/netbird/encryption" + "github.com/netbirdio/netbird/management/internals/controllers/network_map" + "github.com/netbirdio/netbird/management/server/telemetry" + "github.com/netbirdio/netbird/shared/management/proto" + log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func PeerUpdateHandlerFactory( + peerKey wgtypes.Key, + updates chan *network_map.UpdateMessage, + secretsManager SecretsManager, + srv proto.ManagementService_SyncServer, + cleanupfunc func()) *PeerUpdateHandler { + return &PeerUpdateHandler{ + peerKey: peerKey, + updates: updates, + secretsManager: secretsManager, + srv: srv, + encrypter: encryption.DefaultEncrypter{}, + debouncer: NewUpdateDebouncer(1000 * time.Millisecond), + cleanupFunc: cleanupfunc, + } +} + +// PeerUpdateHandler sends updates to the connected peer until the updates channel is closed. +// It implements a backpressure mechanism that sends the first update immediately, +// then debounces subsequent rapid updates, ensuring only the latest update is sent +// after a quiet period. +type PeerUpdateHandler struct { + peerKey wgtypes.Key + updates chan *network_map.UpdateMessage + appMetrics telemetry.AppMetrics + secretsManager SecretsManager + srv syncSender + encrypter encryption.Encrypter + debouncer Debouncer + cleanupFunc func() +} + +func (pu *PeerUpdateHandler) WithMetrics(appMetrics telemetry.AppMetrics) *PeerUpdateHandler { + pu.appMetrics = appMetrics + return pu +} + +//go:generate go tool mockgen -source=./peer_update_handler.go -destination=./sync_sender_mock.go -package=grpc +type syncSender interface { + Send(*proto.EncryptedMessage) error + Context() context.Context +} + +func (pu *PeerUpdateHandler) HandleUpdates(ctx context.Context) error { + log.WithContext(ctx).Tracef("starting to handle updates for peer %s", pu.peerKey.String()) + + defer pu.debouncer.Stop() + + for { + select { + // condition when there are some updates + // todo set the updates channel size to 1 + case update, open := <-pu.updates: + if pu.appMetrics != nil { + pu.appMetrics.GRPCMetrics().UpdateChannelQueueLength(len(pu.updates) + 1) + } + + if !open { + log.WithContext(ctx).Debugf("updates channel for peer %s was closed", pu.peerKey.String()) + pu.cleanupFunc() + return nil + } + + log.WithContext(ctx).Tracef("received an update for peer %s", pu.peerKey.String()) + if pu.debouncer.ProcessUpdate(update) { + // Send immediately (first update or after quiet period) + if err := pu.SendUpdate(ctx, update); err != nil { + log.WithContext(ctx).Debugf("error while sending an update to peer %s: %v", pu.peerKey.String(), err) + return err + } + } + + // Timer expired - quiet period reached, send pending updates if any + case <-pu.debouncer.TimerChannel(): + pendingUpdates := pu.debouncer.GetPendingUpdates() + if len(pendingUpdates) == 0 { + continue + } + log.WithContext(ctx).Debugf("sending %d debounced update(s) for peer %s", len(pendingUpdates), pu.peerKey.String()) + for _, pendingUpdate := range pendingUpdates { + if err := pu.SendUpdate(ctx, pendingUpdate); err != nil { + log.WithContext(ctx).Debugf("error while sending an update to peer %s: %v", pu.peerKey.String(), err) + return err + } + } + + // condition when client <-> server connection has been terminated + case <-pu.srv.Context().Done(): + // happens when connection drops, e.g. client disconnects + log.WithContext(ctx).Debugf("stream of peer %s has been closed", pu.peerKey.String()) + pu.cleanupFunc() + return pu.srv.Context().Err() + } + } +} + +func (pu *PeerUpdateHandler) SendUpdate(ctx context.Context, update *network_map.UpdateMessage) error { + key, err := pu.secretsManager.GetWGKey() + if err != nil { + pu.cleanupFunc() + return status.Errorf(codes.Internal, "failed processing update message") + } + + encryptedResp, err := pu.encrypter.EncryptMessage(pu.peerKey, key, update.Update) + if err != nil { + pu.cleanupFunc() + return status.Errorf(codes.Internal, "failed processing update message") + } + err = pu.srv.Send(&proto.EncryptedMessage{ + WgPubKey: key.PublicKey().String(), + Body: encryptedResp, + }) + if err != nil { + pu.cleanupFunc() + return status.Errorf(codes.Internal, "failed sending update message") + } + log.WithContext(ctx).Tracef("sent an update to peer %s", pu.peerKey.String()) + return nil +} diff --git a/management/internals/shared/grpc/peer_update_handler_test.go b/management/internals/shared/grpc/peer_update_handler_test.go new file mode 100644 index 000000000..02de49c47 --- /dev/null +++ b/management/internals/shared/grpc/peer_update_handler_test.go @@ -0,0 +1,155 @@ +package grpc + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + pb "github.com/golang/protobuf/proto" //nolint + "github.com/netbirdio/netbird/management/internals/controllers/network_map" + "github.com/netbirdio/netbird/shared/management/proto" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +func TestSendPeerUpdates_FirstUpdate(t *testing.T) { + ctrl := gomock.NewController(t) + secretsManager := NewMockSecretsManager(ctrl) + updateDebouncer := NewMockDebouncer(ctrl) + syncSender := NewMocksyncSender(ctrl) + + pu := PeerUpdateHandler{ + peerKey: mustGenerateKey(t), + updates: make(chan *network_map.UpdateMessage), + secretsManager: secretsManager, + encrypter: testEncrypter{}, + debouncer: updateDebouncer, + srv: syncSender, + cleanupFunc: func() {}, + } + + msg := network_map.UpdateMessage{ + Update: &proto.SyncResponse{Version: 1}, + } + + timeCh := make(chan time.Time) + srvCtx := context.TODO() + srvKey := mustGenerateKey(t) + // mock a first update, should send it right away + updateDebouncer.EXPECT().ProcessUpdate(gomock.Eq(&msg)).Return(true) + updateDebouncer.EXPECT().TimerChannel().AnyTimes().Return(timeCh) + syncSender.EXPECT().Context().AnyTimes().Return(srvCtx) + secretsManager.EXPECT().GetWGKey().Return(srvKey, nil) + syncSender.EXPECT().Send(pbMatcher{x: &proto.EncryptedMessage{WgPubKey: srvKey.PublicKey().String(), Body: mustMarshal(t, &msg)}}) + updateDebouncer.EXPECT().Stop() + + var wg sync.WaitGroup + wg.Go(func() { pu.HandleUpdates(context.TODO()) }) //nolint:errcheck + pu.updates <- &msg + close(pu.updates) + wg.Wait() +} + +func TestSendPeerUpdates_TimerUpdate(t *testing.T) { + ctrl := gomock.NewController(t) + secretsManager := NewMockSecretsManager(ctrl) + updateDebouncer := NewMockDebouncer(ctrl) + syncSender := NewMocksyncSender(ctrl) + + pu := PeerUpdateHandler{ + peerKey: mustGenerateKey(t), + updates: make(chan *network_map.UpdateMessage), + secretsManager: secretsManager, + encrypter: testEncrypter{}, + debouncer: updateDebouncer, + srv: syncSender, + cleanupFunc: func() {}, + } + + msg := network_map.UpdateMessage{ + Update: &proto.SyncResponse{Version: 1}, + } + + timeCh := make(chan time.Time) + srvCtx := context.TODO() + srvKey := mustGenerateKey(t) + updateDebouncer.EXPECT().GetPendingUpdates().Return([]*network_map.UpdateMessage{&msg}) + updateDebouncer.EXPECT().TimerChannel().AnyTimes().Return(timeCh) + syncSender.EXPECT().Context().AnyTimes().Return(srvCtx) + secretsManager.EXPECT().GetWGKey().Return(srvKey, nil) + syncSender.EXPECT().Send(pbMatcher{x: &proto.EncryptedMessage{WgPubKey: srvKey.PublicKey().String(), Body: mustMarshal(t, &msg)}}) + updateDebouncer.EXPECT().Stop() + + var wg sync.WaitGroup + wg.Go(func() { pu.HandleUpdates(context.TODO()) }) //nolint:errcheck + timeCh <- time.Now() + close(pu.updates) + wg.Wait() +} + +func TestSendPeerUpdates_ServerContextDone(t *testing.T) { + ctrl := gomock.NewController(t) + secretsManager := NewMockSecretsManager(ctrl) + updateDebouncer := NewMockDebouncer(ctrl) + syncSender := NewMocksyncSender(ctrl) + + pu := PeerUpdateHandler{ + peerKey: mustGenerateKey(t), + updates: make(chan *network_map.UpdateMessage), + secretsManager: secretsManager, + encrypter: testEncrypter{}, + debouncer: updateDebouncer, + srv: syncSender, + cleanupFunc: func() {}, + } + + timeCh := make(chan time.Time) + srvCtx, cancel := context.WithCancel(context.TODO()) + updateDebouncer.EXPECT().TimerChannel().AnyTimes().Return(timeCh) + syncSender.EXPECT().Context().AnyTimes().Return(srvCtx) + updateDebouncer.EXPECT().Stop() + + var wg sync.WaitGroup + wg.Go(func() { pu.HandleUpdates(context.TODO()) }) //nolint:errcheck + cancel() + wg.Wait() +} + +func mustGenerateKey(t *testing.T) wgtypes.Key { + t.Helper() + k, err := wgtypes.GenerateKey() + assert.NoError(t, err) + return k +} + +func mustMarshal(t *testing.T, msg *network_map.UpdateMessage) []byte { + t.Helper() + r, err := pb.Marshal(msg.Update) + assert.NoError(t, err) + return r +} + +type testEncrypter struct{} + +func (testEncrypter) EncryptMessage(remotePubKey wgtypes.Key, ourPrivateKey wgtypes.Key, message pb.Message) ([]byte, error) { + return pb.Marshal(message) +} + +type pbMatcher struct { + x pb.Message +} + +func (pbm pbMatcher) Matches(x any) bool { + msg, ok := x.(pb.Message) + if !ok { + return false + } + return pb.Equal(pbm.x, msg) +} + +func (pbm pbMatcher) String() string { + return fmt.Sprintf("is equal to %s (%T)", pbm.x, pbm.x) +} diff --git a/management/internals/shared/grpc/server.go b/management/internals/shared/grpc/server.go index a9cc0ad36..c178b6fa1 100644 --- a/management/internals/shared/grpc/server.go +++ b/management/internals/shared/grpc/server.go @@ -337,7 +337,8 @@ func (s *Server) Sync(req *proto.EncryptedMessage, srv proto.ManagementService_S s.syncSem.Add(-1) - return s.handleUpdates(ctx, accountID, peerKey, peer, updates, srv, syncStart) + return PeerUpdateHandlerFactory(peerKey, updates, s.secretsManager, srv, func() { s.cancelPeerRoutines(ctx, accountID, peer, syncStart) }). + WithMetrics(s.appMetrics).HandleUpdates(ctx) } func (s *Server) handleHandshake(ctx context.Context, srv proto.ManagementService_JobServer) (wgtypes.Key, error) { @@ -404,91 +405,6 @@ func (s *Server) sendJobsLoop(ctx context.Context, accountID string, peerKey wgt } } -// handleUpdates sends updates to the connected peer until the updates channel is closed. -// It implements a backpressure mechanism that sends the first update immediately, -// then debounces subsequent rapid updates, ensuring only the latest update is sent -// after a quiet period. -func (s *Server) handleUpdates(ctx context.Context, accountID string, peerKey wgtypes.Key, peer *nbpeer.Peer, updates chan *network_map.UpdateMessage, srv proto.ManagementService_SyncServer, streamStartTime time.Time) error { - log.WithContext(ctx).Tracef("starting to handle updates for peer %s", peerKey.String()) - - // Create a debouncer for this peer connection - debouncer := NewUpdateDebouncer(1000 * time.Millisecond) - defer debouncer.Stop() - - for { - select { - // condition when there are some updates - // todo set the updates channel size to 1 - case update, open := <-updates: - if s.appMetrics != nil { - s.appMetrics.GRPCMetrics().UpdateChannelQueueLength(len(updates) + 1) - } - - if !open { - log.WithContext(ctx).Debugf("updates channel for peer %s was closed", peerKey.String()) - s.cancelPeerRoutines(ctx, accountID, peer, streamStartTime) - return nil - } - - log.WithContext(ctx).Tracef("received an update for peer %s", peerKey.String()) - if debouncer.ProcessUpdate(update) { - // Send immediately (first update or after quiet period) - if err := s.sendUpdate(ctx, accountID, peerKey, peer, update, srv, streamStartTime); err != nil { - log.WithContext(ctx).Debugf("error while sending an update to peer %s: %v", peerKey.String(), err) - return err - } - } - - // Timer expired - quiet period reached, send pending updates if any - case <-debouncer.TimerChannel(): - pendingUpdates := debouncer.GetPendingUpdates() - if len(pendingUpdates) == 0 { - continue - } - log.WithContext(ctx).Debugf("sending %d debounced update(s) for peer %s", len(pendingUpdates), peerKey.String()) - for _, pendingUpdate := range pendingUpdates { - if err := s.sendUpdate(ctx, accountID, peerKey, peer, pendingUpdate, srv, streamStartTime); err != nil { - log.WithContext(ctx).Debugf("error while sending an update to peer %s: %v", peerKey.String(), err) - return err - } - } - - // condition when client <-> server connection has been terminated - case <-srv.Context().Done(): - // happens when connection drops, e.g. client disconnects - log.WithContext(ctx).Debugf("stream of peer %s has been closed", peerKey.String()) - s.cancelPeerRoutines(ctx, accountID, peer, streamStartTime) - return srv.Context().Err() - } - } -} - -// sendUpdate encrypts the update message using the peer key and the server's wireguard key, -// then sends the encrypted message to the connected peer via the sync server. -func (s *Server) sendUpdate(ctx context.Context, accountID string, peerKey wgtypes.Key, peer *nbpeer.Peer, update *network_map.UpdateMessage, srv proto.ManagementService_SyncServer, streamStartTime time.Time) error { - key, err := s.secretsManager.GetWGKey() - if err != nil { - s.cancelPeerRoutines(ctx, accountID, peer, streamStartTime) - return status.Errorf(codes.Internal, "failed processing update message") - } - - encryptedResp, err := encryption.EncryptMessage(peerKey, key, update.Update) - if err != nil { - s.cancelPeerRoutines(ctx, accountID, peer, streamStartTime) - return status.Errorf(codes.Internal, "failed processing update message") - } - err = srv.Send(&proto.EncryptedMessage{ - WgPubKey: key.PublicKey().String(), - Body: encryptedResp, - }) - if err != nil { - s.cancelPeerRoutines(ctx, accountID, peer, streamStartTime) - return status.Errorf(codes.Internal, "failed sending update message") - } - log.WithContext(ctx).Tracef("sent an update to peer %s", peerKey.String()) - return nil -} - // sendJob encrypts the update message using the peer key and the server's wireguard key, // then sends the encrypted message to the connected peer via the sync server. func (s *Server) sendJob(ctx context.Context, peerKey wgtypes.Key, job *job.Event, srv proto.ManagementService_JobServer) error { diff --git a/management/internals/shared/grpc/sync_sender_mock.go b/management/internals/shared/grpc/sync_sender_mock.go new file mode 100644 index 000000000..3d1696f59 --- /dev/null +++ b/management/internals/shared/grpc/sync_sender_mock.go @@ -0,0 +1,70 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./peer_update_handler.go +// +// Generated by this command: +// +// mockgen -source=./peer_update_handler.go -destination=./sync_sender_mock.go -package=grpc +// + +// Package grpc is a generated GoMock package. +package grpc + +import ( + context "context" + reflect "reflect" + + proto "github.com/netbirdio/netbird/shared/management/proto" + gomock "go.uber.org/mock/gomock" +) + +// MocksyncSender is a mock of syncSender interface. +type MocksyncSender struct { + ctrl *gomock.Controller + recorder *MocksyncSenderMockRecorder + isgomock struct{} +} + +// MocksyncSenderMockRecorder is the mock recorder for MocksyncSender. +type MocksyncSenderMockRecorder struct { + mock *MocksyncSender +} + +// NewMocksyncSender creates a new mock instance. +func NewMocksyncSender(ctrl *gomock.Controller) *MocksyncSender { + mock := &MocksyncSender{ctrl: ctrl} + mock.recorder = &MocksyncSenderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MocksyncSender) EXPECT() *MocksyncSenderMockRecorder { + return m.recorder +} + +// Context mocks base method. +func (m *MocksyncSender) Context() context.Context { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Context") + ret0, _ := ret[0].(context.Context) + return ret0 +} + +// Context indicates an expected call of Context. +func (mr *MocksyncSenderMockRecorder) Context() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MocksyncSender)(nil).Context)) +} + +// Send mocks base method. +func (m *MocksyncSender) Send(arg0 *proto.EncryptedMessage) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MocksyncSenderMockRecorder) Send(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MocksyncSender)(nil).Send), arg0) +} diff --git a/management/internals/shared/grpc/token_mgr.go b/management/internals/shared/grpc/token_mgr.go index fb2d83a9a..4dab5007f 100644 --- a/management/internals/shared/grpc/token_mgr.go +++ b/management/internals/shared/grpc/token_mgr.go @@ -25,6 +25,8 @@ import ( const defaultDuration = 12 * time.Hour // SecretsManager used to manage TURN and relay secrets +// +//go:generate go tool mockgen -source=./token_mgr.go -destination=./token_mgr_mock.go -package=grpc type SecretsManager interface { GenerateTurnToken() (*Token, error) GenerateRelayToken() (*Token, error) diff --git a/management/internals/shared/grpc/token_mgr_mock.go b/management/internals/shared/grpc/token_mgr_mock.go new file mode 100644 index 000000000..e7dde4e3a --- /dev/null +++ b/management/internals/shared/grpc/token_mgr_mock.go @@ -0,0 +1,111 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./token_mgr.go +// +// Generated by this command: +// +// mockgen -source=./token_mgr.go -destination=./token_mgr_mock.go -package=grpc +// + +// Package grpc is a generated GoMock package. +package grpc + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" + wgtypes "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +// MockSecretsManager is a mock of SecretsManager interface. +type MockSecretsManager struct { + ctrl *gomock.Controller + recorder *MockSecretsManagerMockRecorder + isgomock struct{} +} + +// MockSecretsManagerMockRecorder is the mock recorder for MockSecretsManager. +type MockSecretsManagerMockRecorder struct { + mock *MockSecretsManager +} + +// NewMockSecretsManager creates a new mock instance. +func NewMockSecretsManager(ctrl *gomock.Controller) *MockSecretsManager { + mock := &MockSecretsManager{ctrl: ctrl} + mock.recorder = &MockSecretsManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSecretsManager) EXPECT() *MockSecretsManagerMockRecorder { + return m.recorder +} + +// CancelRefresh mocks base method. +func (m *MockSecretsManager) CancelRefresh(peerKey string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "CancelRefresh", peerKey) +} + +// CancelRefresh indicates an expected call of CancelRefresh. +func (mr *MockSecretsManagerMockRecorder) CancelRefresh(peerKey any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelRefresh", reflect.TypeOf((*MockSecretsManager)(nil).CancelRefresh), peerKey) +} + +// GenerateRelayToken mocks base method. +func (m *MockSecretsManager) GenerateRelayToken() (*Token, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateRelayToken") + ret0, _ := ret[0].(*Token) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateRelayToken indicates an expected call of GenerateRelayToken. +func (mr *MockSecretsManagerMockRecorder) GenerateRelayToken() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateRelayToken", reflect.TypeOf((*MockSecretsManager)(nil).GenerateRelayToken)) +} + +// GenerateTurnToken mocks base method. +func (m *MockSecretsManager) GenerateTurnToken() (*Token, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateTurnToken") + ret0, _ := ret[0].(*Token) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateTurnToken indicates an expected call of GenerateTurnToken. +func (mr *MockSecretsManagerMockRecorder) GenerateTurnToken() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateTurnToken", reflect.TypeOf((*MockSecretsManager)(nil).GenerateTurnToken)) +} + +// GetWGKey mocks base method. +func (m *MockSecretsManager) GetWGKey() (wgtypes.Key, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetWGKey") + ret0, _ := ret[0].(wgtypes.Key) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetWGKey indicates an expected call of GetWGKey. +func (mr *MockSecretsManagerMockRecorder) GetWGKey() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWGKey", reflect.TypeOf((*MockSecretsManager)(nil).GetWGKey)) +} + +// SetupRefresh mocks base method. +func (m *MockSecretsManager) SetupRefresh(ctx context.Context, accountID, peerKey string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetupRefresh", ctx, accountID, peerKey) +} + +// SetupRefresh indicates an expected call of SetupRefresh. +func (mr *MockSecretsManagerMockRecorder) SetupRefresh(ctx, accountID, peerKey any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupRefresh", reflect.TypeOf((*MockSecretsManager)(nil).SetupRefresh), ctx, accountID, peerKey) +} diff --git a/management/internals/shared/grpc/update_debouncer.go b/management/internals/shared/grpc/update_debouncer.go index 8af9c2656..9483007c6 100644 --- a/management/internals/shared/grpc/update_debouncer.go +++ b/management/internals/shared/grpc/update_debouncer.go @@ -6,6 +6,14 @@ import ( "github.com/netbirdio/netbird/management/internals/controllers/network_map" ) +//go:generate go tool mockgen -source=./update_debouncer.go -destination=./update_debouncer_mock.go -package=grpc +type Debouncer interface { + Stop() + TimerChannel() <-chan time.Time + ProcessUpdate(update *network_map.UpdateMessage) bool + GetPendingUpdates() []*network_map.UpdateMessage +} + // UpdateDebouncer implements a backpressure mechanism that: // - Sends the first update immediately // - Coalesces rapid subsequent network map updates (only latest matters) diff --git a/management/internals/shared/grpc/update_debouncer_mock.go b/management/internals/shared/grpc/update_debouncer_mock.go new file mode 100644 index 000000000..4dff632b7 --- /dev/null +++ b/management/internals/shared/grpc/update_debouncer_mock.go @@ -0,0 +1,96 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./update_debouncer.go +// +// Generated by this command: +// +// mockgen -source=./update_debouncer.go -destination=./update_debouncer_mock.go -package=grpc +// + +// Package grpc is a generated GoMock package. +package grpc + +import ( + reflect "reflect" + time "time" + + network_map "github.com/netbirdio/netbird/management/internals/controllers/network_map" + gomock "go.uber.org/mock/gomock" +) + +// MockDebouncer is a mock of Debouncer interface. +type MockDebouncer struct { + ctrl *gomock.Controller + recorder *MockDebouncerMockRecorder + isgomock struct{} +} + +// MockDebouncerMockRecorder is the mock recorder for MockDebouncer. +type MockDebouncerMockRecorder struct { + mock *MockDebouncer +} + +// NewMockDebouncer creates a new mock instance. +func NewMockDebouncer(ctrl *gomock.Controller) *MockDebouncer { + mock := &MockDebouncer{ctrl: ctrl} + mock.recorder = &MockDebouncerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDebouncer) EXPECT() *MockDebouncerMockRecorder { + return m.recorder +} + +// GetPendingUpdates mocks base method. +func (m *MockDebouncer) GetPendingUpdates() []*network_map.UpdateMessage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPendingUpdates") + ret0, _ := ret[0].([]*network_map.UpdateMessage) + return ret0 +} + +// GetPendingUpdates indicates an expected call of GetPendingUpdates. +func (mr *MockDebouncerMockRecorder) GetPendingUpdates() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPendingUpdates", reflect.TypeOf((*MockDebouncer)(nil).GetPendingUpdates)) +} + +// ProcessUpdate mocks base method. +func (m *MockDebouncer) ProcessUpdate(update *network_map.UpdateMessage) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ProcessUpdate", update) + ret0, _ := ret[0].(bool) + return ret0 +} + +// ProcessUpdate indicates an expected call of ProcessUpdate. +func (mr *MockDebouncerMockRecorder) ProcessUpdate(update any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessUpdate", reflect.TypeOf((*MockDebouncer)(nil).ProcessUpdate), update) +} + +// Stop mocks base method. +func (m *MockDebouncer) Stop() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Stop") +} + +// Stop indicates an expected call of Stop. +func (mr *MockDebouncerMockRecorder) Stop() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockDebouncer)(nil).Stop)) +} + +// TimerChannel mocks base method. +func (m *MockDebouncer) TimerChannel() <-chan time.Time { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TimerChannel") + ret0, _ := ret[0].(<-chan time.Time) + return ret0 +} + +// TimerChannel indicates an expected call of TimerChannel. +func (mr *MockDebouncerMockRecorder) TimerChannel() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TimerChannel", reflect.TypeOf((*MockDebouncer)(nil).TimerChannel)) +} From ec0c36b0e7b34fd17948ff3fcd7c77d50104750e Mon Sep 17 00:00:00 2001 From: Brandon Hopkins <76761586+TechHutTV@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:25:10 -0700 Subject: [PATCH 04/75] [client] Add light mode with system, light, and dark theme options (#7344) * desktop UI light mode * Theme review fixes plus macOS window outline fix * Windows runtime chrome re-theming plus apply serialization * Windows chrome threading and theme event ordering fixes * Darken toggle and setting sidebar text * resolve theme appearance, apply on UI thread * read theme once per window * Re-assert Windows dark opt-in after SetTheme * split app-wide GTK theming from per-window chrome * Update Wails dependency and checksums * KDE tray icon panel fix * Five review fixes: theme ordering, cgo dedup, KDE panel resolution * Path guard hardening, toggle contrast, windows comment * non-vacuous escape tests * Default view edits * Polish settings nav, controls, borders, and disc * Profiles settings boarder, modals, and buttons * Additional edits based on feedback * Switch colors away from slight blue hue * Update missing lang * Fix vertical tab active view --- client/ui/frontend/index.html | 18 + client/ui/frontend/src/app.tsx | 57 +-- .../src/assets/logos/netbird-full-light.svg | 19 + client/ui/frontend/src/components/Badge.tsx | 12 +- .../src/components/CopyToClipboard.tsx | 2 +- .../frontend/src/components/DropdownMenu.tsx | 2 +- .../src/components/LanguagePicker.tsx | 6 +- .../ui/frontend/src/components/SquareIcon.tsx | 4 +- .../frontend/src/components/ThemePicker.tsx | 109 ++++++ client/ui/frontend/src/components/Tooltip.tsx | 4 +- .../frontend/src/components/VerticalTabs.tsx | 14 +- .../src/components/buttons/Button.tsx | 49 +-- .../src/components/buttons/IconButton.tsx | 2 +- .../frontend/src/components/dialog/Dialog.tsx | 4 +- .../frontend/src/components/inputs/Input.tsx | 18 +- .../src/components/inputs/SearchInput.tsx | 2 +- .../components/switches/FancyToggleSwitch.tsx | 6 +- .../src/components/switches/SwitchItem.tsx | 6 +- .../components/switches/SwitchItemGroup.tsx | 2 +- .../src/components/switches/ToggleSwitch.tsx | 6 +- .../src/components/typography/HelpText.tsx | 2 +- .../ui/frontend/src/contexts/ThemeContext.tsx | 129 +++++++ client/ui/frontend/src/globals.css | 65 +++- .../ui/frontend/src/layouts/AppRightPanel.tsx | 2 +- client/ui/frontend/src/lib/formatters.ts | 4 +- .../main/MainConnectionStatusSwitch.tsx | 13 +- .../src/modules/main/MainExitNodeSwitcher.tsx | 6 +- .../frontend/src/modules/main/MainHeader.tsx | 4 +- .../src/modules/main/advanced/Navigation.tsx | 2 +- .../main/advanced/networks/NetworkFilters.tsx | 2 +- .../main/advanced/networks/Networks.tsx | 9 +- .../main/advanced/peers/PeerDetailPanel.tsx | 12 +- .../main/advanced/peers/PeerFilters.tsx | 2 +- .../src/modules/main/advanced/peers/Peers.tsx | 6 +- .../src/modules/profiles/ProfileDropdown.tsx | 10 +- .../src/modules/profiles/ProfilesTab.tsx | 8 +- .../src/modules/settings/SettingsAbout.tsx | 16 +- .../src/modules/settings/SettingsGeneral.tsx | 2 + .../src/modules/settings/SettingsSection.tsx | 4 +- .../settings/SettingsTroubleshooting.tsx | 13 +- client/ui/frontend/tailwind.config.ts | 47 +-- client/ui/i18n/locales/de/common.json | 15 + client/ui/i18n/locales/en/common.json | 20 + client/ui/i18n/locales/es/common.json | 15 + client/ui/i18n/locales/fr/common.json | 15 + client/ui/i18n/locales/hu/common.json | 15 + client/ui/i18n/locales/it/common.json | 15 + client/ui/i18n/locales/ja/common.json | 15 + client/ui/i18n/locales/pt/common.json | 15 + client/ui/i18n/locales/ru/common.json | 15 + client/ui/i18n/locales/uk/common.json | 19 +- client/ui/i18n/locales/zh-CN/common.json | 15 + client/ui/main.go | 11 +- client/ui/preferences/store.go | 53 ++- client/ui/services/appappearance_linux.go | 99 +++++ client/ui/services/appappearance_linux_gtk.go | 63 ++++ client/ui/services/appappearance_other.go | 7 + client/ui/services/preferences.go | 4 + client/ui/services/theme.go | 194 ++++++++++ client/ui/services/windowappearance_darwin.go | 53 +++ client/ui/services/windowappearance_other.go | 16 + .../ui/services/windowappearance_windows.go | 54 +++ client/ui/services/windowmanager.go | 123 +++++- client/ui/tray_theme_linux.go | 147 ++++++-- client/ui/tray_theme_linux_test.go | 357 ++++++++++++++++-- client/ui/tray_theme_watcher_linux.go | 51 +-- go.mod | 2 +- go.sum | 4 +- 68 files changed, 1853 insertions(+), 259 deletions(-) create mode 100644 client/ui/frontend/src/assets/logos/netbird-full-light.svg create mode 100644 client/ui/frontend/src/components/ThemePicker.tsx create mode 100644 client/ui/frontend/src/contexts/ThemeContext.tsx create mode 100644 client/ui/services/appappearance_linux.go create mode 100644 client/ui/services/appappearance_linux_gtk.go create mode 100644 client/ui/services/appappearance_other.go create mode 100644 client/ui/services/theme.go create mode 100644 client/ui/services/windowappearance_darwin.go create mode 100644 client/ui/services/windowappearance_other.go create mode 100644 client/ui/services/windowappearance_windows.go diff --git a/client/ui/frontend/index.html b/client/ui/frontend/index.html index e62139956..f4c8b3d68 100644 --- a/client/ui/frontend/index.html +++ b/client/ui/frontend/index.html @@ -6,7 +6,25 @@ NetBird +
diff --git a/client/ui/frontend/src/app.tsx b/client/ui/frontend/src/app.tsx index 7f1359510..6accda36f 100644 --- a/client/ui/frontend/src/app.tsx +++ b/client/ui/frontend/src/app.tsx @@ -13,6 +13,7 @@ import { SkeletonTheme } from "react-loading-skeleton"; import "react-loading-skeleton/dist/skeleton.css"; import { welcome } from "@/lib/welcome"; import LoginWaitingForBrowserDialog from "@/modules/login/LoginWaitingForBrowserDialog.tsx"; +import { ThemeProvider } from "@/contexts/ThemeContext.tsx"; import { initI18n } from "@/lib/i18n"; import { initPlatform } from "@/lib/platform"; import { initLogForwarding } from "@/lib/logs"; @@ -35,30 +36,38 @@ Promise.all([ ]).finally(() => { ReactDOM.createRoot(document.getElementById("root")!).render( - - - - - } - /> - } /> - } - /> - } /> - } /> - - }> - } /> - } /> - } /> - - - - + + + + + + } + /> + } + /> + } + /> + } /> + } /> + + }> + } /> + } /> + } /> + + + + + , ); }); diff --git a/client/ui/frontend/src/assets/logos/netbird-full-light.svg b/client/ui/frontend/src/assets/logos/netbird-full-light.svg new file mode 100644 index 000000000..3457b50c6 --- /dev/null +++ b/client/ui/frontend/src/assets/logos/netbird-full-light.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/client/ui/frontend/src/components/Badge.tsx b/client/ui/frontend/src/components/Badge.tsx index c5e2b5f22..a6f2e886a 100644 --- a/client/ui/frontend/src/components/Badge.tsx +++ b/client/ui/frontend/src/components/Badge.tsx @@ -11,12 +11,14 @@ type Props = HTMLAttributes & { }; const VARIANT_CLASSES: Record = { - info: "bg-sky-900 border border-sky-700 text-sky-200", + info: "bg-sky-100 border border-sky-300 text-sky-800 dark:bg-sky-900 dark:border-sky-700 dark:text-sky-200", neutral: "bg-nb-gray-900 border border-nb-gray-850 text-nb-gray-200", - brand: "bg-netbird/15 border border-netbird/30 text-netbird", - success: "bg-green-900 border border-green-700 text-green-200", - warning: "bg-yellow-900 border border-yellow-700 text-yellow-200", - danger: "bg-red-900 border border-red-700 text-red-200", + brand: "bg-netbird/15 border border-netbird/30 text-netbird-700 dark:text-netbird", + success: + "bg-green-100 border border-green-300 text-green-800 dark:bg-green-900 dark:border-green-700 dark:text-green-200", + warning: + "bg-yellow-100 border border-yellow-300 text-yellow-800 dark:bg-yellow-900 dark:border-yellow-700 dark:text-yellow-200", + danger: "bg-red-100 border border-red-300 text-red-800 dark:bg-red-900 dark:border-red-700 dark:text-red-200", }; export const Badge = forwardRef(function Badge( diff --git a/client/ui/frontend/src/components/CopyToClipboard.tsx b/client/ui/frontend/src/components/CopyToClipboard.tsx index 3cf681a1c..4af4ecc8f 100644 --- a/client/ui/frontend/src/components/CopyToClipboard.tsx +++ b/client/ui/frontend/src/components/CopyToClipboard.tsx @@ -81,7 +81,7 @@ export const CopyToClipboard = ({ aria-live={"polite"} className={cn( "group/copy wails-no-draggable pointer-events-auto inline-flex cursor-default items-center gap-2 rounded-sm text-left outline-none", - "focus-visible:ring-2 focus-visible:ring-white/60 focus-visible:ring-offset-2 focus-visible:ring-offset-nb-gray-940", + "focus-visible:ring-2 focus-visible:ring-nb-gray-50/60 focus-visible:ring-offset-2 focus-visible:ring-offset-nb-gray-940", className, )} > diff --git a/client/ui/frontend/src/components/DropdownMenu.tsx b/client/ui/frontend/src/components/DropdownMenu.tsx index d43c37e1b..8cedcea03 100644 --- a/client/ui/frontend/src/components/DropdownMenu.tsx +++ b/client/ui/frontend/src/components/DropdownMenu.tsx @@ -16,7 +16,7 @@ const menuItemVariants = cva("", { variant: { default: "text-nb-gray-200 hover:bg-nb-gray-900 hover:text-nb-gray-50 focus-visible:bg-nb-gray-900 focus-visible:text-nb-gray-50 data-[state=open]:bg-nb-gray-900 data-[state=open]:text-nb-gray-50", - danger: "text-red-500 hover:bg-red-900/20 hover:text-red-500 focus-visible:bg-red-900/20 focus-visible:text-red-500", + danger: "text-red-500 hover:bg-red-500/10 hover:text-red-500 focus-visible:bg-red-500/10 focus-visible:text-red-500 dark:hover:bg-red-900/20 dark:focus-visible:bg-red-900/20", }, }, defaultVariants: { variant: "default" }, diff --git a/client/ui/frontend/src/components/LanguagePicker.tsx b/client/ui/frontend/src/components/LanguagePicker.tsx index 7a30f8b33..35ef7d5b5 100644 --- a/client/ui/frontend/src/components/LanguagePicker.tsx +++ b/client/ui/frontend/src/components/LanguagePicker.tsx @@ -97,9 +97,9 @@ export function LanguagePicker() { "rounded-md border bg-white dark:bg-nb-gray-900", "border-neutral-200 dark:border-nb-gray-700", "cursor-default text-xs font-semibold text-nb-gray-100 outline-none", - "hover:border-nb-gray-600 data-[state=open]:border-nb-gray-600", + "hover:border-nb-gray-700 data-[state=open]:border-nb-gray-700 dark:hover:border-nb-gray-600 dark:data-[state=open]:border-nb-gray-600", isFocusVisible && - "focus-visible:ring-2 focus-visible:ring-white/60 focus-visible:ring-offset-2 focus-visible:ring-offset-nb-gray-940", + "focus-visible:ring-2 focus-visible:ring-nb-gray-50/60 focus-visible:ring-offset-2 focus-visible:ring-offset-nb-gray-940", "disabled:opacity-50", )} > @@ -157,7 +157,7 @@ export function LanguagePicker() { placeholder={t("settings.general.language.search")} aria-label={t("settings.general.language.search")} className={cn( - "w-full bg-transparent text-xs text-nb-gray-100 placeholder:text-nb-gray-300", + "w-full bg-transparent text-xs text-nb-gray-100 placeholder:text-nb-gray-600 dark:placeholder:text-nb-gray-300", "border-none outline-none", )} /> diff --git a/client/ui/frontend/src/components/SquareIcon.tsx b/client/ui/frontend/src/components/SquareIcon.tsx index e904d2de5..aaf3b1100 100644 --- a/client/ui/frontend/src/components/SquareIcon.tsx +++ b/client/ui/frontend/src/components/SquareIcon.tsx @@ -5,7 +5,7 @@ import { cn } from "@/lib/cn"; export type SquareIconVariant = "default" | "info" | "warning" | "danger"; const variantClass: Record = { - default: "text-white", + default: "text-nb-gray-50", info: "text-sky-400", warning: "text-netbird", danger: "text-red-500", @@ -27,7 +27,7 @@ export const SquareIcon = ({
o.value === theme) ?? OPTIONS[0]; + const CurrentIcon = current.icon; + + const select = async (value: string) => { + if (busy || value === theme) return; + setBusy(true); + try { + await setTheme(value as ThemePreference); + } catch (e) { + await errorDialog({ + Title: t("settings.error.saveTitle"), + Message: formatErrorMessage(e), + }); + } finally { + setBusy(false); + } + }; + + return ( +
+
+ + {t("settings.general.theme.help")} +
+
+ + + + + + void select(v)}> + {OPTIONS.map(({ value, icon: Icon, labelKey }) => ( + + + {t(labelKey)} + + ))} + + + +
+
+ ); +} diff --git a/client/ui/frontend/src/components/Tooltip.tsx b/client/ui/frontend/src/components/Tooltip.tsx index 2c77ba139..d7a85277a 100644 --- a/client/ui/frontend/src/components/Tooltip.tsx +++ b/client/ui/frontend/src/components/Tooltip.tsx @@ -81,12 +81,12 @@ export const Tooltip = ({ onPointerLeave={interactive ? scheduleClose : undefined} onPointerDownOutside={interactive ? undefined : (e) => e.preventDefault()} className={cn( - "z-50 select-none text-xs text-nb-gray-100 shadow-lg", + "z-50 select-none text-xs text-nb-gray-100 shadow-sm dark:shadow-lg", "data-[state=delayed-open]:animate-in data-[state=closed]:animate-out", "data-[state=closed]:fade-out-0 data-[state=delayed-open]:fade-in-0", !interactive && "pointer-events-none", contentClassName ?? - "rounded-md border border-nb-gray-850 bg-nb-gray-900 px-2 py-1", + "rounded-md border border-nb-gray-800 bg-white px-2 py-1 dark:border-nb-gray-850 dark:bg-nb-gray-900", )} > {content} diff --git a/client/ui/frontend/src/components/VerticalTabs.tsx b/client/ui/frontend/src/components/VerticalTabs.tsx index 1aedf82a6..306850ee2 100644 --- a/client/ui/frontend/src/components/VerticalTabs.tsx +++ b/client/ui/frontend/src/components/VerticalTabs.tsx @@ -46,12 +46,12 @@ const Trigger = forwardRef(function VerticalTab (function VerticalTab aria-hidden={"true"} className={cn( "ml-2 shrink-0 transition-colors duration-150", - "text-nb-gray-400 group-data-[state=active]:text-nb-gray-100", + "text-nb-gray-350 dark:text-nb-gray-400", + "group-data-[state=active]:text-nb-gray-100", )} /> {title} diff --git a/client/ui/frontend/src/components/buttons/Button.tsx b/client/ui/frontend/src/components/buttons/Button.tsx index 6b151c17b..931988ead 100644 --- a/client/ui/frontend/src/components/buttons/Button.tsx +++ b/client/ui/frontend/src/components/buttons/Button.tsx @@ -24,71 +24,74 @@ const buttonVariants = cva( variants: { variant: { default: [ - "border-gray-200 bg-white text-gray-900 hover:bg-gray-100 hover:text-black focus:ring-zinc-200/50", - "dark:border-gray-700/30 dark:bg-nb-gray dark:text-gray-400 dark:hover:bg-zinc-800/50 dark:hover:text-white dark:focus:ring-zinc-800/50", + "border-neutral-200 bg-white text-neutral-900 hover:bg-neutral-100 hover:text-black focus:ring-neutral-200/50", + "dark:border-gray-700/30 dark:bg-nb-gray dark:text-gray-400 dark:hover:bg-zinc-800/50 dark:hover:text-nb-gray-50 dark:focus:ring-zinc-800/50", ], primary: [ - "dark:text-gray-100 dark:ring-offset-neutral-950/50 dark:focus:ring-netbird-600/50 enabled:dark:bg-netbird enabled:dark:hover:bg-netbird-500/80 enabled:dark:hover:text-white disabled:dark:bg-nb-gray-900", - "enabled:bg-netbird enabled:text-white enabled:hover:bg-netbird-500 enabled:focus:ring-netbird-400/50", + "dark:text-gray-100 dark:ring-offset-neutral-950/50 dark:focus:ring-netbird-600/50 enabled:dark:bg-netbird enabled:dark:hover:bg-netbird-500/80 enabled:dark:hover:text-nb-gray-50 disabled:dark:bg-nb-gray-900", + "enabled:bg-netbird enabled:text-white enabled:hover:bg-netbird-500 enabled:focus:ring-netbird-400/50 disabled:bg-nb-gray-700", ], secondary: [ - "border-gray-200 bg-white text-gray-900 hover:bg-gray-100 hover:text-black focus:ring-zinc-200/50", - "dark:ring-offset-neutral-950/50 dark:focus:ring-neutral-500/20", - "dark:border-gray-700/40 dark:bg-nb-gray-920 dark:text-gray-400 dark:hover:bg-nb-gray-910 dark:hover:text-white", + "border-neutral-200 bg-white text-neutral-900 hover:border-nb-gray-700 hover:bg-nb-gray-950 hover:text-black focus:ring-nb-gray-500/50 focus:ring-offset-0", + "dark:ring-offset-neutral-950/50 dark:focus:ring-neutral-500/20 dark:focus:ring-offset-1", + "dark:border-gray-700/40 dark:bg-nb-gray-920 dark:text-gray-400 dark:hover:border-gray-700/40 dark:hover:bg-nb-gray-910 dark:hover:text-nb-gray-50", ], secondaryLighter: [ - "border-gray-200 bg-white text-gray-900 hover:bg-gray-100 hover:text-black focus:ring-zinc-200/50", + "border-neutral-200 bg-white text-neutral-900 hover:bg-neutral-100 hover:text-black focus:ring-neutral-200/50", "dark:ring-offset-neutral-950/50 dark:focus:ring-neutral-500/20", - "dark:border-gray-700/70 dark:bg-nb-gray-900/70 dark:text-gray-400 dark:hover:bg-nb-gray-800/60 dark:hover:text-white", + "dark:border-gray-700/70 dark:bg-nb-gray-900/70 dark:text-gray-400 dark:hover:bg-nb-gray-800/60 dark:hover:text-nb-gray-50", ], subtle: [ - "border-nb-gray-200 bg-nb-gray-50 text-nb-gray-900 hover:bg-nb-gray-100 focus:ring-nb-gray-200/60", + "border-neutral-200 bg-neutral-50 text-neutral-900 hover:bg-neutral-100 focus:ring-neutral-200/60", "dark:ring-offset-neutral-950/50 dark:focus:ring-nb-gray-200/40", "dark:border-nb-gray-200 dark:bg-nb-gray-50 dark:text-nb-gray-900 dark:hover:bg-nb-gray-100 dark:hover:text-nb-gray-950", ], input: [ - "border-neutral-200 bg-white text-gray-900 hover:bg-gray-100 hover:text-black focus:ring-zinc-200/50", + "border-neutral-200 bg-white text-neutral-900 hover:bg-neutral-100 hover:text-black focus:ring-neutral-200/50", "dark:ring-offset-neutral-950/50 dark:focus:ring-neutral-500/20", "dark:border-nb-gray-700 dark:bg-nb-gray-900 dark:text-gray-400 dark:hover:bg-nb-gray-900/80", ], dropdown: [ - "border-neutral-200 bg-white text-gray-900 hover:bg-gray-100 hover:text-black focus:ring-zinc-200/50", + "border-neutral-200 bg-white text-neutral-900 hover:bg-neutral-100 hover:text-black focus:ring-neutral-200/50", "dark:ring-offset-neutral-950/50 dark:focus:ring-neutral-500/20", "dark:border-nb-gray-900 dark:bg-nb-gray-900/40 dark:text-gray-400 dark:hover:bg-nb-gray-900/50", ], dotted: [ - "border-dashed border-gray-200 bg-white text-gray-900 hover:bg-gray-100 hover:text-black focus:ring-zinc-200/50", + "border-dashed border-neutral-200 bg-white text-neutral-900 hover:bg-neutral-100 hover:text-black focus:ring-neutral-200/50", "dark:ring-offset-neutral-950/50 dark:focus:ring-neutral-500/20", - "dark:border-gray-500/40 dark:bg-nb-gray-900/30 dark:text-gray-400 dark:hover:bg-nb-gray-900/50 dark:hover:text-white", + "dark:border-gray-500/40 dark:bg-nb-gray-900/30 dark:text-gray-400 dark:hover:bg-nb-gray-900/50 dark:hover:text-nb-gray-50", ], tertiary: [ - "border-gray-200 bg-white text-gray-900 hover:bg-gray-100 hover:text-black focus:ring-zinc-200/50", + "border-neutral-200 bg-white text-neutral-900 hover:bg-neutral-100 hover:text-black focus:ring-neutral-200/50", "dark:border-gray-700/40 dark:bg-white dark:text-gray-800 dark:hover:bg-neutral-200 dark:focus:ring-zinc-800/50 disabled:dark:bg-nb-gray-920 disabled:dark:text-nb-gray-300", ], white: [ - "border-white bg-white text-gray-800 outline-none hover:bg-neutral-200 focus:ring-white/50 disabled:dark:bg-nb-gray-920 disabled:dark:text-nb-gray-300", + "border-white bg-white text-neutral-800 outline-none hover:bg-neutral-200 focus:ring-white/50 dark:text-gray-800 disabled:dark:bg-nb-gray-920 disabled:dark:text-nb-gray-300", "disabled:dark:border-nb-gray-900 disabled:dark:bg-nb-gray-900 disabled:dark:text-nb-gray-300", ], outline: [ - "border-gray-200 bg-white text-gray-900 hover:bg-gray-100 hover:text-black focus:ring-zinc-200/50", + "border-neutral-200 bg-white text-neutral-900 hover:bg-neutral-100 hover:text-black focus:ring-neutral-200/50", "dark:border-netbird dark:bg-transparent dark:text-netbird dark:hover:bg-nb-gray-900/30 dark:focus:ring-zinc-800/50", ], "danger-outline": [ + "bg-transparent text-red-600 enabled:hover:bg-red-50 enabled:focus:ring-red-200/50", "dark:bg-transparent dark:text-red-500 enabled:dark:hover:border-red-800/50 enabled:hover:dark:bg-red-950/50 enabled:dark:focus:bg-red-950/40 enabled:dark:focus:ring-red-800/20", ], "danger-text": [ - "rounded-sm !px-0 !py-0 !shadow-none focus:ring-red-500/30 dark:border-transparent dark:bg-transparent dark:text-red-500 dark:ring-offset-neutral-950/50 dark:hover:text-red-600", + "rounded-sm border-transparent bg-transparent !px-0 !py-0 text-red-600 !shadow-none hover:text-red-700 focus:ring-red-500/30", + "dark:border-transparent dark:bg-transparent dark:text-red-500 dark:ring-offset-neutral-950/50 dark:hover:text-red-600", ], "default-outline": [ - "dark:ring-offset-nb-gray-950/50 dark:focus:ring-nb-gray-500/20", - "dark:border-transparent dark:bg-transparent dark:text-nb-gray-400 dark:hover:border-nb-gray-800/50 dark:hover:bg-nb-gray-900/30 dark:hover:text-white", - "data-[state=open]:dark:border-nb-gray-800/50 data-[state=open]:dark:bg-nb-gray-900/30 data-[state=open]:dark:text-white", + "ring-offset-nb-gray-950/50 focus:ring-nb-gray-500/20", + "border-transparent bg-transparent text-nb-gray-400 hover:border-nb-gray-800/50 hover:bg-nb-gray-900/30 hover:text-nb-gray-50", + "data-[state=open]:border-nb-gray-800/50 data-[state=open]:bg-nb-gray-900/30 data-[state=open]:text-nb-gray-50", ], ghost: [ - "dark:ring-offset-nb-gray-950/50 dark:focus:ring-nb-gray-500/20", - "dark:border-transparent dark:bg-transparent dark:text-nb-gray-400 dark:hover:bg-nb-gray-900/30 dark:hover:text-white", + "ring-offset-nb-gray-950/50 focus:ring-nb-gray-500/20", + "border-transparent bg-transparent text-nb-gray-400 hover:bg-nb-gray-900/30 hover:text-nb-gray-50", ], danger: [ + "bg-red-600 text-red-50 hover:bg-red-700 focus:bg-red-700 focus:ring-red-700/20", "dark:bg-red-600 dark:text-red-100 dark:hover:border-red-800/50 hover:dark:bg-red-700 dark:focus:bg-red-700 dark:focus:ring-red-700/20", ], }, diff --git a/client/ui/frontend/src/components/buttons/IconButton.tsx b/client/ui/frontend/src/components/buttons/IconButton.tsx index 3d36bc111..8d688cb0b 100644 --- a/client/ui/frontend/src/components/buttons/IconButton.tsx +++ b/client/ui/frontend/src/components/buttons/IconButton.tsx @@ -24,7 +24,7 @@ export const IconButton = forwardRef(function IconButt "flex h-10 w-10 cursor-default items-center justify-center rounded-lg outline-none", "text-nb-gray-400 hover:bg-nb-gray-900 hover:text-nb-gray-300", isFocusVisible && - "focus-visible:ring-2 focus-visible:ring-white/60 focus-visible:ring-offset-2 focus-visible:ring-offset-nb-gray-940", + "focus-visible:ring-2 focus-visible:ring-nb-gray-50/60 focus-visible:ring-offset-2 focus-visible:ring-offset-nb-gray-940", "wails-no-draggable transition-colors duration-150", className, )} diff --git a/client/ui/frontend/src/components/dialog/Dialog.tsx b/client/ui/frontend/src/components/dialog/Dialog.tsx index fa8007d9f..c43c04b0b 100644 --- a/client/ui/frontend/src/components/dialog/Dialog.tsx +++ b/client/ui/frontend/src/components/dialog/Dialog.tsx @@ -23,7 +23,7 @@ const Overlay = forwardRef, OverlayPr ref={ref} className={cn( "fixed inset-0 z-50 grid items-center justify-items-center overflow-y-auto px-10 py-16", - "bg-black/60", + "bg-black/25 dark:bg-black/60", "data-[state=open]:animate-in data-[state=open]:fade-in-0", exitAnimation && "data-[state=closed]:animate-out data-[state=closed]:fade-out-0", @@ -67,7 +67,7 @@ export const Content = forwardRef, Co className={cn( "relative z-[52] mx-auto w-full outline-none ring-0", "focus:outline-none focus:ring-0 focus-visible:outline-none focus-visible:ring-0", - "rounded-lg border border-nb-gray-900 bg-nb-gray py-7 shadow-2xl", + "rounded-lg border border-nb-gray-800 bg-nb-gray-940 py-7 shadow-2xl dark:border-nb-gray-900 dark:bg-nb-gray", "data-[state=open]:animate-in data-[state=open]:fade-in-0", "data-[state=open]:zoom-in-95 data-[state=open]:slide-in-from-left-1", exitAnimation && diff --git a/client/ui/frontend/src/components/inputs/Input.tsx b/client/ui/frontend/src/components/inputs/Input.tsx index 2dad80d7a..eada79a1f 100644 --- a/client/ui/frontend/src/components/inputs/Input.tsx +++ b/client/ui/frontend/src/components/inputs/Input.tsx @@ -32,19 +32,19 @@ const inputVariants = cva("", { variants: { variant: { default: [ - "border-neutral-200 placeholder:text-neutral-500 dark:border-nb-gray-700 dark:bg-nb-gray-900 dark:placeholder:text-neutral-400/70", + "border-neutral-200 placeholder:text-nb-gray-600 dark:border-nb-gray-700 dark:bg-nb-gray-900 dark:placeholder:text-neutral-400/70", "ring-offset-neutral-200/20 focus-visible:ring-neutral-300/10 dark:ring-offset-neutral-950/50 dark:focus-visible:ring-neutral-500/20", ], darker: [ - "border-neutral-300 placeholder:text-neutral-500 dark:border-nb-gray-800 dark:bg-nb-gray-920 dark:placeholder:text-neutral-400/70", + "border-neutral-300 placeholder:text-nb-gray-600 dark:border-nb-gray-800 dark:bg-nb-gray-920 dark:placeholder:text-neutral-400/70", "ring-offset-neutral-200/20 focus-visible:ring-neutral-300/10 dark:ring-offset-neutral-950/50 dark:focus-visible:ring-neutral-500/20", ], error: [ - "border-neutral-200 text-red-500 placeholder:text-neutral-500 dark:border-red-500 dark:bg-nb-gray-900 dark:placeholder:text-neutral-400/70", + "border-neutral-200 text-red-500 placeholder:text-nb-gray-600 dark:border-red-500 dark:bg-nb-gray-900 dark:placeholder:text-neutral-400/70", "ring-offset-red-500/10 focus-visible:ring-red-500/10 dark:ring-offset-red-500/10 dark:focus-visible:ring-red-500/10", ], warning: [ - "border-neutral-200 text-orange-400 placeholder:text-neutral-500 dark:border-orange-400 dark:bg-nb-gray-900 dark:placeholder:text-neutral-400/70", + "border-neutral-200 text-orange-400 placeholder:text-nb-gray-600 dark:border-orange-400 dark:bg-nb-gray-900 dark:placeholder:text-neutral-400/70", "ring-offset-orange-400/10 focus-visible:ring-orange-400/10 dark:ring-offset-orange-400/10 dark:focus-visible:ring-orange-400/10", ], }, @@ -158,7 +158,7 @@ function NumberStepper({ className={cn( "flex h-[40px] shrink-0 flex-col overflow-hidden", "rounded-r-md border border-l-0", - "border-neutral-200 dark:border-nb-gray-700 dark:bg-nb-gray-900", + "border-neutral-200 bg-white dark:border-nb-gray-700 dark:bg-nb-gray-900", error && "dark:border-red-500", disabled && "pointer-events-none opacity-40", )} @@ -274,7 +274,9 @@ export const Input = forwardRef(function Input( - - - void select(v)}> - {OPTIONS.map(({ value, icon: Icon, labelKey }) => ( - - - {t(labelKey)} - - ))} - - - + ({ + value, + icon, + label: t(`settings.troubleshooting.anonymize.${value}`), + }))} + onChange={setAnonymizeLevel} + ariaLabel={t("settings.troubleshooting.anonymize.label")} + />
Date: Tue, 29 Sep 2026 18:29:55 +0300 Subject: [PATCH 63/75] [management,proxy] Use single-use codes for OIDC session handoff (#7635) * Generalize PKCE verifier store into SingleUseStore * Generalize PKCE verifier store into SingleUseStore * Extend single-use store to generate one-time retrieval codes * Hand off proxy OIDC session via one-time code instead of URL token * Use the single-use store in integration tests * Read active proxy versions by cluster * Detect proxy clusters that support session codes * Bind OIDC session handoff mode to signed state * Deprecate legacy OIDC session token handoff * Remove unrelated session code test stub * fix tests * fix merge * Fix session code compatibility detection * Isolate proxy session codes in shared cache * bump min session version --- .../modules/reverseproxy/proxy/manager.go | 1 + .../reverseproxy/proxy/manager/manager.go | 20 + .../proxy/manager/manager_test.go | 35 ++ .../reverseproxy/proxy/manager_mock.go | 14 + management/internals/shared/grpc/proxy.go | 71 ++- .../internals/shared/grpc/proxy_test.go | 28 +- .../internals/shared/grpc/single_use.go | 20 + .../internals/shared/grpc/single_use_test.go | 37 ++ .../shared/grpc/validate_session_test.go | 48 ++ management/server/http/handlers/proxy/auth.go | 15 +- .../proxy/auth_callback_integration_test.go | 87 +++- management/server/store/sql_store_proxy.go | 15 + management/server/store/store.go | 1 + management/server/store/store_mock.go | 15 + proxy/internal/auth/middleware.go | 49 +- proxy/internal/auth/middleware_test.go | 26 ++ proxy/internal/auth/oidc.go | 11 +- proxy/internal/proxy/reverseproxy.go | 7 +- proxy/management_integration_test.go | 10 +- proxy/web/web.go | 1 + shared/management/proto/proxy_service.pb.go | 439 +++++++++--------- shared/management/proto/proxy_service.proto | 7 +- 22 files changed, 676 insertions(+), 281 deletions(-) diff --git a/management/internals/modules/reverseproxy/proxy/manager.go b/management/internals/modules/reverseproxy/proxy/manager.go index a591b86ca..c0b8435ec 100644 --- a/management/internals/modules/reverseproxy/proxy/manager.go +++ b/management/internals/modules/reverseproxy/proxy/manager.go @@ -20,6 +20,7 @@ type Manager interface { ClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool ClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool ClusterSupportsPrivate(ctx context.Context, clusterAddr string) *bool + ClusterSupportsSessionCode(ctx context.Context, clusterAddr string) bool CleanupStale(ctx context.Context, inactivityDuration time.Duration) error GetAccountProxy(ctx context.Context, accountID string) (*Proxy, error) CountAccountProxies(ctx context.Context, accountID string) (int64, error) diff --git a/management/internals/modules/reverseproxy/proxy/manager/manager.go b/management/internals/modules/reverseproxy/proxy/manager/manager.go index edfa32aa9..7ddb66eec 100644 --- a/management/internals/modules/reverseproxy/proxy/manager/manager.go +++ b/management/internals/modules/reverseproxy/proxy/manager/manager.go @@ -8,6 +8,7 @@ import ( "go.opentelemetry.io/otel/metric" "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" + nbversion "github.com/netbirdio/netbird/version" ) // store defines the interface for proxy persistence operations @@ -22,6 +23,7 @@ type store interface { GetClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool GetClusterSupportsPrivate(ctx context.Context, clusterAddr string) *bool + GetActiveProxyVersions(ctx context.Context, clusterAddr string) ([]string, error) CleanupStaleProxies(ctx context.Context, inactivityDuration time.Duration) error GetProxyByAccountID(ctx context.Context, accountID string) (*proxy.Proxy, error) CountProxiesByAccountID(ctx context.Context, accountID string) (int64, error) @@ -29,6 +31,8 @@ type store interface { DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error } +const minSessionCodeVersion = "0.81.0" + // Manager handles all proxy operations type Manager struct { store store @@ -145,6 +149,22 @@ func (m Manager) ClusterSupportsPrivate(ctx context.Context, clusterAddr string) return m.store.GetClusterSupportsPrivate(ctx, clusterAddr) } +// ClusterSupportsSessionCode reports whether all active proxies support session codes. +func (m Manager) ClusterSupportsSessionCode(ctx context.Context, clusterAddr string) bool { + versions, err := m.store.GetActiveProxyVersions(ctx, clusterAddr) + if err != nil || len(versions) == 0 { + return false + } + + for _, version := range versions { + if supported, err := nbversion.MeetsMinVersion(minSessionCodeVersion, version); err != nil || !supported { + return false + } + } + + return true +} + // CleanupStale removes proxies that haven't sent heartbeat in the specified duration func (m *Manager) CleanupStale(ctx context.Context, inactivityDuration time.Duration) error { if err := m.store.CleanupStaleProxies(ctx, inactivityDuration); err != nil { diff --git a/management/internals/modules/reverseproxy/proxy/manager/manager_test.go b/management/internals/modules/reverseproxy/proxy/manager/manager_test.go index d5a3ce777..66ddb95bd 100644 --- a/management/internals/modules/reverseproxy/proxy/manager/manager_test.go +++ b/management/internals/modules/reverseproxy/proxy/manager/manager_test.go @@ -22,6 +22,7 @@ type mockStore struct { updateProxyHeartbeatFunc func(ctx context.Context, p *proxy.Proxy) error getActiveProxyClusterAddressesFunc func(ctx context.Context) ([]string, error) getActiveProxyClusterAddressesForAccFunc func(ctx context.Context, accountID string) ([]string, error) + getActiveProxyVersionsFunc func(ctx context.Context, clusterAddress string) ([]string, error) cleanupStaleProxiesFunc func(ctx context.Context, d time.Duration) error getProxyByAccountIDFunc func(ctx context.Context, accountID string) (*proxy.Proxy, error) countProxiesByAccountIDFunc func(ctx context.Context, accountID string) (int64, error) @@ -104,6 +105,12 @@ func (m *mockStore) GetClusterSupportsCrowdSec(_ context.Context, _ string) *boo func (m *mockStore) GetClusterSupportsPrivate(_ context.Context, _ string) *bool { return nil } +func (m *mockStore) GetActiveProxyVersions(ctx context.Context, clusterAddress string) ([]string, error) { + if m.getActiveProxyVersionsFunc != nil { + return m.getActiveProxyVersionsFunc(ctx, clusterAddress) + } + return nil, nil +} func newTestManager(s store) *Manager { meter := noop.NewMeterProvider().Meter("test") @@ -114,6 +121,34 @@ func newTestManager(s store) *Manager { return m } +func TestClusterSupportsSessionCode(t *testing.T) { + tests := []struct { + name string + versions []string + storeErr error + want bool + }{ + {name: "all supported", versions: []string{"0.81.0", "0.81.2"}, want: true}, + {name: "one old proxy", versions: []string{"0.81.0", "0.80.0"}}, + {name: "missing version", versions: []string{"0.81.0", ""}}, + {name: "no active proxies"}, + {name: "store error", storeErr: errors.New("db error")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &mockStore{ + getActiveProxyVersionsFunc: func(_ context.Context, _ string) ([]string, error) { + return tt.versions, tt.storeErr + }, + } + + got := newTestManager(s).ClusterSupportsSessionCode(context.Background(), "cluster.example.com") + assert.Equal(t, tt.want, got) + }) + } +} + func TestConnect_WithAccountID(t *testing.T) { accountID := "acc-123" diff --git a/management/internals/modules/reverseproxy/proxy/manager_mock.go b/management/internals/modules/reverseproxy/proxy/manager_mock.go index ec6df8a4a..d6f7197d7 100644 --- a/management/internals/modules/reverseproxy/proxy/manager_mock.go +++ b/management/internals/modules/reverseproxy/proxy/manager_mock.go @@ -112,6 +112,20 @@ func (mr *MockManagerMockRecorder) ClusterSupportsPrivate(ctx, clusterAddr any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterSupportsPrivate", reflect.TypeOf((*MockManager)(nil).ClusterSupportsPrivate), ctx, clusterAddr) } +// ClusterSupportsSessionCode mocks base method. +func (m *MockManager) ClusterSupportsSessionCode(ctx context.Context, clusterAddr string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClusterSupportsSessionCode", ctx, clusterAddr) + ret0, _ := ret[0].(bool) + return ret0 +} + +// ClusterSupportsSessionCode indicates an expected call of ClusterSupportsSessionCode. +func (mr *MockManagerMockRecorder) ClusterSupportsSessionCode(ctx, clusterAddr any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterSupportsSessionCode", reflect.TypeOf((*MockManager)(nil).ClusterSupportsSessionCode), ctx, clusterAddr) +} + // Connect mocks base method. func (m *MockManager) Connect(ctx context.Context, proxyID, sessionID, clusterAddress, ipAddress, version string, accountID *string, capabilities *Capabilities) (*Proxy, error) { m.ctrl.T.Helper() diff --git a/management/internals/shared/grpc/proxy.go b/management/internals/shared/grpc/proxy.go index e5d89ab3d..00355087d 100644 --- a/management/internals/shared/grpc/proxy.go +++ b/management/internals/shared/grpc/proxy.go @@ -27,8 +27,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/netbirdio/netbird/shared/management/domain" - "github.com/netbirdio/netbird/management/internals/modules/agentnetwork" "github.com/netbirdio/netbird/management/internals/modules/peers" "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs" @@ -42,6 +40,7 @@ import ( "github.com/netbirdio/netbird/management/server/users" proxyauth "github.com/netbirdio/netbird/proxy/auth" "github.com/netbirdio/netbird/shared/hash/argon2id" + "github.com/netbirdio/netbird/shared/management/domain" "github.com/netbirdio/netbird/shared/management/proto" nbstatus "github.com/netbirdio/netbird/shared/management/status" ) @@ -142,7 +141,7 @@ type ProxyServiceServer struct { // OIDC configuration for proxy authentication oidcConfig ProxyOIDCConfig - // Store for PKCE verifiers + // singleUseStore backs both PKCE verifiers and OIDC session exchange codes. singleUseStore *SingleUseStore // tokenTTL is the lifetime of one-time tokens generated for proxy @@ -158,6 +157,13 @@ type ProxyServiceServer struct { const pkceVerifierTTL = 10 * time.Minute +const sessionCodeTTL = 60 * time.Second + +const sessionCodeCacheNamespace = "proxy:session" + +// The signed nonce binds the handoff mode without changing the state format. +const sessionCodeNoncePrefix = "code." + const defaultProxyTokenTTL = 5 * time.Minute const defaultSnapshotBatchSize = 500 @@ -306,6 +312,16 @@ func (s *ProxyServiceServer) proxyConnectAuthorizer() ProxyConnectAuthorizer { return s.connectAuthorizer } +// GenerateSessionCode creates a single-use code for the given session token. +func (s *ProxyServiceServer) GenerateSessionCode(sessionToken string) (code string, ok bool) { + code, err := s.singleUseStore.Generate(sessionCodeCacheNamespace, sessionToken, sessionCodeTTL) + if err != nil { + log.WithError(err).Error("failed to generate proxy session code") + return "", false + } + return code, true +} + // CheckLLMPolicyLimits is the pre-flight policy gate the proxy calls before // forwarding an LLM request upstream. Delegates to the agent-network selector, // which scores applicable policies by remaining headroom and returns the @@ -1536,18 +1552,20 @@ func (s *ProxyServiceServer) GetOIDCURL(ctx context.Context, req *proto.GetOIDCU log.WithContext(ctx).Errorf("failed to get account services: %v", err) return nil, status.Errorf(codes.FailedPrecondition, "get account services: %v", err) } - var found bool + var matchedService *rpservice.Service for _, service := range services { if service.Domain == redirectURL.Hostname() { - found = true + matchedService = service break } } - if !found { + if matchedService == nil { log.WithContext(ctx).Debugf("OIDC redirect URL %q does not match any service domain", redirectURL.Hostname()) return nil, status.Errorf(codes.FailedPrecondition, "service not found in store") } + useSessionCode := s.proxyManager.ClusterSupportsSessionCode(ctx, matchedService.ProxyCluster) + provider, err := oidc.NewProvider(ctx, s.oidcConfig.Issuer) if err != nil { log.WithContext(ctx).Errorf("failed to create OIDC provider: %v", err) @@ -1567,9 +1585,12 @@ func (s *ProxyServiceServer) GetOIDCURL(ctx context.Context, req *proto.GetOIDCU return nil, status.Errorf(codes.Internal, "generate nonce: %v", err) } nonceB64 := base64.URLEncoding.EncodeToString(nonce) + if useSessionCode { + nonceB64 = sessionCodeNoncePrefix + nonceB64 + } // Using an HMAC here to avoid redirection state being modified. - // State format: base64(redirectURL)|nonce|hmac(redirectURL|nonce) + // State format: base64(redirectURL)|[code.]nonce|hmac(redirectURL|nonce) payload := redirectURL.String() + "|" + nonceB64 hmacSum := s.generateHMAC(payload) state := fmt.Sprintf("%s|%s|%s", base64.URLEncoding.EncodeToString([]byte(redirectURL.String())), nonceB64, hmacSum) @@ -1612,15 +1633,12 @@ func (s *ProxyServiceServer) generateHMAC(input string) string { return hex.EncodeToString(mac.Sum(nil)) } -// ValidateState validates the state parameter from an OAuth callback. -// Returns the original redirect URL if valid, or an error if invalid. -// The HMAC is verified before consuming the PKCE verifier to prevent -// an attacker from invalidating a legitimate user's auth flow. -func (s *ProxyServiceServer) ValidateState(state string) (verifier, redirectURL string, err error) { - // State format: base64(redirectURL)|nonce|hmac(redirectURL|nonce) +// ValidateState validates and consumes an OIDC state. +func (s *ProxyServiceServer) ValidateState(state string) (verifier, redirectURL string, useSessionCode bool, err error) { + // State format: base64(redirectURL)|[code.]nonce|hmac(redirectURL|nonce) parts := strings.Split(state, "|") if len(parts) != 3 { - return "", "", errors.New("invalid state format") + return "", "", false, errors.New("invalid state format") } encodedURL := parts[0] @@ -1629,7 +1647,7 @@ func (s *ProxyServiceServer) ValidateState(state string) (verifier, redirectURL redirectURLBytes, err := base64.URLEncoding.DecodeString(encodedURL) if err != nil { - return "", "", fmt.Errorf("invalid state encoding: %w", err) + return "", "", false, fmt.Errorf("invalid state encoding: %w", err) } redirectURL = string(redirectURLBytes) @@ -1637,16 +1655,17 @@ func (s *ProxyServiceServer) ValidateState(state string) (verifier, redirectURL expectedHMAC := s.generateHMAC(payload) if !hmac.Equal([]byte(providedHMAC), []byte(expectedHMAC)) { - return "", "", errors.New("invalid state signature") + return "", "", false, errors.New("invalid state signature") } + useSessionCode = strings.HasPrefix(nonce, sessionCodeNoncePrefix) // Consume the PKCE verifier only after HMAC validation passes. verifier, ok := s.singleUseStore.LoadAndDelete(state) if !ok { - return "", "", errors.New("no verifier for state") + return "", "", false, errors.New("no verifier for state") } - return verifier, redirectURL, nil + return verifier, redirectURL, useSessionCode, nil } // Denied reasons reported to the proxy when access is refused because of the @@ -1851,7 +1870,20 @@ func (s *ProxyServiceServer) getAccountServiceByDomain(ctx context.Context, acco // ValidateSession validates a session token and checks if the user has access to the domain. func (s *ProxyServiceServer) ValidateSession(ctx context.Context, req *proto.ValidateSessionRequest) (*proto.ValidateSessionResponse, error) { domain := req.GetDomain() - sessionToken := req.GetSessionToken() + sessionToken := req.GetSessionToken() //nolint:staticcheck + + // A one-time code from the OIDC callback is redeemed here for the durable + // token, so the token never travels in a redirect URL. The redeemed token + // is returned to the proxy (mintedToken) to install as the session cookie. + mintedToken := "" + if code := req.GetSessionCode(); code != "" { + redeemed, found := s.singleUseStore.LoadAndDelete(singleUseCacheKey(sessionCodeCacheNamespace, code)) + if !found { + return deniedSessionResponse("invalid or expired session code"), nil + } + sessionToken = redeemed + mintedToken = redeemed + } if domain == "" || sessionToken == "" { return deniedSessionResponse("missing domain or session_token"), nil @@ -1924,6 +1956,7 @@ func (s *ProxyServiceServer) ValidateSession(ctx context.Context, req *proto.Val UserEmail: user.Email, PeerGroupIds: groupIDs, PeerGroupNames: groupNames, + SessionToken: mintedToken, }, nil } diff --git a/management/internals/shared/grpc/proxy_test.go b/management/internals/shared/grpc/proxy_test.go index 060689da1..1e718a2c4 100644 --- a/management/internals/shared/grpc/proxy_test.go +++ b/management/internals/shared/grpc/proxy_test.go @@ -313,7 +313,7 @@ func TestValidateState_RejectsOldTwoPartFormat(t *testing.T) { err := s.singleUseStore.Store("base64url|hmac", "test", 10*time.Minute) require.NoError(t, err) - _, _, err = s.ValidateState("base64url|hmac") + _, _, _, err = s.ValidateState("base64url|hmac") require.Error(t, err) assert.Contains(t, err.Error(), "invalid state format") } @@ -385,11 +385,35 @@ func TestValidateState_RejectsInvalidHMAC(t *testing.T) { err := s.singleUseStore.Store("dGVzdA==|nonce|wrong-hmac", "test", 10*time.Minute) require.NoError(t, err) - _, _, err = s.ValidateState("dGVzdA==|nonce|wrong-hmac") + _, _, _, err = s.ValidateState("dGVzdA==|nonce|wrong-hmac") require.Error(t, err) assert.Contains(t, err.Error(), "invalid state signature") } +func TestSessionCodeCannotConsumeOIDCState(t *testing.T) { + const verifier = "pkce-verifier" + + store := NewSingleUseStore(context.Background(), testCacheStore(t)) + server := &ProxyServiceServer{ + oidcConfig: ProxyOIDCConfig{ + HMACKey: []byte("test-hmac-key"), + }, + singleUseStore: store, + } + state := generateState(server, "https://service.example.com/callback") + require.NoError(t, store.Store(state, verifier, time.Minute)) + + response, err := server.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + SessionCode: state, + }) + require.NoError(t, err) + assert.False(t, response.GetValid()) + + gotVerifier, _, _, err := server.ValidateState(state) + require.NoError(t, err) + assert.Equal(t, verifier, gotVerifier) +} + func TestSendServiceUpdateToCluster_FiltersOnCapability(t *testing.T) { tokenStore := NewOneTimeTokenStore(context.Background(), testCacheStore(t)) diff --git a/management/internals/shared/grpc/single_use.go b/management/internals/shared/grpc/single_use.go index 1698707d7..467ee6020 100644 --- a/management/internals/shared/grpc/single_use.go +++ b/management/internals/shared/grpc/single_use.go @@ -2,6 +2,8 @@ package grpc import ( "context" + "crypto/rand" + "encoding/base64" "fmt" "time" @@ -33,6 +35,24 @@ func (s *SingleUseStore) Store(key, value string, ttl time.Duration) error { return nil } +// Generate stores a value under a namespaced random key and returns the random key. +func (s *SingleUseStore) Generate(namespace, value string, ttl time.Duration) (string, error) { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return "", fmt.Errorf("generate single-use key: %w", err) + } + + key := base64.RawURLEncoding.EncodeToString(buf) + if err := s.Store(singleUseCacheKey(namespace, key), value, ttl); err != nil { + return "", err + } + return key, nil +} + +func singleUseCacheKey(namespace, key string) string { + return namespace + ":" + key +} + // LoadAndDelete retrieves and removes the value for a key. func (s *SingleUseStore) LoadAndDelete(key string) (string, bool) { value, found, err := s.cache.GetDel(s.ctx, key) diff --git a/management/internals/shared/grpc/single_use_test.go b/management/internals/shared/grpc/single_use_test.go index 357046720..8243ca43b 100644 --- a/management/internals/shared/grpc/single_use_test.go +++ b/management/internals/shared/grpc/single_use_test.go @@ -83,3 +83,40 @@ func TestSingleUseStoreLoadAndDelete(t *testing.T) { } }) } + +func TestSingleUseStore_GenerateAndConsumeOnce(t *testing.T) { + const namespace = "test" + s := NewSingleUseStore(context.Background(), testCacheStore(t)) + + key, err := s.Generate(namespace, "the-value", time.Minute) + if err != nil { + t.Fatalf("generate: %v", err) + } + if key == "" || key == "the-value" { + t.Fatalf("unexpected key %q", key) + } + + value, found := s.LoadAndDelete(singleUseCacheKey(namespace, key)) + if !found || value != "the-value" { + t.Fatalf("expected to load the stored value, got %q found=%v", value, found) + } + + if _, found := s.LoadAndDelete(singleUseCacheKey(namespace, key)); found { + t.Fatal("value must be consumed on first LoadAndDelete") + } +} + +func TestSingleUseStore_GenerateUniqueKeys(t *testing.T) { + s := NewSingleUseStore(context.Background(), testCacheStore(t)) + a, err := s.Generate("test", "v", time.Minute) + if err != nil { + t.Fatalf("generate a: %v", err) + } + b, err := s.Generate("test", "v", time.Minute) + if err != nil { + t.Fatalf("generate b: %v", err) + } + if a == b { + t.Fatal("generated keys must be distinct") + } +} diff --git a/management/internals/shared/grpc/validate_session_test.go b/management/internals/shared/grpc/validate_session_test.go index 2af250a85..4b36e74cf 100644 --- a/management/internals/shared/grpc/validate_session_test.go +++ b/management/internals/shared/grpc/validate_session_test.go @@ -634,6 +634,10 @@ func (m *testValidateSessionProxyManager) ClusterSupportsPrivate(_ context.Conte return nil } +func (m *testValidateSessionProxyManager) ClusterSupportsSessionCode(_ context.Context, _ string) bool { + return false +} + type testValidateSessionUsersManager struct { store store.Store } @@ -662,3 +666,47 @@ func (m *testValidateSessionUsersManager) GetUserWithGroups(ctx context.Context, } return user, groups, nil } + +func TestValidateSession_RedeemsSessionCode(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + proxy, err := setup.store.GetServiceByID(context.Background(), store.LockingStrengthNone, "testAccountId", "testProxyId") + require.NoError(t, err) + + token := createSessionToken(t, proxy.SessionPrivateKey, "allowedUserId", "test-proxy.example.com") + code, ok := setup.proxyService.GenerateSessionCode(token) + require.True(t, ok) + require.NotEqual(t, token, code, "code must not be the token itself") + + resp, err := setup.proxyService.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + Domain: "test-proxy.example.com", + SessionCode: code, + }) + require.NoError(t, err) + assert.True(t, resp.Valid, "redeemed code should authorize the user") + assert.Equal(t, "allowedUserId", resp.UserId) + assert.Equal(t, token, resp.GetSessionToken(), "response must carry the durable token for the cookie") + + // Single-use: the same code must not redeem again. + resp2, err := setup.proxyService.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + Domain: "test-proxy.example.com", + SessionCode: code, + }) + require.NoError(t, err) + assert.False(t, resp2.Valid, "a consumed code must be rejected") + assert.Empty(t, resp2.GetSessionToken()) +} + +func TestValidateSession_InvalidSessionCode(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + resp, err := setup.proxyService.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + Domain: "test-proxy.example.com", + SessionCode: "does-not-exist", + }) + require.NoError(t, err) + assert.False(t, resp.Valid) + assert.Empty(t, resp.GetSessionToken()) +} diff --git a/management/server/http/handlers/proxy/auth.go b/management/server/http/handlers/proxy/auth.go index 15327fd30..298fb503e 100644 --- a/management/server/http/handlers/proxy/auth.go +++ b/management/server/http/handlers/proxy/auth.go @@ -59,7 +59,7 @@ func (h *AuthCallbackHandler) handleCallback(w http.ResponseWriter, r *http.Requ state := r.URL.Query().Get("state") - codeVerifier, originalURL, err := h.proxyService.ValidateState(state) + codeVerifier, originalURL, useSessionCode, err := h.proxyService.ValidateState(state) if err != nil { log.WithError(err).Error("OAuth callback state validation failed") http.Error(w, "Invalid state parameter", http.StatusBadRequest) @@ -119,10 +119,19 @@ func (h *AuthCallbackHandler) handleCallback(w http.ResponseWriter, r *http.Requ redirectURL.Scheme = "https" query := redirectURL.Query() - query.Set("session_token", sessionToken) + if useSessionCode { + code, ok := h.proxyService.GenerateSessionCode(sessionToken) + if !ok { + http.Error(w, "Failed to create session", http.StatusInternalServerError) + return + } + query.Set("session_code", code) + } else { + query.Set("session_token", sessionToken) + } redirectURL.RawQuery = query.Encode() - log.WithField("redirect", redirectURL.Host).Debug("OAuth callback: redirecting user with session token") + log.WithField("redirect", redirectURL.Host).Debug("OAuth callback: redirecting user to proxy") http.Redirect(w, r, redirectURL.String(), http.StatusFound) } diff --git a/management/server/http/handlers/proxy/auth_callback_integration_test.go b/management/server/http/handlers/proxy/auth_callback_integration_test.go index 41ecb5f23..964841a63 100644 --- a/management/server/http/handlers/proxy/auth_callback_integration_test.go +++ b/management/server/http/handlers/proxy/auth_callback_integration_test.go @@ -181,6 +181,10 @@ func (m *testAccessLogManager) GetAllAccessLogs(_ context.Context, _, _ string, } func setupAuthCallbackTest(t *testing.T) *testSetup { + return setupAuthCallbackTestWithProxyManager(t, testSessionCodeManager{}) +} + +func setupAuthCallbackTestWithProxyManager(t *testing.T, proxyManager nbproxy.Manager) *testSetup { t.Helper() ctx := context.Background() @@ -217,7 +221,7 @@ func setupAuthCallbackTest(t *testing.T) *testSetup { nil, usersManager, nil, - nil, + proxyManager, nil, ) @@ -242,6 +246,15 @@ func setupAuthCallbackTest(t *testing.T) *testSetup { } } +type testSessionCodeManager struct { + nbproxy.Manager + supported bool +} + +func (m testSessionCodeManager) ClusterSupportsSessionCode(_ context.Context, _ string) bool { + return m.supported +} + func createTestReverseProxies(t *testing.T, ctx context.Context, testStore store.Store) { t.Helper() @@ -252,10 +265,11 @@ func createTestReverseProxies(t *testing.T, ctx context.Context, testStore store privKey := base64.StdEncoding.EncodeToString(priv) testProxy := &service.Service{ - ID: "testProxyId", - AccountID: "testAccountId", - Name: "Test Proxy", - Domain: "test-proxy.example.com", + ID: "testProxyId", + AccountID: "testAccountId", + Name: "Test Proxy", + Domain: "test-proxy.example.com", + ProxyCluster: "cluster.example.com", Targets: []*service.Target{{ Path: strPtr("/"), Host: "localhost", @@ -512,29 +526,56 @@ func createTestState(t *testing.T, ps *nbgrpc.ProxyServiceServer, redirectURL st } func TestAuthCallback_UserAllowedToLogin(t *testing.T) { - setup := setupAuthCallbackTest(t) - defer setup.cleanup() + tests := []struct { + name string + manager nbproxy.Manager + wantParam string + absentParam string + }{ + {name: "legacy proxy", manager: testSessionCodeManager{}, wantParam: "session_token", absentParam: "session_code"}, + {name: "compatible proxy", manager: testSessionCodeManager{supported: true}, wantParam: "session_code", absentParam: "session_token"}, + } - setup.oidcServer.tokenSubject = "allowedUserId" + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setup := setupAuthCallbackTestWithProxyManager(t, tt.manager) + defer setup.cleanup() - state := createTestState(t, setup.proxyService, "https://test-proxy.example.com/dashboard") + setup.oidcServer.tokenSubject = "allowedUserId" + state := createTestState(t, setup.proxyService, "https://test-proxy.example.com/dashboard") + req := httptest.NewRequest(http.MethodGet, "/reverse-proxy/callback?code=test-auth-code&state="+url.QueryEscape(state), nil) + rec := httptest.NewRecorder() + setup.router.ServeHTTP(rec, req) + require.Equal(t, http.StatusFound, rec.Code) - req := httptest.NewRequest(http.MethodGet, "/reverse-proxy/callback?code=test-auth-code&state="+url.QueryEscape(state), nil) - rec := httptest.NewRecorder() + location, err := url.Parse(rec.Header().Get("Location")) + require.NoError(t, err) + require.Equal(t, "test-proxy.example.com", location.Host) + require.NotEmpty(t, location.Query().Get(tt.wantParam)) + require.Empty(t, location.Query().Get(tt.absentParam)) + require.Empty(t, location.Query().Get("error")) - setup.router.ServeHTTP(rec, req) + if tt.wantParam == "session_code" { + code := location.Query().Get("session_code") + response, err := setup.proxyService.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + Domain: location.Hostname(), + SessionCode: code, + }) + require.NoError(t, err) + require.True(t, response.GetValid()) + require.NotEmpty(t, response.GetSessionToken()) + require.NotEqual(t, code, response.GetSessionToken()) - require.Equal(t, http.StatusFound, rec.Code) - - location := rec.Header().Get("Location") - require.NotEmpty(t, location) - - parsedLocation, err := url.Parse(location) - require.NoError(t, err) - - require.Equal(t, "test-proxy.example.com", parsedLocation.Host) - require.NotEmpty(t, parsedLocation.Query().Get("session_token"), "Should include session token") - require.Empty(t, parsedLocation.Query().Get("error"), "Should not have error parameter") + replayed, err := setup.proxyService.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + Domain: location.Hostname(), + SessionCode: code, + }) + require.NoError(t, err) + require.False(t, replayed.GetValid()) + require.Empty(t, replayed.GetSessionToken()) + } + }) + } } // TestAuthCallback_UserDeniedByAccountStatus asserts that a user whose account diff --git a/management/server/store/sql_store_proxy.go b/management/server/store/sql_store_proxy.go index 5d1a9afba..58fa86468 100644 --- a/management/server/store/sql_store_proxy.go +++ b/management/server/store/sql_store_proxy.go @@ -367,6 +367,21 @@ func (s *SqlStore) GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr s return s.getClusterUnanimousCapability(ctx, clusterAddr, "supports_crowdsec") } +// GetActiveProxyVersions returns every active proxy version in a cluster. +func (s *SqlStore) GetActiveProxyVersions(ctx context.Context, clusterAddr string) ([]string, error) { + var versions []string + err := s.db.WithContext(ctx). + Model(&proxy.Proxy{}). + Where("LOWER(cluster_address) = LOWER(?) AND status = ? AND last_seen > ?", + clusterAddr, proxy.StatusConnected, time.Now().Add(-proxyActiveThreshold)). + Pluck("version", &versions).Error + if err != nil { + log.WithContext(ctx).Errorf("failed to get active proxy versions for %s: %v", clusterAddr, err) + return nil, status.Errorf(status.Internal, "get active proxy versions") + } + return versions, nil +} + // getClusterUnanimousCapability returns an aggregated boolean capability // requiring all active proxies in the cluster to report true. func (s *SqlStore) getClusterUnanimousCapability(ctx context.Context, clusterAddr, column string) *bool { diff --git a/management/server/store/store.go b/management/server/store/store.go index e935d2d88..bd55e3176 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -337,6 +337,7 @@ type Store interface { GetClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool GetClusterSupportsPrivate(ctx context.Context, clusterAddr string) *bool + GetActiveProxyVersions(ctx context.Context, clusterAddr string) ([]string, error) CleanupStaleProxies(ctx context.Context, inactivityDuration time.Duration) error GetAllProxies(ctx context.Context) ([]*proxy.Proxy, error) DisconnectAllProxies(ctx context.Context) (int64, error) diff --git a/management/server/store/store_mock.go b/management/server/store/store_mock.go index 068ac6ff2..4dd7f9286 100644 --- a/management/server/store/store_mock.go +++ b/management/server/store/store_mock.go @@ -1539,6 +1539,21 @@ func (mr *MockStoreMockRecorder) GetActiveProxyClusterAddressesForAccount(ctx, a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveProxyClusterAddressesForAccount", reflect.TypeOf((*MockStore)(nil).GetActiveProxyClusterAddressesForAccount), ctx, accountID) } +// GetActiveProxyVersions mocks base method. +func (m *MockStore) GetActiveProxyVersions(ctx context.Context, clusterAddr string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetActiveProxyVersions", ctx, clusterAddr) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetActiveProxyVersions indicates an expected call of GetActiveProxyVersions. +func (mr *MockStoreMockRecorder) GetActiveProxyVersions(ctx, clusterAddr any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveProxyVersions", reflect.TypeOf((*MockStore)(nil).GetActiveProxyVersions), ctx, clusterAddr) +} + // GetAgentNetworkAccessLogSessions mocks base method. func (m *MockStore) GetAgentNetworkAccessLogSessions(ctx context.Context, lockStrength LockingStrength, accountID string, filter types.AgentNetworkAccessLogFilter) ([]*types.AgentNetworkAccessLogSession, int64, error) { m.ctrl.T.Helper() diff --git a/proxy/internal/auth/middleware.go b/proxy/internal/auth/middleware.go index 311dd2cbb..672286748 100644 --- a/proxy/internal/auth/middleware.go +++ b/proxy/internal/auth/middleware.go @@ -46,7 +46,7 @@ type Scheme interface { // an authenticated user. An empty token indicates an unauthenticated // request; optionally, promptData may be returned for the login UI. // An error indicates an infrastructure failure (e.g. gRPC unavailable). - Authenticate(*http.Request) (token string, promptData string, err error) + Authenticate(*http.Request) (token, promptData string, err error) } // DomainConfig holds the authentication and restriction settings for a protected domain. @@ -77,6 +77,8 @@ type validationResult struct { // Groups for tokens minted before names were embedded; the consumer // falls back to ids for missing positions. GroupNames []string + // MintedToken is the session token issued when a one-time code is redeemed. + MintedToken string } // Middleware applies per-domain authentication and IP restriction checks. @@ -581,7 +583,8 @@ func (mw *Middleware) authenticateWithSchemes(w http.ResponseWriter, r *http.Req // handleAuthenticatedToken validates the token, handles denied access, and on // success sets a session cookie and redirects to the original URL. func (mw *Middleware) handleAuthenticatedToken(w http.ResponseWriter, r *http.Request, host, token string, config DomainConfig, scheme Scheme) { - result, err := mw.validateSessionToken(r.Context(), host, token, config.SessionPublicKey, scheme.Type()) + isCode := scheme.Type() == auth.MethodOIDC && r.URL.Query().Get("session_code") != "" + result, err := mw.validateSessionToken(r.Context(), host, token, isCode, config.SessionPublicKey, scheme.Type()) if err != nil { if cd := proxy.CapturedDataFromContext(r.Context()); cd != nil { cd.SetOrigin(proxy.OriginAuth) @@ -612,7 +615,13 @@ func (mw *Middleware) handleAuthenticatedToken(w http.ResponseWriter, r *http.Re return } - setSessionCookie(w, token, config.SessionExpiration) + // When a code was redeemed, the cookie must hold the durable token the + // server returned, not the single-use code. + cookieValue := token + if result.MintedToken != "" { + cookieValue = result.MintedToken + } + setSessionCookie(w, cookieValue, config.SessionExpiration) // Redirect instead of forwarding the auth POST to the backend. // The browser will follow with a GET carrying the new session cookie. @@ -652,7 +661,7 @@ func wasCredentialSubmitted(r *http.Request, method auth.Method) bool { case auth.MethodPassword: return credentialFormValue(r, passwordFormId) != "" case auth.MethodOIDC: - return r.URL.Query().Get("session_token") != "" + return r.URL.Query().Get("session_token") != "" || r.URL.Query().Get("session_code") != "" } return false } @@ -706,12 +715,15 @@ func (mw *Middleware) RemoveDomain(domain string) { // validateSessionToken validates a session token. OIDC tokens with a configured // validator go through gRPC for group access checks; other methods validate locally. -func (mw *Middleware) validateSessionToken(ctx context.Context, host, token string, publicKey ed25519.PublicKey, method auth.Method) (*validationResult, error) { +func (mw *Middleware) validateSessionToken(ctx context.Context, host, token string, isCode bool, publicKey ed25519.PublicKey, method auth.Method) (*validationResult, error) { if method == auth.MethodOIDC && mw.sessionValidator != nil { - resp, err := mw.sessionValidator.ValidateSession(ctx, &proto.ValidateSessionRequest{ - Domain: host, - SessionToken: token, - }) + req := &proto.ValidateSessionRequest{Domain: host} + if isCode { + req.SessionCode = token + } else { + req.SessionToken = token //nolint:staticcheck + } + resp, err := mw.sessionValidator.ValidateSession(ctx, req) if err != nil { return nil, fmt.Errorf("%w: %w", errValidationUnavailable, err) } @@ -729,11 +741,12 @@ func (mw *Middleware) validateSessionToken(ctx context.Context, host, token stri }, nil } return &validationResult{ - UserID: resp.UserId, - UserEmail: resp.GetUserEmail(), - Valid: true, - Groups: resp.GetPeerGroupIds(), - GroupNames: resp.GetPeerGroupNames(), + UserID: resp.UserId, + UserEmail: resp.GetUserEmail(), + Valid: true, + Groups: resp.GetPeerGroupIds(), + GroupNames: resp.GetPeerGroupNames(), + MintedToken: resp.GetSessionToken(), }, nil } @@ -788,14 +801,16 @@ func sessionGroupsAllowed(allowed map[string]struct{}, method auth.Method, group } } -// stripSessionTokenParam returns the request URI with the session_token query -// parameter removed so it doesn't linger in the browser's address bar or history. +// stripSessionTokenParam returns the request URI with the session hand-off +// query parameters removed so they don't linger in the browser's address bar +// or history. func stripSessionTokenParam(u *url.URL) string { q := u.Query() - if !q.Has("session_token") { + if !q.Has("session_token") && !q.Has("session_code") { return u.RequestURI() } q.Del("session_token") + q.Del("session_code") clean := *u clean.RawQuery = q.Encode() return clean.RequestURI() diff --git a/proxy/internal/auth/middleware_test.go b/proxy/internal/auth/middleware_test.go index f1242f95e..88c900f97 100644 --- a/proxy/internal/auth/middleware_test.go +++ b/proxy/internal/auth/middleware_test.go @@ -783,6 +783,12 @@ func TestWasCredentialSubmitted(t *testing.T) { query: url.Values{"session_token": {"abc123"}}, expected: true, }, + { + name: "OIDC code in query", + method: auth.MethodOIDC, + query: url.Values{"session_code": {"abc123"}}, + expected: true, + }, { name: "OIDC token not in query", method: auth.MethodOIDC, @@ -1571,3 +1577,23 @@ func TestProtect_TunnelPeerFastPath_TakesPathWithInboundMarker(t *testing.T) { assert.Equal(t, http.StatusOK, rec.Code, "a successful tunnel-peer validation must forward to the next handler") } + +func TestStripSessionTokenParam(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"strips session_token", "https://ex.com/p?a=1&session_token=tok", "/p?a=1"}, + {"strips session_code", "https://ex.com/p?a=1&session_code=code", "/p?a=1"}, + {"strips both", "https://ex.com/p?session_token=tok&session_code=code&a=1", "/p?a=1"}, + {"no-op when absent", "https://ex.com/p?a=1", "/p?a=1"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + u, err := url.Parse(tc.in) + require.NoError(t, err) + assert.Equal(t, tc.want, stripSessionTokenParam(u)) + }) + } +} diff --git a/proxy/internal/auth/oidc.go b/proxy/internal/auth/oidc.go index a60e6437a..739777924 100644 --- a/proxy/internal/auth/oidc.go +++ b/proxy/internal/auth/oidc.go @@ -40,9 +40,14 @@ func (OIDC) Type() auth.Method { // Authenticate checks for an OIDC session token or obtains the OIDC redirect URL. func (o OIDC) Authenticate(r *http.Request) (string, string, error) { - // Check for the session_token query param (from OIDC redirects). - // The management server passes the token in the URL because it cannot set - // cookies for the proxy's domain (cookies are domain-scoped per RFC 6265). + // Check for the session credential returned by the OIDC callback. The management + // server passes it in the URL because it cannot set a cookie for the proxy's + // domain (cookies are domain-scoped per RFC 6265). The current flow uses a + // single-use session_code to keep the durable token out of the URL. + // session_token remains supported for backward compatibility. + if code := r.URL.Query().Get("session_code"); code != "" { + return code, "", nil + } if token := r.URL.Query().Get("session_token"); token != "" { return token, "", nil } diff --git a/proxy/internal/proxy/reverseproxy.go b/proxy/internal/proxy/reverseproxy.go index 7c9e21261..7b0acd1b3 100644 --- a/proxy/internal/proxy/reverseproxy.go +++ b/proxy/internal/proxy/reverseproxy.go @@ -721,12 +721,13 @@ func stripSessionCookie(r *httputil.ProxyRequest) { } } -// stripSessionTokenQuery removes the OIDC session_token query parameter from -// the outgoing URL to prevent credential leakage to backends. +// stripSessionTokenQuery removes the OIDC session hand-off query parameters +// from the outgoing URL to prevent credential leakage to backends. func stripSessionTokenQuery(r *httputil.ProxyRequest) { q := r.Out.URL.Query() - if q.Has("session_token") { + if q.Has("session_token") || q.Has("session_code") { q.Del("session_token") + q.Del("session_code") r.Out.URL.RawQuery = q.Encode() } } diff --git a/proxy/management_integration_test.go b/proxy/management_integration_test.go index befd1f1ee..000d8ce72 100644 --- a/proxy/management_integration_test.go +++ b/proxy/management_integration_test.go @@ -131,7 +131,7 @@ func setupIntegrationTest(t *testing.T) *integrationTestSetup { HMACKey: []byte("test-hmac-key"), } - proxyManager := &testProxyManager{} + proxyManager := &testProxyManager{supportsSessionCode: true} proxyService := nbgrpc.NewProxyServiceServer( &testAccessLogManager{}, @@ -202,7 +202,9 @@ func (m *testAccessLogManager) GetAllAccessLogs(_ context.Context, _, _ string, } // testProxyManager is a mock implementation of proxy.Manager for testing. -type testProxyManager struct{} +type testProxyManager struct { + supportsSessionCode bool +} func (m *testProxyManager) Connect(_ context.Context, proxyID, sessionID, _, _, _ string, _ *string, _ *nbproxy.Capabilities) (*nbproxy.Proxy, error) { return &nbproxy.Proxy{ID: proxyID, SessionID: sessionID, Status: nbproxy.StatusConnected}, nil @@ -244,6 +246,10 @@ func (m *testProxyManager) ClusterSupportsPrivate(_ context.Context, _ string) * return nil } +func (m *testProxyManager) ClusterSupportsSessionCode(_ context.Context, _ string) bool { + return m.supportsSessionCode +} + func (m *testProxyManager) CleanupStale(_ context.Context, _ time.Duration) error { return nil } diff --git a/proxy/web/web.go b/proxy/web/web.go index 6773a9c1a..a45fc8730 100644 --- a/proxy/web/web.go +++ b/proxy/web/web.go @@ -181,6 +181,7 @@ func ServeAccessDeniedPage(w http.ResponseWriter, r *http.Request, code int, tit func stripAuthParams(u *url.URL) string { q := u.Query() q.Del("session_token") + q.Del("session_code") q.Del("error") q.Del("error_description") clean := *u diff --git a/shared/management/proto/proxy_service.pb.go b/shared/management/proto/proxy_service.pb.go index 496774a4b..09dff0d36 100644 --- a/shared/management/proto/proxy_service.pb.go +++ b/shared/management/proto/proxy_service.pb.go @@ -2265,8 +2265,11 @@ type ValidateSessionRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Domain string `protobuf:"bytes,1,opt,name=domain,proto3" json:"domain,omitempty"` + Domain string `protobuf:"bytes,1,opt,name=domain,proto3" json:"domain,omitempty"` + // Deprecated: Do not use. SessionToken string `protobuf:"bytes,2,opt,name=session_token,json=sessionToken,proto3" json:"session_token,omitempty"` + // session_code is a short-lived, single-use code exchanged for a session token. + SessionCode string `protobuf:"bytes,3,opt,name=session_code,json=sessionCode,proto3" json:"session_code,omitempty"` } func (x *ValidateSessionRequest) Reset() { @@ -2308,6 +2311,7 @@ func (x *ValidateSessionRequest) GetDomain() string { return "" } +// Deprecated: Do not use. func (x *ValidateSessionRequest) GetSessionToken() string { if x != nil { return x.SessionToken @@ -2315,6 +2319,13 @@ func (x *ValidateSessionRequest) GetSessionToken() string { return "" } +func (x *ValidateSessionRequest) GetSessionCode() string { + if x != nil { + return x.SessionCode + } + return "" +} + type ValidateSessionResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2333,6 +2344,8 @@ type ValidateSessionResponse struct { // Stamped onto upstream requests as X-NetBird-Groups so downstream // services can read names rather than opaque ids. PeerGroupNames []string `protobuf:"bytes,6,rep,name=peer_group_names,json=peerGroupNames,proto3" json:"peer_group_names,omitempty"` + // session_token contains the durable token issued when session_code is redeemed. + SessionToken string `protobuf:"bytes,7,opt,name=session_token,json=sessionToken,proto3" json:"session_token,omitempty"` } func (x *ValidateSessionResponse) Reset() { @@ -2409,6 +2422,13 @@ func (x *ValidateSessionResponse) GetPeerGroupNames() []string { return nil } +func (x *ValidateSessionResponse) GetSessionToken() string { + if x != nil { + return x.SessionToken + } + return "" +} + // ValidateTunnelPeerRequest carries the inbound peer's tunnel IP and the // service domain whose group requirements should gate access. The calling // account is inferred from the proxy's gRPC metadata (ProxyToken). @@ -3519,223 +3539,228 @@ var file_proxy_service_proto_rawDesc = []byte{ 0x09, 0x52, 0x0b, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x72, 0x6c, 0x22, 0x26, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x55, 0x0a, 0x16, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x7c, 0x0a, 0x16, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xdc, 0x01, - 0x0a, 0x17, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, + 0x02, 0x18, 0x01, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x43, 0x6f, 0x64, 0x65, 0x22, 0x81, 0x02, 0x0a, 0x17, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, + 0x1d, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x23, + 0x0a, 0x0d, 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x52, 0x65, 0x61, + 0x73, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, + 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x65, 0x65, + 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x65, 0x65, + 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x06, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x50, 0x0a, 0x19, 0x56, 0x61, 0x6c, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, + 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x75, 0x6e, 0x6e, 0x65, 0x6c, + 0x49, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x84, 0x02, 0x0a, 0x1a, 0x56, + 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, - 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0e, - 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, - 0x64, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x65, - 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x50, 0x0a, 0x19, - 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x75, 0x6e, - 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x75, - 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x22, 0x84, - 0x02, 0x0a, 0x1a, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, - 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, - 0x75, 0x73, 0x65, 0x72, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x64, - 0x65, 0x6e, 0x69, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, - 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, - 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, - 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x70, - 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, - 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x13, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, - 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x32, 0x0a, - 0x04, 0x69, 0x6e, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, - 0x70, 0x69, 0x6e, 0x67, 0x73, 0x49, 0x6e, 0x69, 0x74, 0x48, 0x00, 0x52, 0x04, 0x69, 0x6e, 0x69, - 0x74, 0x12, 0x2f, 0x0a, 0x03, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, - 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x03, 0x61, - 0x63, 0x6b, 0x42, 0x05, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x22, 0xdf, 0x01, 0x0a, 0x10, 0x53, 0x79, - 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x19, - 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x18, - 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x41, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, - 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x78, - 0x79, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x0c, 0x63, - 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x53, - 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x41, 0x63, 0x6b, 0x22, 0x7e, - 0x0a, 0x14, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x52, 0x07, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x6e, - 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, - 0x65, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e, 0x69, 0x74, 0x69, - 0x61, 0x6c, 0x53, 0x79, 0x6e, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x22, 0xa9, - 0x01, 0x0a, 0x1b, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, - 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, - 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, - 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, - 0x49, 0x64, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0xff, 0x01, 0x0a, 0x1c, 0x43, + 0x64, 0x65, 0x6e, 0x69, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, + 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, + 0x6e, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, + 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x47, + 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x65, 0x65, 0x72, 0x5f, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, + 0x73, 0x22, 0x81, 0x01, 0x0a, 0x13, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x32, 0x0a, 0x04, 0x69, 0x6e, 0x69, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, + 0x73, 0x49, 0x6e, 0x69, 0x74, 0x48, 0x00, 0x52, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x12, 0x2f, 0x0a, + 0x03, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, + 0x69, 0x6e, 0x67, 0x73, 0x41, 0x63, 0x6b, 0x48, 0x00, 0x52, 0x03, 0x61, 0x63, 0x6b, 0x42, 0x05, + 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x22, 0xdf, 0x01, 0x0a, 0x10, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x70, 0x72, + 0x6f, 0x78, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, + 0x6f, 0x78, 0x79, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x12, 0x41, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x43, 0x61, 0x70, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x4d, + 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x41, 0x63, 0x6b, 0x22, 0x7e, 0x0a, 0x14, 0x53, 0x79, + 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x07, 0x6d, + 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, + 0x6c, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x53, 0x79, + 0x6e, 0x63, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x22, 0xa9, 0x01, 0x0a, 0x1b, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, - 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x64, - 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, - 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x6c, - 0x69, 0x63, 0x79, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x0d, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x1b, - 0x0a, 0x09, 0x64, 0x65, 0x6e, 0x79, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x64, 0x65, 0x6e, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, - 0x65, 0x6e, 0x79, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x64, 0x65, 0x6e, 0x79, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x91, 0x02, 0x0a, - 0x15, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x19, - 0x0a, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x6e, - 0x64, 0x6f, 0x77, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0d, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, - 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x49, 0x6e, - 0x70, 0x75, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x6f, 0x75, - 0x74, 0x70, 0x75, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x73, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6f, 0x73, 0x74, - 0x5f, 0x75, 0x73, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x63, 0x6f, 0x73, 0x74, - 0x55, 0x73, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, - 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, - 0x22, 0x18, 0x0a, 0x16, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, - 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x64, 0x0a, 0x16, 0x50, 0x72, - 0x6f, 0x78, 0x79, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, - 0x14, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x4f, 0x44, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x02, - 0x2a, 0x46, 0x0a, 0x0f, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x4d, - 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x52, 0x45, 0x57, 0x52, - 0x49, 0x54, 0x45, 0x5f, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x10, 0x00, 0x12, 0x19, 0x0a, - 0x15, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x52, 0x45, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x50, 0x52, - 0x45, 0x53, 0x45, 0x52, 0x56, 0x45, 0x10, 0x01, 0x2a, 0x90, 0x01, 0x0a, 0x0e, 0x4d, 0x69, 0x64, - 0x64, 0x6c, 0x65, 0x77, 0x61, 0x72, 0x65, 0x53, 0x6c, 0x6f, 0x74, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, - 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, 0x55, - 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, - 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, - 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, - 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, - 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x10, 0x02, 0x12, 0x1c, 0x0a, - 0x18, 0x4d, 0x49, 0x44, 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, - 0x5f, 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0xc8, 0x01, 0x0a, 0x0b, - 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x14, 0x50, - 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, - 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, 0x23, - 0x0a, 0x1f, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x54, - 0x55, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, - 0x44, 0x10, 0x02, 0x12, 0x24, 0x0a, 0x20, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x55, 0x53, 0x5f, 0x43, 0x45, 0x52, 0x54, 0x49, 0x46, 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, - 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x23, 0x0a, 0x1f, 0x50, 0x52, 0x4f, - 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x45, 0x52, 0x54, 0x49, 0x46, - 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x16, - 0x0a, 0x12, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, - 0x52, 0x52, 0x4f, 0x52, 0x10, 0x05, 0x32, 0xfc, 0x07, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4d, 0x61, - 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x70, - 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, - 0x74, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x55, 0x0a, 0x0c, 0x53, 0x79, 0x6e, 0x63, - 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, - 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, - 0x6e, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, - 0x54, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, - 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, - 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, - 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, 0x65, 0x6e, 0x64, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, - 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x6e, + 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x12, + 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x64, + 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0xff, 0x01, 0x0a, 0x1c, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x63, 0x69, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x65, 0x63, 0x69, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, + 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x49, + 0x64, 0x12, 0x30, 0x0a, 0x14, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x12, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x73, 0x65, + 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x77, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x65, + 0x6e, 0x79, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, + 0x65, 0x6e, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x6e, 0x79, 0x5f, + 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x65, + 0x6e, 0x79, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x91, 0x02, 0x0a, 0x15, 0x52, 0x65, 0x63, + 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, + 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x67, 0x72, + 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x67, 0x72, + 0x6f, 0x75, 0x70, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, + 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x77, + 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x0a, 0x0c, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, + 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x4f, 0x75, + 0x74, 0x70, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6f, 0x73, 0x74, 0x5f, 0x75, 0x73, 0x64, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x63, 0x6f, 0x73, 0x74, 0x55, 0x73, 0x64, 0x12, + 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x08, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x73, 0x22, 0x18, 0x0a, 0x16, + 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x64, 0x0a, 0x16, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x4d, + 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x50, 0x44, + 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x46, 0x0a, 0x0f, + 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, + 0x18, 0x0a, 0x14, 0x50, 0x41, 0x54, 0x48, 0x5f, 0x52, 0x45, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, + 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x50, 0x41, 0x54, + 0x48, 0x5f, 0x52, 0x45, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x53, 0x45, 0x52, + 0x56, 0x45, 0x10, 0x01, 0x2a, 0x90, 0x01, 0x0a, 0x0e, 0x4d, 0x69, 0x64, 0x64, 0x6c, 0x65, 0x77, + 0x61, 0x72, 0x65, 0x53, 0x6c, 0x6f, 0x74, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x49, 0x44, 0x44, 0x4c, + 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x4d, 0x49, 0x44, 0x44, + 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, 0x4f, 0x4e, 0x5f, 0x52, + 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x49, 0x44, 0x44, + 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, 0x4f, 0x4e, 0x5f, 0x52, + 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x10, 0x02, 0x12, 0x1c, 0x0a, 0x18, 0x4d, 0x49, 0x44, + 0x44, 0x4c, 0x45, 0x57, 0x41, 0x52, 0x45, 0x5f, 0x53, 0x4c, 0x4f, 0x54, 0x5f, 0x54, 0x45, 0x52, + 0x4d, 0x49, 0x4e, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0xc8, 0x01, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, + 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x14, 0x50, 0x52, 0x4f, 0x58, 0x59, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, + 0x00, 0x12, 0x17, 0x0a, 0x13, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x50, 0x52, + 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x54, 0x55, 0x4e, 0x4e, 0x45, + 0x4c, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, + 0x24, 0x0a, 0x20, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x43, 0x45, 0x52, 0x54, 0x49, 0x46, 0x49, 0x43, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x45, 0x4e, 0x44, + 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x23, 0x0a, 0x1f, 0x50, 0x52, 0x4f, 0x58, 0x59, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x45, 0x52, 0x54, 0x49, 0x46, 0x49, 0x43, 0x41, 0x54, + 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x16, 0x0a, 0x12, 0x50, 0x52, + 0x4f, 0x58, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, + 0x10, 0x05, 0x32, 0xfc, 0x07, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x5f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x70, + 0x70, 0x69, 0x6e, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x30, 0x01, 0x12, 0x55, 0x0a, 0x0c, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, + 0x69, 0x6e, 0x67, 0x73, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x54, 0x0a, 0x0d, 0x53, + 0x65, 0x6e, 0x64, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x12, 0x20, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, + 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x41, + 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, + 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, - 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, - 0x4c, 0x12, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, - 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1e, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, - 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x5a, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x6f, 0x78, 0x79, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x4b, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, 0x43, 0x55, 0x52, 0x4c, 0x12, 0x1d, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x49, + 0x44, 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x49, 0x44, + 0x43, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, + 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, + 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x12, - 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, - 0x65, 0x72, 0x12, 0x25, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, - 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x69, 0x0a, 0x14, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, - 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x27, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, - 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, - 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, - 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x63, 0x6f, - 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, - 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x12, 0x56, 0x61, 0x6c, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x12, 0x25, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x75, 0x6e, 0x6e, 0x65, + 0x6c, 0x50, 0x65, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, + 0x14, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x27, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, + 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x4c, 0x4c, 0x4d, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x52, 0x65, 0x63, 0x6f, + 0x72, 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4c, 0x4c, + 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, + 0x64, 0x4c, 0x4c, 0x4d, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/shared/management/proto/proxy_service.proto b/shared/management/proto/proxy_service.proto index facadc4d5..7c5fb08eb 100644 --- a/shared/management/proto/proxy_service.proto +++ b/shared/management/proto/proxy_service.proto @@ -363,7 +363,9 @@ message GetOIDCURLResponse { message ValidateSessionRequest { string domain = 1; - string session_token = 2; + string session_token = 2 [deprecated = true]; + // session_code is a short-lived, single-use code exchanged for a session token. + string session_code = 3; } message ValidateSessionResponse { @@ -380,6 +382,8 @@ message ValidateSessionResponse { // Stamped onto upstream requests as X-NetBird-Groups so downstream // services can read names rather than opaque ids. repeated string peer_group_names = 6; + // session_token contains the durable token issued when session_code is redeemed. + string session_token = 7; } // ValidateTunnelPeerRequest carries the inbound peer's tunnel IP and the @@ -506,4 +510,3 @@ message RecordLLMUsageRequest { message RecordLLMUsageResponse { } - From 8edc120370e57bea07482412675ca9a2ac9bf808 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:41:49 +0900 Subject: [PATCH 64/75] [client] Replace the eBPF WireGuard proxy with loopback endpoint addressing (#7316) --- client/firewall/iptables/family_linux.go | 2 + client/firewall/iptables/manager_linux.go | 141 +-------- client/firewall/manager/firewall.go | 4 - client/firewall/nftables/manager_linux.go | 182 ----------- client/firewall/nftables/routing_linux.go | 2 +- client/firewall/uspfilter/filter.go | 6 - client/iface/iface.go | 7 - client/iface/iface_close_test.go | 1 - client/iface/wgproxy/ebpf/portlookup.go | 32 -- client/iface/wgproxy/ebpf/portlookup_test.go | 45 --- client/iface/wgproxy/ebpf/proxy.go | 243 --------------- client/iface/wgproxy/ebpf/proxy_test.go | 56 ---- client/iface/wgproxy/factory_kernel.go | 51 +-- client/iface/wgproxy/factory_usp.go | 5 - client/iface/wgproxy/loopback/addr.go | 70 +++++ client/iface/wgproxy/loopback/addr_test.go | 114 +++++++ client/iface/wgproxy/loopback/proxy.go | 291 ++++++++++++++++++ .../wgproxy/loopback/proxy_privileged_test.go | 196 ++++++++++++ .../wgproxy/{ebpf => loopback}/wrapper.go | 46 ++- client/iface/wgproxy/proxy_linux_test.go | 34 +- client/iface/wgproxy/redirect_test.go | 46 +-- client/internal/ebpf/ebpf/bpf_bpfeb.go | 148 --------- client/internal/ebpf/ebpf/bpf_bpfeb.o | Bin 8712 -> 0 bytes client/internal/ebpf/ebpf/bpf_bpfel.go | 148 --------- client/internal/ebpf/ebpf/bpf_bpfel.o | Bin 8712 -> 0 bytes client/internal/ebpf/ebpf/manager_linux.go | 115 ------- .../internal/ebpf/ebpf/manager_linux_test.go | 31 -- client/internal/ebpf/ebpf/src/bpf_map_def.h | 16 - client/internal/ebpf/ebpf/src/prog.c | 54 ---- client/internal/ebpf/ebpf/src/readme.md | 27 -- client/internal/ebpf/ebpf/src/wg_proxy.c | 60 ---- client/internal/ebpf/ebpf/wg_proxy_linux.go | 41 --- client/internal/ebpf/instantiater_linux.go | 15 - client/internal/ebpf/instantiater_nonlinux.go | 10 - client/internal/ebpf/manager/manager.go | 7 - client/internal/engine.go | 21 -- client/internal/engine_test.go | 7 - client/internal/iface_common.go | 1 - docs/testing-privileged.md | 3 +- go.mod | 2 +- go.sum | 2 - 41 files changed, 775 insertions(+), 1507 deletions(-) delete mode 100644 client/iface/wgproxy/ebpf/portlookup.go delete mode 100644 client/iface/wgproxy/ebpf/portlookup_test.go delete mode 100644 client/iface/wgproxy/ebpf/proxy.go delete mode 100644 client/iface/wgproxy/ebpf/proxy_test.go create mode 100644 client/iface/wgproxy/loopback/addr.go create mode 100644 client/iface/wgproxy/loopback/addr_test.go create mode 100644 client/iface/wgproxy/loopback/proxy.go create mode 100644 client/iface/wgproxy/loopback/proxy_privileged_test.go rename client/iface/wgproxy/{ebpf => loopback}/wrapper.go (84%) delete mode 100644 client/internal/ebpf/ebpf/bpf_bpfeb.go delete mode 100644 client/internal/ebpf/ebpf/bpf_bpfeb.o delete mode 100644 client/internal/ebpf/ebpf/bpf_bpfel.go delete mode 100644 client/internal/ebpf/ebpf/bpf_bpfel.o delete mode 100644 client/internal/ebpf/ebpf/manager_linux.go delete mode 100644 client/internal/ebpf/ebpf/manager_linux_test.go delete mode 100644 client/internal/ebpf/ebpf/src/bpf_map_def.h delete mode 100644 client/internal/ebpf/ebpf/src/prog.c delete mode 100644 client/internal/ebpf/ebpf/src/readme.md delete mode 100644 client/internal/ebpf/ebpf/src/wg_proxy.c delete mode 100644 client/internal/ebpf/ebpf/wg_proxy_linux.go delete mode 100644 client/internal/ebpf/instantiater_linux.go delete mode 100644 client/internal/ebpf/instantiater_nonlinux.go delete mode 100644 client/internal/ebpf/manager/manager.go diff --git a/client/firewall/iptables/family_linux.go b/client/firewall/iptables/family_linux.go index c5ed8cc20..0e1ce5440 100644 --- a/client/firewall/iptables/family_linux.go +++ b/client/firewall/iptables/family_linux.go @@ -24,6 +24,7 @@ const ( tableFilter = "filter" tableNat = "nat" tableMangle = "mangle" + tableRaw = "raw" // chainACLInput is the peer ACL chain that holds installed // peer-filtering rules. @@ -34,6 +35,7 @@ const ( mangleForwardKey chainKey = "MANGLE-FORWARD" chainInput = "INPUT" + chainOutput = "OUTPUT" chainPostrouting = "POSTROUTING" chainPrerouting = "PREROUTING" chainForward = "FORWARD" diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index 49b88f1ea..0f0b0110e 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -25,9 +25,8 @@ type Manager struct { wgIface iFaceMapper - ipv4Client *iptables.IPTables - family4 *family - rawSupported bool + ipv4Client *iptables.IPTables + family4 *family // IPv6 counterparts, nil when no v6 overlay ipv6Client *iptables.IPTables @@ -108,10 +107,6 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { return err } - if err := m.initNoTrackChain(); err != nil { - log.Warnf("raw table not available, notrack rules will be disabled: %v", err) - } - // Trust after all fatal init steps so a later failure doesn't leave the // interface in firewalld's trusted zone without a corresponding Close. if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { @@ -285,10 +280,6 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { var merr *multierror.Error - if err := m.cleanupNoTrackChain(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("cleanup notrack chain: %w", err)) - } - if m.hasIPv6() { if err := m.family6.Reset(); err != nil { merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err)) @@ -440,134 +431,6 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } -const ( - chainNameRaw = "NETBIRD-RAW" - chainOutput = "OUTPUT" - tableRaw = "raw" -) - -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which -// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark). -// -// Traffic flows that need NOTRACK: -// -// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite) -// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort -// Matched by: sport=wgPort -// -// 2. Egress: Proxy -> WireGuard (via raw socket) -// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 3. Ingress: Packets to WireGuard -// dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 4. Ingress: Packets to proxy (after eBPF rewrite) -// dst=127.0.0.1:proxyPort -// Matched by: dport=proxyPort -// -// Rules are cleaned up when the firewall manager is closed. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if !m.rawSupported { - return fmt.Errorf("raw table not available") - } - - wgPortStr := fmt.Sprintf("%d", wgPort) - proxyPortStr := fmt.Sprintf("%d", proxyPort) - - // Egress rules: match outgoing loopback UDP packets - outputRuleSport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--sport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleSport...); err != nil { - return fmt.Errorf("add output sport notrack rule: %w", err) - } - - outputRuleDport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleDport...); err != nil { - return fmt.Errorf("add output dport notrack rule: %w", err) - } - - // Ingress rules: match incoming loopback UDP packets - preroutingRuleWg := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleWg...); err != nil { - return fmt.Errorf("add prerouting wg notrack rule: %w", err) - } - - preroutingRuleProxy := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", proxyPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleProxy...); err != nil { - return fmt.Errorf("add prerouting proxy notrack rule: %w", err) - } - - log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort) - return nil -} - -func (m *Manager) initNoTrackChain() error { - if err := m.cleanupNoTrackChain(); err != nil { - log.Debugf("cleanup notrack chain: %v", err) - } - - if err := m.ipv4Client.NewChain(tableRaw, chainNameRaw); err != nil { - return fmt.Errorf("create chain: %w", err) - } - - jumpRule := []string{"-j", chainNameRaw} - - if err := m.ipv4Client.InsertUnique(tableRaw, chainOutput, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { - log.Debugf("delete orphan chain: %v", delErr) - } - return fmt.Errorf("add output jump rule: %w", err) - } - - if err := m.ipv4Client.InsertUnique(tableRaw, chainPrerouting, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); delErr != nil { - log.Debugf("delete output jump rule: %v", delErr) - } - if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { - log.Debugf("delete orphan chain: %v", delErr) - } - return fmt.Errorf("add prerouting jump rule: %w", err) - } - - m.rawSupported = true - return nil -} - -func (m *Manager) cleanupNoTrackChain() error { - exists, err := m.ipv4Client.ChainExists(tableRaw, chainNameRaw) - if err != nil { - if !m.rawSupported { - return nil - } - return fmt.Errorf("check chain exists: %w", err) - } - if !exists { - return nil - } - - jumpRule := []string{"-j", chainNameRaw} - - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); err != nil { - return fmt.Errorf("remove output jump rule: %w", err) - } - - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainPrerouting, jumpRule...); err != nil { - return fmt.Errorf("remove prerouting jump rule: %w", err) - } - - if err := m.ipv4Client.ClearAndDeleteChain(tableRaw, chainNameRaw); err != nil { - return fmt.Errorf("clear and delete chain: %w", err) - } - - m.rawSupported = false - return nil -} - func getConntrackEstablished() []string { return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"} } diff --git a/client/firewall/manager/firewall.go b/client/firewall/manager/firewall.go index 97a94d0f5..0eb376875 100644 --- a/client/firewall/manager/firewall.go +++ b/client/firewall/manager/firewall.go @@ -192,10 +192,6 @@ type Manager interface { // RemoveOutputDNAT removes an OUTPUT chain DNAT rule. RemoveOutputDNAT(localAddr netip.Addr, protocol Protocol, originalPort, translatedPort uint16) error - - // SetupEBPFProxyNoTrack creates static notrack rules for eBPF proxy loopback traffic. - // This prevents conntrack from interfering with WireGuard proxy communication. - SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error } // GenKey builds the rule id for this pair from the given format. diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index dbd5e4fa2..87651761f 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -12,7 +12,6 @@ import ( "github.com/google/nftables/expr" "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" nberrors "github.com/netbirdio/netbird/client/errors" firewall "github.com/netbirdio/netbird/client/firewall/manager" @@ -55,9 +54,6 @@ type Manager struct { // IPv6 counterpart, nil when no v6 overlay. family6 *family - notrackOutputChain *nftables.Chain - notrackPreroutingChain *nftables.Chain - extMonitor *externalChainMonitor } @@ -170,10 +166,6 @@ func (m *Manager) initFirewall() (err error) { } } - if err := m.initNoTrackChains(workTable); err != nil { - log.Warnf("raw priority chains not available, notrack rules will be disabled: %v", err) - } - return nil } @@ -455,10 +447,6 @@ func (m *Manager) Flush() error { } } - if err := m.refreshNoTrackChains(); err != nil { - log.Errorf("failed to refresh notrack chains: %v", err) - } - return nil } @@ -571,176 +559,6 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } -const ( - chainNameRawOutput = "netbird-raw-out" - chainNameRawPrerouting = "netbird-raw-pre" -) - -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which -// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark). -// -// Traffic flows that need NOTRACK: -// -// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite) -// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort -// Matched by: sport=wgPort -// -// 2. Egress: Proxy -> WireGuard (via raw socket) -// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 3. Ingress: Packets to WireGuard -// dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 4. Ingress: Packets to proxy (after eBPF rewrite) -// dst=127.0.0.1:proxyPort -// Matched by: dport=proxyPort -// -// Rules are cleaned up when the firewall manager is closed. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.notrackOutputChain == nil || m.notrackPreroutingChain == nil { - return fmt.Errorf("notrack chains not initialized") - } - - proxyPortBytes := binaryutil.BigEndian.PutUint16(proxyPort) - wgPortBytes := binaryutil.BigEndian.PutUint16(wgPort) - loopback := []byte{127, 0, 0, 1} - - // Egress rules: match outgoing loopback UDP packets - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackOutputChain.Table, - Chain: m.notrackOutputChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // sport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackOutputChain.Table, - Chain: m.notrackOutputChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - - // Ingress rules: match incoming loopback UDP packets - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackPreroutingChain.Table, - Chain: m.notrackPreroutingChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackPreroutingChain.Table, - Chain: m.notrackPreroutingChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: proxyPortBytes}, // dport=proxyPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush notrack rules: %w", err) - } - - log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort) - return nil -} - -func (m *Manager) initNoTrackChains(table *nftables.Table) error { - m.notrackOutputChain = m.rConn.AddChain(&nftables.Chain{ - Name: chainNameRawOutput, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookOutput, - Priority: nftables.ChainPriorityRaw, - }) - - m.notrackPreroutingChain = m.rConn.AddChain(&nftables.Chain{ - Name: chainNameRawPrerouting, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityRaw, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush chain creation: %w", err) - } - - return nil -} - -func (m *Manager) refreshNoTrackChains() error { - chains, err := m.rConn.ListChainsOfTableFamily(nftables.TableFamilyIPv4) - if err != nil { - return fmt.Errorf("list chains: %w", err) - } - - tableName := getTableName() - for _, c := range chains { - if c.Table.Name != tableName { - continue - } - switch c.Name { - case chainNameRawOutput: - m.notrackOutputChain = c - case chainNameRawPrerouting: - m.notrackPreroutingChain = c - } - } - - return nil -} - func (m *Manager) createWorkTable() (*nftables.Table, error) { return m.createWorkTableFamily(nftables.TableFamilyIPv4) } diff --git a/client/firewall/nftables/routing_linux.go b/client/firewall/nftables/routing_linux.go index 4115c94bd..d619c5543 100644 --- a/client/firewall/nftables/routing_linux.go +++ b/client/firewall/nftables/routing_linux.go @@ -192,7 +192,7 @@ func (r *family) addPostroutingRules() { Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade), }, - // We need to exclude the loopback interface as this changes the ebpf proxy port + // We need to exclude the loopback interface as this changes the wg proxy port &expr.Meta{ Key: expr.MetaKeyOIFNAME, Register: 1, diff --git a/client/firewall/uspfilter/filter.go b/client/firewall/uspfilter/filter.go index 5e1366c1f..0c73400f1 100644 --- a/client/firewall/uspfilter/filter.go +++ b/client/firewall/uspfilter/filter.go @@ -879,12 +879,6 @@ func (m *Manager) resetState() { } } -// SetupEBPFProxyNoTrack is not supported by the userspace firewall: eBPF isn't -// used in userspace mode, so this should never be called. -func (m *Manager) SetupEBPFProxyNoTrack(uint16, uint16) error { - return errNotSupported -} - // UpdateSet updates the rule destinations associated with the given set // by merging the existing prefixes with the new ones, then deduplicating. func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { diff --git a/client/iface/iface.go b/client/iface/iface.go index 247f421a2..f6006fa87 100644 --- a/client/iface/iface.go +++ b/client/iface/iface.go @@ -51,7 +51,6 @@ func ValidateMTU(mtu uint16) error { type wgProxyFactory interface { GetProxy() wgproxy.Proxy - GetProxyPort() uint16 Free() error } @@ -81,12 +80,6 @@ func (w *WGIface) GetProxy() wgproxy.Proxy { return w.wgProxyFactory.GetProxy() } -// GetProxyPort returns the proxy port used by the WireGuard proxy. -// Returns 0 if no proxy port is used (e.g., for userspace WireGuard). -func (w *WGIface) GetProxyPort() uint16 { - return w.wgProxyFactory.GetProxyPort() -} - // GetBind returns the EndpointManager userspace bind mode. func (w *WGIface) GetBind() device.EndpointManager { w.mu.Lock() diff --git a/client/iface/iface_close_test.go b/client/iface/iface_close_test.go index 171e15d0a..ea3115ec0 100644 --- a/client/iface/iface_close_test.go +++ b/client/iface/iface_close_test.go @@ -52,7 +52,6 @@ func (f *fakeTunDevice) Close() error { type fakeProxyFactory struct{} func (fakeProxyFactory) GetProxy() wgproxy.Proxy { return nil } -func (fakeProxyFactory) GetProxyPort() uint16 { return 0 } func (fakeProxyFactory) Free() error { return nil } // TestWGIface_CloseReleasesMutexBeforeTunClose guards against a deadlock diff --git a/client/iface/wgproxy/ebpf/portlookup.go b/client/iface/wgproxy/ebpf/portlookup.go deleted file mode 100644 index fce8f1507..000000000 --- a/client/iface/wgproxy/ebpf/portlookup.go +++ /dev/null @@ -1,32 +0,0 @@ -package ebpf - -import ( - "fmt" - "net" -) - -var ( - portRangeStart = 3128 - portRangeEnd = portRangeStart + 100 -) - -type portLookup struct { -} - -func (pl portLookup) searchFreePort() (int, error) { - for i := portRangeStart; i <= portRangeEnd; i++ { - if pl.tryToBind(i) == nil { - return i, nil - } - } - return 0, fmt.Errorf("failed to bind free port for eBPF proxy") -} - -func (pl portLookup) tryToBind(port int) error { - l, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) - if err != nil { - return err - } - _ = l.Close() - return nil -} diff --git a/client/iface/wgproxy/ebpf/portlookup_test.go b/client/iface/wgproxy/ebpf/portlookup_test.go deleted file mode 100644 index a2e92fc79..000000000 --- a/client/iface/wgproxy/ebpf/portlookup_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package ebpf - -import ( - "fmt" - "net" - "testing" -) - -func Test_portLookup_searchFreePort(t *testing.T) { - pl := portLookup{} - _, err := pl.searchFreePort() - if err != nil { - t.Fatal(err) - } -} - -func Test_portLookup_on_allocated(t *testing.T) { - pl := portLookup{} - - portRangeStart = 4128 - portRangeEnd = portRangeStart + 100 - - allocatedPort, err := allocatePort(portRangeStart) - if err != nil { - t.Fatal(err) - } - defer allocatedPort.Close() - - fp, err := pl.searchFreePort() - if err != nil { - t.Fatal(err) - } - - if fp != (portRangeStart + 1) { - t.Errorf("invalid free port, expected: %d, got: %d", portRangeStart+1, fp) - } -} - -func allocatePort(port int) (net.PacketConn, error) { - c, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) - if err != nil { - return nil, err - } - return c, err -} diff --git a/client/iface/wgproxy/ebpf/proxy.go b/client/iface/wgproxy/ebpf/proxy.go deleted file mode 100644 index 91c741c0d..000000000 --- a/client/iface/wgproxy/ebpf/proxy.go +++ /dev/null @@ -1,243 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - "context" - "fmt" - "net" - "sync" - - "github.com/hashicorp/go-multierror" - "github.com/pion/transport/v3" - log "github.com/sirupsen/logrus" - - nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/iface/bufsize" - "github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket" - "github.com/netbirdio/netbird/client/internal/ebpf" - ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - loopbackAddr = "127.0.0.1" -) - -// WGEBPFProxy definition for proxy with EBPF support -type WGEBPFProxy struct { - localWGListenPort int - proxyPort int - mtu uint16 - - ebpfManager ebpfMgr.Manager - relayedConnStore map[uint16]net.Conn - relayedConnMutex sync.Mutex - - lastUsedPort uint16 - rawConnIPv4 net.PacketConn - rawConnIPv6 net.PacketConn - conn transport.UDPConn - - ctx context.Context - ctxCancel context.CancelFunc -} - -// NewWGEBPFProxy create new WGEBPFProxy instance -func NewWGEBPFProxy(wgPort int, mtu uint16) *WGEBPFProxy { - log.Debugf("instantiate ebpf proxy") - wgProxy := &WGEBPFProxy{ - localWGListenPort: wgPort, - mtu: mtu, - ebpfManager: ebpf.GetEbpfManagerInstance(), - relayedConnStore: make(map[uint16]net.Conn), - } - return wgProxy -} - -// Listen load ebpf program and listen the proxy -func (p *WGEBPFProxy) Listen() error { - pl := portLookup{} - proxyPort, err := pl.searchFreePort() - if err != nil { - return err - } - p.proxyPort = proxyPort - - // Prepare IPv4 raw socket (required) - p.rawConnIPv4, err = rawsocket.PrepareSenderRawSocketIPv4() - if err != nil { - return err - } - - // Prepare IPv6 raw socket (optional) - p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6() - if err != nil { - log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err) - } - - err = p.ebpfManager.LoadWgProxy(proxyPort, p.localWGListenPort) - if err != nil { - if closeErr := p.rawConnIPv4.Close(); closeErr != nil { - log.Warnf("failed to close IPv4 raw socket: %v", closeErr) - } - if p.rawConnIPv6 != nil { - if closeErr := p.rawConnIPv6.Close(); closeErr != nil { - log.Warnf("failed to close IPv6 raw socket: %v", closeErr) - } - } - return err - } - - addr := net.UDPAddr{ - Port: proxyPort, - IP: net.ParseIP(loopbackAddr), - } - - p.ctx, p.ctxCancel = context.WithCancel(context.Background()) - - conn, err := nbnet.ListenUDP("udp", &addr) - if err != nil { - if cErr := p.Free(); cErr != nil { - log.Errorf("Failed to close the wgproxy: %s", cErr) - } - return err - } - p.conn = conn - - go p.proxyToRemote() - log.Infof("local wg proxy listening on: %d", proxyPort) - return nil -} - -// AddRelayedConn add new relayed connection for the proxy -func (p *WGEBPFProxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, error) { - wgEndpointPort, err := p.storeRelayedConn(relayedConn) - if err != nil { - return nil, err - } - - log.Infof("relayed conn added to wg proxy store: %s, endpoint port: :%d", relayedConn.RemoteAddr(), wgEndpointPort) - - wgEndpoint := &net.UDPAddr{ - IP: net.ParseIP(loopbackAddr), - Port: int(wgEndpointPort), - } - return wgEndpoint, nil -} - -// Free resources except the remoteConns will be keep open. -func (p *WGEBPFProxy) Free() error { - log.Debugf("free up ebpf wg proxy") - if p.ctx != nil && p.ctx.Err() != nil { - //nolint - return nil - } - - p.ctxCancel() - - var result *multierror.Error - if p.conn != nil { - if err := p.conn.Close(); err != nil { - result = multierror.Append(result, err) - } - } - - if err := p.ebpfManager.FreeWGProxy(); err != nil { - result = multierror.Append(result, err) - } - - if p.rawConnIPv4 != nil { - if err := p.rawConnIPv4.Close(); err != nil { - result = multierror.Append(result, err) - } - } - - if p.rawConnIPv6 != nil { - if err := p.rawConnIPv6.Close(); err != nil { - result = multierror.Append(result, err) - } - } - return nberrors.FormatErrorOrNil(result) -} - -// GetProxyPort returns the proxy listening port. -func (p *WGEBPFProxy) GetProxyPort() uint16 { - return uint16(p.proxyPort) -} - -// proxyToRemote read messages from local WireGuard interface and forward it to remote conn -// From this go routine has only one instance. -func (p *WGEBPFProxy) proxyToRemote() { - buf := make([]byte, p.mtu+bufsize.WGBufferOverhead) - for p.ctx.Err() == nil { - if err := p.readAndForwardPacket(buf); err != nil { - if p.ctx.Err() != nil { - return - } - log.Errorf("failed to proxy packet to remote conn: %s", err) - } - } -} - -func (p *WGEBPFProxy) readAndForwardPacket(buf []byte) error { - n, addr, err := p.conn.ReadFromUDP(buf) - if err != nil { - return fmt.Errorf("failed to read UDP packet from WG: %w", err) - } - - p.relayedConnMutex.Lock() - conn, ok := p.relayedConnStore[uint16(addr.Port)] - p.relayedConnMutex.Unlock() - if !ok { - if p.ctx.Err() == nil { - log.Debugf("relayed conn not found by port because conn already has been closed: %d", addr.Port) - } - return nil - } - - if _, err := conn.Write(buf[:n]); err != nil { - return fmt.Errorf("forward local WG packet (%d) to remote relayed conn: %w", addr.Port, err) - } - return nil -} - -func (p *WGEBPFProxy) storeRelayedConn(relayedConn net.Conn) (uint16, error) { - p.relayedConnMutex.Lock() - defer p.relayedConnMutex.Unlock() - - np, err := p.nextFreePort() - if err != nil { - return np, err - } - p.relayedConnStore[np] = relayedConn - return np, nil -} - -func (p *WGEBPFProxy) removeRelayedConn(relayedConnID uint16) { - p.relayedConnMutex.Lock() - defer p.relayedConnMutex.Unlock() - - _, ok := p.relayedConnStore[relayedConnID] - if ok { - log.Debugf("remove relayed conn from store by port: %d", relayedConnID) - } - delete(p.relayedConnStore, relayedConnID) -} - -func (p *WGEBPFProxy) nextFreePort() (uint16, error) { - if len(p.relayedConnStore) == 65535 { - return 0, fmt.Errorf("reached maximum relayed connection numbers") - } -generatePort: - if p.lastUsedPort == 65535 { - p.lastUsedPort = 1 - } else { - p.lastUsedPort++ - } - - if _, ok := p.relayedConnStore[p.lastUsedPort]; ok { - goto generatePort - } - return p.lastUsedPort, nil -} diff --git a/client/iface/wgproxy/ebpf/proxy_test.go b/client/iface/wgproxy/ebpf/proxy_test.go deleted file mode 100644 index 228c06c9b..000000000 --- a/client/iface/wgproxy/ebpf/proxy_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - "testing" -) - -func TestWGEBPFProxy_connStore(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - p, _ := wgProxy.storeRelayedConn(nil) - if p != 1 { - t.Errorf("invalid initial port: %d", wgProxy.lastUsedPort) - } - - numOfConns := 10 - for i := 0; i < numOfConns; i++ { - p, _ = wgProxy.storeRelayedConn(nil) - } - if p != uint16(numOfConns)+1 { - t.Errorf("invalid last used port: %d, expected: %d", p, numOfConns+1) - } - if len(wgProxy.relayedConnStore) != numOfConns+1 { - t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), numOfConns+1) - } -} - -func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - _, _ = wgProxy.storeRelayedConn(nil) - wgProxy.lastUsedPort = 65535 - p, _ := wgProxy.storeRelayedConn(nil) - - if len(wgProxy.relayedConnStore) != 2 { - t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), 2) - } - - if p != 2 { - t.Errorf("invalid last used port: %d, expected: %d", p, 2) - } -} - -func TestWGEBPFProxy_portCalculation_maxConn(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - for i := 0; i < 65535; i++ { - _, _ = wgProxy.storeRelayedConn(nil) - } - - _, err := wgProxy.storeRelayedConn(nil) - if err == nil { - t.Errorf("invalid relayed conn store calculation") - } -} diff --git a/client/iface/wgproxy/factory_kernel.go b/client/iface/wgproxy/factory_kernel.go index 7821df3de..0b2329b96 100644 --- a/client/iface/wgproxy/factory_kernel.go +++ b/client/iface/wgproxy/factory_kernel.go @@ -8,11 +8,13 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) const ( + envDisableKernelWGProxy = "NB_DISABLE_KERNEL_WG_PROXY" + // envDisableEBPFWGProxy is a deprecated alias for envDisableKernelWGProxy. envDisableEBPFWGProxy = "NB_DISABLE_EBPF_WG_PROXY" ) @@ -20,7 +22,7 @@ type KernelFactory struct { wgPort int mtu uint16 - ebpfProxy *ebpf.WGEBPFProxy + loopbackProxy *loopback.Proxy } func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory { @@ -29,55 +31,56 @@ func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory { mtu: mtu, } - if isEBPFDisabled() { + if isKernelProxyDisabled() { log.Infof("WireGuard Proxy Factory will produce UDP proxy") - log.Infof("eBPF WireGuard proxy is disabled via %s environment variable", envDisableEBPFWGProxy) return f } - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, mtu) - if err := ebpfProxy.Listen(); err != nil { + loopbackProxy := loopback.NewProxy(wgPort, mtu) + if err := loopbackProxy.Listen(); err != nil { log.Infof("WireGuard Proxy Factory will produce UDP proxy") - log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err) + log.Warnf("failed to initialize loopback proxy, fallback to user space proxy: %s", err) return f } - log.Infof("WireGuard Proxy Factory will produce eBPF proxy") - f.ebpfProxy = ebpfProxy + log.Infof("WireGuard Proxy Factory will produce loopback proxy") + f.loopbackProxy = loopbackProxy return f } func (w *KernelFactory) GetProxy() Proxy { - if w.ebpfProxy == nil { + if w.loopbackProxy == nil { return udpProxy.NewWGUDPProxy(w.wgPort, w.mtu) } - return ebpf.NewProxyWrapper(w.ebpfProxy) -} - -// GetProxyPort returns the eBPF proxy port, or 0 if eBPF is not active. -func (w *KernelFactory) GetProxyPort() uint16 { - if w.ebpfProxy == nil { - return 0 - } - return w.ebpfProxy.GetProxyPort() + return loopback.NewProxyWrapper(w.loopbackProxy) } func (w *KernelFactory) Free() error { - if w.ebpfProxy == nil { + if w.loopbackProxy == nil { return nil } - return w.ebpfProxy.Free() + return w.loopbackProxy.Free() } -func isEBPFDisabled() bool { - val := os.Getenv(envDisableEBPFWGProxy) +func isKernelProxyDisabled() bool { + env := envDisableKernelWGProxy + val := os.Getenv(env) + if val == "" { + env = envDisableEBPFWGProxy + val = os.Getenv(env) + } if val == "" { return false } + disabled, err := strconv.ParseBool(val) if err != nil { - log.Warnf("failed to parse %s: %v", envDisableEBPFWGProxy, err) + log.Warnf("failed to parse %s: %v", env, err) return false } + + if disabled { + log.Infof("kernel WireGuard proxy is disabled via %s", env) + } return disabled } diff --git a/client/iface/wgproxy/factory_usp.go b/client/iface/wgproxy/factory_usp.go index bbd67e076..a1b1c34d7 100644 --- a/client/iface/wgproxy/factory_usp.go +++ b/client/iface/wgproxy/factory_usp.go @@ -24,11 +24,6 @@ func (w *USPFactory) GetProxy() Proxy { return proxyBind.NewProxyBind(w.bind, w.mtu) } -// GetProxyPort returns 0 as userspace WireGuard doesn't use a separate proxy port. -func (w *USPFactory) GetProxyPort() uint16 { - return 0 -} - func (w *USPFactory) Free() error { return nil } diff --git a/client/iface/wgproxy/loopback/addr.go b/client/iface/wgproxy/loopback/addr.go new file mode 100644 index 000000000..52feee295 --- /dev/null +++ b/client/iface/wgproxy/loopback/addr.go @@ -0,0 +1,70 @@ +//go:build linux && !android + +package loopback + +import ( + "fmt" + "net/netip" +) + +// Peer endpoints live in the upper half of 127.0.0.0/8. Everything in that +// range is delivered to the loopback device without any address or route being +// configured, and staying out of 127.0.0.0/9 keeps well-known squatters such as +// 127.0.0.53 (systemd-resolved) and 127.0.1.1 out of the way. +const ( + addrRangeBase uint32 = 0x7f800000 // 127.128.0.0 + addrRangeSize uint32 = 1 << 23 // /9 + addrRangePrefix = "127.128.0.0/9" +) + +// allocator hands out one loopback address per relayed connection. The address +// is the peer's identity: WireGuard sends to it, and the proxy recovers which +// peer a packet belongs to from the destination address. +type allocator struct { + cursor uint32 +} + +// next returns the first free address at or after the cursor, wrapping once. +// inUse reports whether an address is already handed out. +func (a *allocator) next(inUse func(netip.Addr) bool) (netip.Addr, error) { + for i := uint32(0); i < addrRangeSize; i++ { + a.cursor = (a.cursor + 1) % addrRangeSize + addr := addrFromOffset(a.cursor) + if !addr.IsValid() { + continue + } + if inUse(addr) { + continue + } + return addr, nil + } + return netip.Addr{}, fmt.Errorf("no free endpoint address in %s", addrRangePrefix) +} + +// addrFromOffset maps an offset in the range to an address, skipping the .0 and +// .255 hosts. They are unremarkable on loopback, but tools and firewall rules +// tend to treat them as network and broadcast addresses. +func addrFromOffset(offset uint32) netip.Addr { + last := offset & 0xff + if last == 0 || last == 0xff { + return netip.Addr{} + } + + v := addrRangeBase + offset + return netip.AddrFrom4([4]byte{ + byte(v >> 24), + byte(v >> 16), + byte(v >> 8), + byte(v), + }) +} + +// inRange reports whether addr is one this proxy could have handed out. +func inRange(addr netip.Addr) bool { + if !addr.Is4() { + return false + } + b := addr.As4() + v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]) + return v >= addrRangeBase && v < addrRangeBase+addrRangeSize && b[3] != 0 && b[3] != 0xff +} diff --git a/client/iface/wgproxy/loopback/addr_test.go b/client/iface/wgproxy/loopback/addr_test.go new file mode 100644 index 000000000..3755b7269 --- /dev/null +++ b/client/iface/wgproxy/loopback/addr_test.go @@ -0,0 +1,114 @@ +//go:build linux && !android + +package loopback + +import ( + "net/netip" + "testing" +) + +func TestAllocatorHandsOutDistinctAddresses(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + + for i := 0; i < 1000; i++ { + addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] }) + if err != nil { + t.Fatalf("allocate %d: %v", i, err) + } + if taken[addr] { + t.Fatalf("address %s handed out twice", addr) + } + if !inRange(addr) { + t.Fatalf("address %s outside %s", addr, addrRangePrefix) + } + taken[addr] = true + } +} + +func TestAllocatorSkipsNetworkAndBroadcastHosts(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + + // enough allocations to walk past a .255/.0 boundary + for i := 0; i < 600; i++ { + addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] }) + if err != nil { + t.Fatalf("allocate %d: %v", i, err) + } + last := addr.As4()[3] + if last == 0 || last == 255 { + t.Fatalf("address %s ends in .%d", addr, last) + } + taken[addr] = true + } +} + +func TestAllocatorReusesReleasedAddresses(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + inUse := func(candidate netip.Addr) bool { return taken[candidate] } + alloc := func() netip.Addr { + t.Helper() + addr, err := a.next(inUse) + if err != nil { + t.Fatalf("allocate: %v", err) + } + taken[addr] = true + return addr + } + + first := alloc() + second := alloc() + delete(taken, first) + + // The cursor only moves forward, so a released address comes back after a + // wrap. Park the cursor near the end of the range instead of allocating + // 2^23 addresses: the next call takes the last usable address, and the one + // after that wraps past the skipped .255 and .0 hosts to the released one. + a.cursor = addrRangeSize - 3 + last := alloc() + if want := netip.MustParseAddr("127.255.255.254"); last != want { + t.Fatalf("expected the last usable address %s before the wrap, got %s", want, last) + } + + if reused := alloc(); reused != first { + t.Fatalf("expected the released address %s after the wrap, got %s", first, reused) + } + + // second is still held, so the allocator must step over it. + if next := alloc(); next == second { + t.Fatalf("allocator handed out %s while it was still in use", second) + } +} + +func TestInRange(t *testing.T) { + tests := []struct { + addr string + want bool + }{ + {"127.128.0.1", true}, + {"127.255.255.254", true}, + {"127.128.0.0", false}, // network host, never handed out + {"127.128.5.255", false}, // broadcast host, never handed out + {"127.127.255.255", false}, // below the range, where 127.0.0.53 and friends live + {"127.0.0.1", false}, + {"127.0.0.53", false}, + {"127.0.1.1", false}, + {"128.0.0.1", false}, + {"10.0.0.1", false}, + } + + for _, tc := range tests { + addr := netip.MustParseAddr(tc.addr) + if got := inRange(addr); got != tc.want { + t.Errorf("inRange(%s) = %v, want %v", tc.addr, got, tc.want) + } + } +} + +func TestInRangeIgnoresIPv6(t *testing.T) { + if inRange(netip.MustParseAddr("::1")) { + t.Error("inRange(::1) = true, want false") + } +} diff --git a/client/iface/wgproxy/loopback/proxy.go b/client/iface/wgproxy/loopback/proxy.go new file mode 100644 index 000000000..f9364c766 --- /dev/null +++ b/client/iface/wgproxy/loopback/proxy.go @@ -0,0 +1,291 @@ +//go:build linux && !android + +package loopback + +import ( + "context" + "fmt" + "net" + "net/netip" + "sync" + "syscall" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "golang.org/x/net/ipv4" + "golang.org/x/sys/unix" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/iface/bufsize" + "github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket" +) + +const ( + loopbackDevice = "lo" + + portRangeStart = 3128 + portRangeEnd = portRangeStart + 100 +) + +// Proxy forwards packets between relayed connections and a local kernel +// WireGuard instance. Every relayed peer gets its own loopback address as its +// WireGuard endpoint, so a single socket serves all of them: the destination +// address of an incoming packet identifies the peer. +type Proxy struct { + localWGListenPort int + mtu uint16 + proxyPort int + + conn *net.UDPConn + packetConn *ipv4.PacketConn + loIndex int + rawConnIPv4 net.PacketConn + rawConnIPv6 net.PacketConn + + relayedConnMutex sync.Mutex + relayedConnStore map[netip.Addr]net.Conn + addrs allocator + + ctx context.Context + ctxCancel context.CancelFunc +} + +// NewProxy creates a proxy for the WireGuard instance listening on wgPort. +func NewProxy(wgPort int, mtu uint16) *Proxy { + log.Debugf("instantiate loopback wg proxy") + return &Proxy{ + localWGListenPort: wgPort, + mtu: mtu, + relayedConnStore: make(map[netip.Addr]net.Conn), + } +} + +// Listen opens the shared socket and starts forwarding WireGuard packets to the +// relayed connections. +func (p *Proxy) Listen() error { + rawConnIPv4, err := rawsocket.PrepareSenderRawSocketIPv4() + if err != nil { + return fmt.Errorf("prepare IPv4 raw socket: %w", err) + } + p.rawConnIPv4 = rawConnIPv4 + + p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6() + if err != nil { + log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err) + } + + loopback, err := net.InterfaceByName(loopbackDevice) + if err != nil { + if freeErr := p.Free(); freeErr != nil { + log.Errorf("failed to free the wgproxy: %s", freeErr) + } + return fmt.Errorf("look up %s: %w", loopbackDevice, err) + } + p.loIndex = loopback.Index + + if err := p.listen(); err != nil { + if freeErr := p.Free(); freeErr != nil { + log.Errorf("failed to free the wgproxy: %s", freeErr) + } + return err + } + + p.ctx, p.ctxCancel = context.WithCancel(context.Background()) + + go p.proxyToRemote() + log.Infof("local wg proxy listening on %s:%d", addrRangePrefix, p.proxyPort) + return nil +} + +// listen binds the shared socket on the first free port of the range. The bind +// has to be a wildcard one to receive every peer address in the range, so it is +// restricted to the loopback device: without that the port would be reachable +// on every interface. +func (p *Proxy) listen() error { + var lastErr error + for port := portRangeStart; port <= portRangeEnd; port++ { + err := p.listenOn(port) + if err == nil { + p.proxyPort = port + return nil + } + lastErr = err + } + return fmt.Errorf("bind proxy port in range %d-%d: %w", portRangeStart, portRangeEnd, lastErr) +} + +func (p *Proxy) listenOn(proxyPort int) error { + lc := net.ListenConfig{ + Control: func(_, _ string, c syscall.RawConn) error { + var sockErr error + if err := c.Control(func(fd uintptr) { + if err := unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, loopbackDevice); err != nil { + sockErr = fmt.Errorf("bind to %s: %w", loopbackDevice, err) + return + } + }); err != nil { + return fmt.Errorf("control socket: %w", err) + } + return sockErr + }, + } + + conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", proxyPort)) + if err != nil { + return fmt.Errorf("listen on :%d: %w", proxyPort, err) + } + + udpConn, ok := conn.(*net.UDPConn) + if !ok { + if closeErr := conn.Close(); closeErr != nil { + log.Errorf("failed to close proxy conn: %s", closeErr) + } + return fmt.Errorf("unexpected conn type %T", conn) + } + + packetConn := ipv4.NewPacketConn(udpConn) + // the destination address carries the peer identity, the interface index is + // checked on receive as a second line of defense behind SO_BINDTODEVICE + if err := packetConn.SetControlMessage(ipv4.FlagDst|ipv4.FlagInterface, true); err != nil { + if closeErr := udpConn.Close(); closeErr != nil { + log.Errorf("failed to close proxy conn: %s", closeErr) + } + return fmt.Errorf("request destination address: %w", err) + } + + p.conn = udpConn + p.packetConn = packetConn + return nil +} + +// AddRelayedConn assigns an endpoint address to the relayed connection and +// returns the address WireGuard should send to, along with the key the +// connection is stored under. +func (p *Proxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, netip.Addr, error) { + addr, err := p.storeRelayedConn(relayedConn) + if err != nil { + return nil, netip.Addr{}, err + } + + log.Infof("relayed conn added to wg proxy store: %s, endpoint address: %s", relayedConn.RemoteAddr(), addr) + + return &net.UDPAddr{ + IP: addr.AsSlice(), + Port: p.proxyPort, + }, addr, nil +} + +// Free releases the proxy resources. The relayed connections are left open. +func (p *Proxy) Free() error { + log.Debugf("free up loopback wg proxy") + if p.ctx != nil && p.ctx.Err() != nil { + //nolint + return nil + } + + if p.ctxCancel != nil { + p.ctxCancel() + } + + var result *multierror.Error + if p.conn != nil { + if err := p.conn.Close(); err != nil { + result = multierror.Append(result, err) + } + } + + if p.rawConnIPv4 != nil { + if err := p.rawConnIPv4.Close(); err != nil { + result = multierror.Append(result, err) + } + } + + if p.rawConnIPv6 != nil { + if err := p.rawConnIPv6.Close(); err != nil { + result = multierror.Append(result, err) + } + } + return nberrors.FormatErrorOrNil(result) +} + +// proxyToRemote reads packets from the local WireGuard instance and forwards +// them to the relayed connection the destination address belongs to. +func (p *Proxy) proxyToRemote() { + buf := make([]byte, p.mtu+bufsize.WGBufferOverhead) + for p.ctx.Err() == nil { + if err := p.readAndForwardPacket(buf); err != nil { + if p.ctx.Err() != nil { + return + } + log.Errorf("failed to proxy packet to remote conn: %s", err) + } + } +} + +func (p *Proxy) readAndForwardPacket(buf []byte) error { + n, cm, _, err := p.packetConn.ReadFrom(buf) + if err != nil { + return fmt.Errorf("read UDP packet from WG: %w", err) + } + + if cm == nil { + return fmt.Errorf("no control message on packet") + } + + if cm.IfIndex != p.loIndex { + log.Tracef("dropping packet received on interface %d instead of %s", cm.IfIndex, loopbackDevice) + return nil + } + + dst, ok := netip.AddrFromSlice(cm.Dst.To4()) + if !ok || !inRange(dst) { + log.Tracef("dropping packet for unexpected destination %s", cm.Dst) + return nil + } + + p.relayedConnMutex.Lock() + conn, ok := p.relayedConnStore[dst] + p.relayedConnMutex.Unlock() + if !ok { + if p.ctx.Err() == nil { + log.Debugf("relayed conn not found by address because conn already has been closed: %s", dst) + } + return nil + } + + if _, err := conn.Write(buf[:n]); err != nil { + return fmt.Errorf("forward local WG packet (%s) to remote relayed conn: %w", dst, err) + } + return nil +} + +func (p *Proxy) storeRelayedConn(relayedConn net.Conn) (netip.Addr, error) { + p.relayedConnMutex.Lock() + defer p.relayedConnMutex.Unlock() + + addr, err := p.addrs.next(func(a netip.Addr) bool { + _, ok := p.relayedConnStore[a] + return ok + }) + if err != nil { + return netip.Addr{}, err + } + + p.relayedConnStore[addr] = relayedConn + return addr, nil +} + +// removeRelayedConn releases an endpoint address. It only removes the entry +// while it still belongs to relayedConn, so a late release cannot take an +// address away from the peer it was handed to next. +func (p *Proxy) removeRelayedConn(addr netip.Addr, relayedConn net.Conn) { + p.relayedConnMutex.Lock() + defer p.relayedConnMutex.Unlock() + + if stored, ok := p.relayedConnStore[addr]; !ok || stored != relayedConn { + return + } + + log.Debugf("remove relayed conn from store by address: %s", addr) + delete(p.relayedConnStore, addr) +} diff --git a/client/iface/wgproxy/loopback/proxy_privileged_test.go b/client/iface/wgproxy/loopback/proxy_privileged_test.go new file mode 100644 index 000000000..6314fe2de --- /dev/null +++ b/client/iface/wgproxy/loopback/proxy_privileged_test.go @@ -0,0 +1,196 @@ +//go:build linux && !android && privileged + +package loopback + +import ( + "context" + "net" + "strconv" + "testing" + "time" +) + +const testWGPort = 51862 + +// relayEnd stands in for a relayed connection: the proxy writes what it read +// from WireGuard into it, and the test reads it back out here. +func relayEnd(t *testing.T) (proxySide net.Conn, testSide *net.UDPConn) { + t.Helper() + + testSide, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatalf("relay listener: %v", err) + } + t.Cleanup(func() { + if err := testSide.Close(); err != nil { + t.Logf("close relay listener: %v", err) + } + }) + + proxySide, err = net.Dial("udp", testSide.LocalAddr().String()) + if err != nil { + t.Fatalf("relay conn: %v", err) + } + t.Cleanup(func() { + if err := proxySide.Close(); err != nil { + t.Logf("close relay conn: %v", err) + } + }) + + return proxySide, testSide +} + +// TestProxyDemuxesByDestinationAddress is the core of the design: one socket +// serves every peer, and the destination address decides which relayed +// connection a WireGuard packet belongs to. +func TestProxyDemuxesByDestinationAddress(t *testing.T) { + proxy := NewProxy(testWGPort, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + const peers = 3 + endpoints := make([]*net.UDPAddr, 0, peers) + readers := make([]*net.UDPConn, 0, peers) + for i := 0; i < peers; i++ { + proxySide, testSide := relayEnd(t) + endpoint, _, err := proxy.AddRelayedConn(proxySide) + if err != nil { + t.Fatalf("add relayed conn %d: %v", i, err) + } + if endpoint.Port != proxy.proxyPort { + t.Errorf("peer %d endpoint port = %d, want the shared proxy port %d", i, endpoint.Port, proxy.proxyPort) + } + endpoints = append(endpoints, endpoint) + readers = append(readers, testSide) + } + + // every peer must have its own address, otherwise they are indistinguishable + seen := make(map[string]bool, peers) + for i, endpoint := range endpoints { + if seen[endpoint.IP.String()] { + t.Fatalf("peer %d reuses endpoint address %s", i, endpoint.IP) + } + seen[endpoint.IP.String()] = true + } + + wgSock, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatalf("wg socket: %v", err) + } + defer func() { + if err := wgSock.Close(); err != nil { + t.Logf("close wg socket: %v", err) + } + }() + + for i, endpoint := range endpoints { + payload := []byte{byte(i), 'p', 'k', 't'} + if _, err := wgSock.WriteTo(payload, endpoint); err != nil { + t.Fatalf("write to peer %d endpoint %s: %v", i, endpoint, err) + } + + buf := make([]byte, 1500) + if err := readers[i].SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + n, _, err := readers[i].ReadFrom(buf) + if err != nil { + t.Fatalf("peer %d did not receive its packet: %v", i, err) + } + if string(buf[:n]) != string(payload) { + t.Errorf("peer %d got %q, want %q", i, buf[:n], payload) + } + + // no other peer may see it + for j, other := range readers { + if j == i { + continue + } + if err := other.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + if _, _, err := other.ReadFrom(buf); err == nil { + t.Errorf("packet for peer %d also delivered to peer %d", i, j) + } + } + } +} + +// TestProxyDropsPacketsOutsideTheRange guards the wildcard bind: anything that +// is not addressed to a handed-out endpoint must not reach a relayed peer. +func TestProxyDropsPacketsOutsideTheRange(t *testing.T) { + proxy := NewProxy(testWGPort+1, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + proxySide, testSide := relayEnd(t) + if _, _, err := proxy.AddRelayedConn(proxySide); err != nil { + t.Fatalf("add relayed conn: %v", err) + } + + sender, err := net.Dial("udp", net.JoinHostPort("127.0.0.1", strconv.Itoa(proxy.proxyPort))) + if err != nil { + t.Fatalf("sender: %v", err) + } + defer func() { + if err := sender.Close(); err != nil { + t.Logf("close sender: %v", err) + } + }() + + if _, err := sender.Write([]byte("stray")); err != nil { + t.Fatalf("write stray packet: %v", err) + } + + buf := make([]byte, 1500) + if err := testSide.SetReadDeadline(time.Now().Add(500 * time.Millisecond)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + if _, _, err := testSide.ReadFrom(buf); err == nil { + t.Error("packet addressed to 127.0.0.1 was forwarded to a relayed peer") + } +} + +// A wrapper that is closed before it starts forwarding still has to give its +// endpoint address back, otherwise the range leaks an address per attempt. +func TestClosingBeforeWorkReleasesTheAddress(t *testing.T) { + proxy := NewProxy(testWGPort+2, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + proxySide, _ := relayEnd(t) + wrapper := NewProxyWrapper(proxy) + if err := wrapper.AddRelayedConn(context.Background(), nil, proxySide); err != nil { + t.Fatalf("add relayed conn: %v", err) + } + + if got := len(proxy.relayedConnStore); got != 1 { + t.Fatalf("store holds %d entries after adding one conn, want 1", got) + } + + if err := wrapper.CloseConn(); err != nil { + t.Fatalf("close conn: %v", err) + } + + if got := len(proxy.relayedConnStore); got != 0 { + t.Errorf("store holds %d entries after close, want 0", got) + } +} diff --git a/client/iface/wgproxy/ebpf/wrapper.go b/client/iface/wgproxy/loopback/wrapper.go similarity index 84% rename from client/iface/wgproxy/ebpf/wrapper.go rename to client/iface/wgproxy/loopback/wrapper.go index f75e21aa6..a9cb1ab59 100644 --- a/client/iface/wgproxy/ebpf/wrapper.go +++ b/client/iface/wgproxy/loopback/wrapper.go @@ -1,6 +1,6 @@ //go:build linux && !android -package ebpf +package loopback import ( "context" @@ -8,6 +8,7 @@ import ( "fmt" "io" "net" + "net/netip" "sync" "github.com/google/gopacket" @@ -95,13 +96,14 @@ func NewPacketHeaders(localWGListenPort int, endpoint *net.UDPAddr) (*PacketHead // ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call type ProxyWrapper struct { - wgeBPFProxy *WGEBPFProxy + proxy *Proxy remoteConn net.Conn ctx context.Context cancel context.CancelFunc wgRelayedEndpointAddr *net.UDPAddr + peerAddr netip.Addr headers *PacketHeaders headerCurrentUsed *PacketHeaders rawConn net.PacketConn @@ -113,36 +115,44 @@ type ProxyWrapper struct { closeListener *listener.CloseListener } -func NewProxyWrapper(proxy *WGEBPFProxy) *ProxyWrapper { +func NewProxyWrapper(proxy *Proxy) *ProxyWrapper { return &ProxyWrapper{ - wgeBPFProxy: proxy, + proxy: proxy, pausedCond: sync.NewCond(&sync.Mutex{}), closeListener: listener.NewCloseListener(), } } func (p *ProxyWrapper) AddRelayedConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error { - addr, err := p.wgeBPFProxy.AddRelayedConn(remoteConn) + addr, peerAddr, err := p.proxy.AddRelayedConn(remoteConn) if err != nil { return fmt.Errorf("add relayed conn: %w", err) } - headers, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, addr) + // the endpoint address is otherwise only released by the forwarding + // goroutine, which never starts when the setup below fails + release := func() { p.proxy.removeRelayedConn(peerAddr, remoteConn) } + + headers, err := NewPacketHeaders(p.proxy.localWGListenPort, addr) if err != nil { + release() return fmt.Errorf("create packet sender: %w", err) } // Check if required raw connection is available - if !headers.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil { + if !headers.isIPv4 && p.proxy.rawConnIPv6 == nil { + release() return errIPv6ConnNotAvailable } - if headers.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil { + if headers.isIPv4 && p.proxy.rawConnIPv4 == nil { + release() return errIPv4ConnNotAvailable } p.remoteConn = remoteConn p.ctx, p.cancel = context.WithCancel(ctx) p.wgRelayedEndpointAddr = addr + p.peerAddr = peerAddr p.headers = headers p.rawConn = p.selectRawConn(headers) return nil @@ -193,18 +203,18 @@ func (p *ProxyWrapper) RedirectAs(endpoint *net.UDPAddr) { return } - header, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, endpoint) + header, err := NewPacketHeaders(p.proxy.localWGListenPort, endpoint) if err != nil { log.Errorf("failed to create packet headers: %s", err) return } // Check if required raw connection is available - if !header.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil { + if !header.isIPv4 && p.proxy.rawConnIPv6 == nil { log.Error(errIPv6ConnNotAvailable) return } - if header.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil { + if header.isIPv4 && p.proxy.rawConnIPv4 == nil { log.Error(errIPv4ConnNotAvailable) return } @@ -240,6 +250,10 @@ func (p *ProxyWrapper) CloseConn() error { p.closeListener.SetCloseListener(nil) + // releases the endpoint address for a wrapper that was never started, and + // is a no-op once the forwarding goroutine has released it + p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn) + p.pausedCond.L.Lock() p.paused = false p.pausedCond.Signal() @@ -252,9 +266,9 @@ func (p *ProxyWrapper) CloseConn() error { } func (p *ProxyWrapper) proxyToLocal(ctx context.Context) { - defer p.wgeBPFProxy.removeRelayedConn(uint16(p.wgRelayedEndpointAddr.Port)) + defer p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn) - buf := make([]byte, p.wgeBPFProxy.mtu+bufsize.WGBufferOverhead) + buf := make([]byte, p.proxy.mtu+bufsize.WGBufferOverhead) for { n, err := p.readFromRemote(ctx, buf) if err != nil { @@ -286,7 +300,7 @@ func (p *ProxyWrapper) readFromRemote(ctx context.Context, buf []byte) (int, err } p.closeListener.Notify() if !errors.Is(err, io.EOF) { - log.Errorf("failed to read from relayed conn (endpoint: :%d): %s", p.wgRelayedEndpointAddr.Port, err) + log.Errorf("failed to read from relayed conn (endpoint: %s): %s", p.wgRelayedEndpointAddr, err) } return 0, err } @@ -314,7 +328,7 @@ func (p *ProxyWrapper) sendPkg(data []byte, header *PacketHeaders) error { func (p *ProxyWrapper) selectRawConn(header *PacketHeaders) net.PacketConn { if header.isIPv4 { - return p.wgeBPFProxy.rawConnIPv4 + return p.proxy.rawConnIPv4 } - return p.wgeBPFProxy.rawConnIPv6 + return p.proxy.rawConnIPv6 } diff --git a/client/iface/wgproxy/proxy_linux_test.go b/client/iface/wgproxy/proxy_linux_test.go index e34dd3b6b..88d4588a5 100644 --- a/client/iface/wgproxy/proxy_linux_test.go +++ b/client/iface/wgproxy/proxy_linux_test.go @@ -9,25 +9,25 @@ import ( "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/wgaddr" bindproxy "github.com/netbirdio/netbird/client/iface/wgproxy/bind" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) func seedProxies() ([]proxyInstance, error) { pl := make([]proxyInstance, 0) - ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280) - if err := ebpfProxy.Listen(); err != nil { - return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err) + loopbackProxy := loopback.NewProxy(51831, 1280) + if err := loopbackProxy.Listen(); err != nil { + return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err) } - pEbpf := proxyInstance{ - name: "ebpf kernel proxy", - proxy: ebpf.NewProxyWrapper(ebpfProxy), + pLoopback := proxyInstance{ + name: "loopback kernel proxy", + proxy: loopback.NewProxyWrapper(loopbackProxy), wgPort: 51831, - closeFn: ebpfProxy.Free, + closeFn: loopbackProxy.Free, } - pl = append(pl, pEbpf) + pl = append(pl, pLoopback) pUDP := proxyInstance{ name: "udp kernel proxy", @@ -42,18 +42,18 @@ func seedProxies() ([]proxyInstance, error) { func seedProxyForProxyCloseByRemoteConn() ([]proxyInstance, error) { pl := make([]proxyInstance, 0) - ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280) - if err := ebpfProxy.Listen(); err != nil { - return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err) + loopbackProxy := loopback.NewProxy(51831, 1280) + if err := loopbackProxy.Listen(); err != nil { + return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err) } - pEbpf := proxyInstance{ - name: "ebpf kernel proxy", - proxy: ebpf.NewProxyWrapper(ebpfProxy), + pLoopback := proxyInstance{ + name: "loopback kernel proxy", + proxy: loopback.NewProxyWrapper(loopbackProxy), wgPort: 51831, - closeFn: ebpfProxy.Free, + closeFn: loopbackProxy.Free, } - pl = append(pl, pEbpf) + pl = append(pl, pLoopback) pUDP := proxyInstance{ name: "udp kernel proxy", diff --git a/client/iface/wgproxy/redirect_test.go b/client/iface/wgproxy/redirect_test.go index f0d59cc64..47f571f2b 100644 --- a/client/iface/wgproxy/redirect_test.go +++ b/client/iface/wgproxy/redirect_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) @@ -198,20 +198,20 @@ func testRedirectAs(t *testing.T, proxy Proxy, wgPort int, nbAddr, p2pEndpoint * } } -// TestRedirectAs_eBPF_IPv4 tests RedirectAs with eBPF proxy using IPv4 addresses -func TestRedirectAs_eBPF_IPv4(t *testing.T) { +// TestRedirectAs_Loopback_IPv4 tests RedirectAs with the loopback proxy using IPv4 addresses +func TestRedirectAs_Loopback_IPv4(t *testing.T) { wgPort := 51850 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) // NetBird UDP address of the remote peer nbAddr := &net.UDPAddr{ @@ -227,20 +227,20 @@ func TestRedirectAs_eBPF_IPv4(t *testing.T) { testRedirectAs(t, proxy, wgPort, nbAddr, p2pEndpoint) } -// TestRedirectAs_eBPF_IPv6 tests RedirectAs with eBPF proxy using IPv6 addresses -func TestRedirectAs_eBPF_IPv6(t *testing.T) { +// TestRedirectAs_Loopback_IPv6 tests RedirectAs with the loopback proxy using IPv6 addresses +func TestRedirectAs_Loopback_IPv6(t *testing.T) { wgPort := 51851 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) // NetBird UDP address of the remote peer nbAddr := &net.UDPAddr{ @@ -259,17 +259,17 @@ func TestRedirectAs_eBPF_IPv6(t *testing.T) { // TestRedirectAs_Multiple_Switches tests switching between multiple endpoints func TestRedirectAs_Multiple_Switches(t *testing.T) { wgPort := 51856 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) ctx := context.Background() diff --git a/client/internal/ebpf/ebpf/bpf_bpfeb.go b/client/internal/ebpf/ebpf/bpf_bpfeb.go deleted file mode 100644 index 4b6230217..000000000 --- a/client/internal/ebpf/ebpf/bpf_bpfeb.go +++ /dev/null @@ -1,148 +0,0 @@ -// Code generated by bpf2go; DO NOT EDIT. -//go:build mips || mips64 || ppc64 || s390x - -package ebpf - -import ( - "bytes" - _ "embed" - "fmt" - "io" - - "github.com/cilium/ebpf" -) - -// loadBpf returns the embedded CollectionSpec for bpf. -func loadBpf() (*ebpf.CollectionSpec, error) { - reader := bytes.NewReader(_BpfBytes) - spec, err := ebpf.LoadCollectionSpecFromReader(reader) - if err != nil { - return nil, fmt.Errorf("can't load bpf: %w", err) - } - - return spec, err -} - -// loadBpfObjects loads bpf and converts it into a struct. -// -// The following types are suitable as obj argument: -// -// *bpfObjects -// *bpfPrograms -// *bpfMaps -// -// See ebpf.CollectionSpec.LoadAndAssign documentation for details. -func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { - spec, err := loadBpf() - if err != nil { - return err - } - - return spec.LoadAndAssign(obj, opts) -} - -// bpfSpecs contains maps and programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfSpecs struct { - bpfProgramSpecs - bpfMapSpecs - bpfVariableSpecs -} - -// bpfProgramSpecs contains programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfProgramSpecs struct { - NbXdpProg *ebpf.ProgramSpec `ebpf:"nb_xdp_prog"` -} - -// bpfMapSpecs contains maps before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfMapSpecs struct { - NbFeatures *ebpf.MapSpec `ebpf:"nb_features"` - NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"` -} - -// bpfVariableSpecs contains global variables before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfVariableSpecs struct { - FlagFeatureWgProxy *ebpf.VariableSpec `ebpf:"flag_feature_wg_proxy"` - MapKeyFeatures *ebpf.VariableSpec `ebpf:"map_key_features"` - MapKeyProxyPort *ebpf.VariableSpec `ebpf:"map_key_proxy_port"` - MapKeyWgPort *ebpf.VariableSpec `ebpf:"map_key_wg_port"` - ProxyPort *ebpf.VariableSpec `ebpf:"proxy_port"` - WgPort *ebpf.VariableSpec `ebpf:"wg_port"` -} - -// bpfObjects contains all objects after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfObjects struct { - bpfPrograms - bpfMaps - bpfVariables -} - -func (o *bpfObjects) Close() error { - return _BpfClose( - &o.bpfPrograms, - &o.bpfMaps, - ) -} - -// bpfMaps contains all maps after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfMaps struct { - NbFeatures *ebpf.Map `ebpf:"nb_features"` - NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"` -} - -func (m *bpfMaps) Close() error { - return _BpfClose( - m.NbFeatures, - m.NbWgProxySettingsMap, - ) -} - -// bpfVariables contains all global variables after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfVariables struct { - FlagFeatureWgProxy *ebpf.Variable `ebpf:"flag_feature_wg_proxy"` - MapKeyFeatures *ebpf.Variable `ebpf:"map_key_features"` - MapKeyProxyPort *ebpf.Variable `ebpf:"map_key_proxy_port"` - MapKeyWgPort *ebpf.Variable `ebpf:"map_key_wg_port"` - ProxyPort *ebpf.Variable `ebpf:"proxy_port"` - WgPort *ebpf.Variable `ebpf:"wg_port"` -} - -// bpfPrograms contains all programs after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfPrograms struct { - NbXdpProg *ebpf.Program `ebpf:"nb_xdp_prog"` -} - -func (p *bpfPrograms) Close() error { - return _BpfClose( - p.NbXdpProg, - ) -} - -func _BpfClose(closers ...io.Closer) error { - for _, closer := range closers { - if err := closer.Close(); err != nil { - return err - } - } - return nil -} - -// Do not access this directly. -// -//go:embed bpf_bpfeb.o -var _BpfBytes []byte diff --git a/client/internal/ebpf/ebpf/bpf_bpfeb.o b/client/internal/ebpf/ebpf/bpf_bpfeb.o deleted file mode 100644 index b435d49647544d14fc150e72031fe0251e485a42..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8712 zcmds6Z;Vw(6`%X|57^drt3cP5wzCztZz(LhfRuzl9taf-DNh8UhI;qyzPr26ef!?y z-M6qvwHs3nsSV*nO;%&NhGEpi17nPP57XZ#2EbjX3pH5 zH?R@yHz%1p=Xd7JIcLtCnLGEr*Z1syDVK{>RwDHe&>kb}0rAXog9`HOQBiM?p|eM? z&PYRi-NNYNh$U7kWkytFTqr-)XXPGLFZ6YDgwuE3>g`!dulNyNHlvjKmNlm?k6PmL zaodLWE00_L${y84YN`Iq;{Rc-;M%=%pisrA2w~jdW7wS=2I;{u()2s#E^S}9fy0= z1~iYm^|0|!5!Qbl`$Iqf@AIQyJ;Qn5CqJGe|G&tOJmryBYuF-R%y=(93_r{d%A!Qx z$;ABP+OvHKmE|7Zr;Gd5pWi`7W6AiykP%zE%S0wA?6}BC!3LD_oLh!OANHj-4XDQa3IPu(8rwLz+gvTl0NL$=(x~`Pjph`I2HQxjYT;#G4h4w!V;YYPfk7) zY>|oH2LDl1l;}Hnkk+6VM8RXAe;_n%y(_jED#$&EF1siY#W@!J1Oqu1Dx;s;7Ecvj z6`D4FEwl#h;x+^xCqut+#74hC3q8?wv>!ox!}*0)1HUh8juUAzwDV5_!z1fP>q}8g_CQ>!nN@iPXa6{T0+i-FsZZzw&rv}HAMmbTc&Ln!f z9&L=r)1^jSt=Fe!8gWuhrt|%^(b$P+$|$u?_fvK%Ic0Sja|7Fq!pgYL_Zh7N`n2H~ zb4CPq)|xIW`ax~eH_THE3xbWwR=t+yttE7RMO7M;<)*4kR@I55nO5pG)vBkct+<-h zRHdxOo27}kK0dBmt*U~*t@;=uIX0P$O{qAZ*{Wu0X=S38l=awTsR`C-GQ5R7cxGMc zB%A>=otXx^R4zADm9kM~^W>;a8#xNkY*eMzQZr@aU|OGPjwPy`q%CC}Yp_RAVz%7K zmVylJGxBMxIWyMMW|~^BtzdzKWi{)yXx(yff&3-eu7~t%ujwp&z`kguv3`f~U-#L_ zR`VN&Hx0cwv}Nj5=RjGvIyMPx$aF0^8JE+R%Lcnri{8jm8cinpSj{1u zES>_Su~T3q889p4yRh9DbFyAP#i7@CWt$1`bQvqvDw#)|)GUgp6R@z2nn{|*mGMfg zoXo1`Y}I~vR;evF0kWCsWk)B zaCoUL>O{O<=~(`r%y_abPt+?g#R+q+(4V)Qfqb=In;3u(TzK1SqqjJMC&$twtTgkC z+tQPXY~--DDY~y%Jb2*Hf%x$5q6;=Pu61$J{d&BLV?l7X{V06z>6fu2?Uh@t za)MLkR3lM5Eg1-9jy+3Gmu3j86LU`8-B!yxXB}G9 z&dsPJ!sEO3NmTcaMGnfcvo=mHp=j>z}CIbH0WVmOL;ggk1lu^ z{+qk0M%Z6g>ifjkY`?+qH*)^ZD)k=krZ=UZ7+0L=mWL^STky9L3f!r3J%WFs)Gg4t zUctXt>Sr1lmGR2mRq7h<@VOz2JFvFD;?-rV#fEIYySza> zOuyuPmS-++8q)9bW>&Dv8_ds)CuW{3Se)ez{5SqGkKn({vmEr8ytyR(#LSy39;UpT zpX5!JSCThbUP<0WcVxWFo8=Z~dDCO#$-KE9p>N63fgs=88S$)eeEKdW>H{JCNDC89 z#P46n=UiO*7YN~-Ty}f~yPX-xh=^+|e6x3?hoAe7Z`R)N^NW15Cc;;fn1J)&@#xnA zHW|Lh+sX={ZQyseajm^ifzR0bg?eB1o;eBH@*XBtUTH~nmOa}X3(6k!}~$wo#yhdvkaR4I{gjMjF({Zn<``FF!lL( zH!$~*>*sv^e&&mhX9LW5`2F(%J|Eyq0lpgG8v(u*;5+R&S{~ru0Otd|HNblVd^Et5 z0iF%;nE=lR_9-0}eT1~?z!tpP3qd+VPY2{7}?@1G0sxd1N& z_(FiM1o&EjZwB~wJ2u_)37U1^8xw zZ?|K!O}+V>^TfwGkRJ+gA;84|j|8~k@KdXLS3bS^+4kRO?*C0T*i5SOLma7Hhh7@w zuW}FYDQP~(%;(pAe%o{7v(G#KQ;#T_c4f{l3Jxv zHJF~7Hm!UYn)zTBYX7cX!|_%&8{-!(`Xrm7c#~xgn|3(fD3Hy|k zFox>J2kX#z$w=GZ)P*#SKmLp0IDf+NH)W77PnPfJb>OA!-=r`~INr&#?+&Vu$&=Fg zM@u}!9P^8>?{DZ>D*jw|mUF0#zx%(?&6n}th`NOR$F86crE?AszxehC(XoX6-Z?v{ zvxt`VdADNm?f<-fcfX1Ioayo(tFH-i_jcFm`^y;l>)~Y3t|(YwzI!+(oPY1X0mtFu A(f|Me diff --git a/client/internal/ebpf/ebpf/bpf_bpfel.go b/client/internal/ebpf/ebpf/bpf_bpfel.go deleted file mode 100644 index f56efc901..000000000 --- a/client/internal/ebpf/ebpf/bpf_bpfel.go +++ /dev/null @@ -1,148 +0,0 @@ -// Code generated by bpf2go; DO NOT EDIT. -//go:build 386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64 || wasm - -package ebpf - -import ( - "bytes" - _ "embed" - "fmt" - "io" - - "github.com/cilium/ebpf" -) - -// loadBpf returns the embedded CollectionSpec for bpf. -func loadBpf() (*ebpf.CollectionSpec, error) { - reader := bytes.NewReader(_BpfBytes) - spec, err := ebpf.LoadCollectionSpecFromReader(reader) - if err != nil { - return nil, fmt.Errorf("can't load bpf: %w", err) - } - - return spec, err -} - -// loadBpfObjects loads bpf and converts it into a struct. -// -// The following types are suitable as obj argument: -// -// *bpfObjects -// *bpfPrograms -// *bpfMaps -// -// See ebpf.CollectionSpec.LoadAndAssign documentation for details. -func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { - spec, err := loadBpf() - if err != nil { - return err - } - - return spec.LoadAndAssign(obj, opts) -} - -// bpfSpecs contains maps and programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfSpecs struct { - bpfProgramSpecs - bpfMapSpecs - bpfVariableSpecs -} - -// bpfProgramSpecs contains programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfProgramSpecs struct { - NbXdpProg *ebpf.ProgramSpec `ebpf:"nb_xdp_prog"` -} - -// bpfMapSpecs contains maps before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfMapSpecs struct { - NbFeatures *ebpf.MapSpec `ebpf:"nb_features"` - NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"` -} - -// bpfVariableSpecs contains global variables before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfVariableSpecs struct { - FlagFeatureWgProxy *ebpf.VariableSpec `ebpf:"flag_feature_wg_proxy"` - MapKeyFeatures *ebpf.VariableSpec `ebpf:"map_key_features"` - MapKeyProxyPort *ebpf.VariableSpec `ebpf:"map_key_proxy_port"` - MapKeyWgPort *ebpf.VariableSpec `ebpf:"map_key_wg_port"` - ProxyPort *ebpf.VariableSpec `ebpf:"proxy_port"` - WgPort *ebpf.VariableSpec `ebpf:"wg_port"` -} - -// bpfObjects contains all objects after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfObjects struct { - bpfPrograms - bpfMaps - bpfVariables -} - -func (o *bpfObjects) Close() error { - return _BpfClose( - &o.bpfPrograms, - &o.bpfMaps, - ) -} - -// bpfMaps contains all maps after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfMaps struct { - NbFeatures *ebpf.Map `ebpf:"nb_features"` - NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"` -} - -func (m *bpfMaps) Close() error { - return _BpfClose( - m.NbFeatures, - m.NbWgProxySettingsMap, - ) -} - -// bpfVariables contains all global variables after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfVariables struct { - FlagFeatureWgProxy *ebpf.Variable `ebpf:"flag_feature_wg_proxy"` - MapKeyFeatures *ebpf.Variable `ebpf:"map_key_features"` - MapKeyProxyPort *ebpf.Variable `ebpf:"map_key_proxy_port"` - MapKeyWgPort *ebpf.Variable `ebpf:"map_key_wg_port"` - ProxyPort *ebpf.Variable `ebpf:"proxy_port"` - WgPort *ebpf.Variable `ebpf:"wg_port"` -} - -// bpfPrograms contains all programs after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfPrograms struct { - NbXdpProg *ebpf.Program `ebpf:"nb_xdp_prog"` -} - -func (p *bpfPrograms) Close() error { - return _BpfClose( - p.NbXdpProg, - ) -} - -func _BpfClose(closers ...io.Closer) error { - for _, closer := range closers { - if err := closer.Close(); err != nil { - return err - } - } - return nil -} - -// Do not access this directly. -// -//go:embed bpf_bpfel.o -var _BpfBytes []byte diff --git a/client/internal/ebpf/ebpf/bpf_bpfel.o b/client/internal/ebpf/ebpf/bpf_bpfel.o deleted file mode 100644 index a388b6d6d747827318287b94a4d069e8b15748e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8712 zcmds6VQf`N6`q#vZVMDjcgw=U%4Na&+J*88$dVAqTZFY3vOE!7*0A2b_T9G6zP|T( z_ZEt*ZnIfUqlWMU6C}p%4--kuCM23r{ZmXY93jsQjMRQ=QEnks1|6h^OV=s z&+C!<78+Gt2yl$H^3)}VXU|m{RYF+w4)6>1i&i75HrrQlO z{k-Y@eHGRid0X96nptXmTKqp*-?pfp@0X2Fn|e{}+kNd zahQ+SK0P6qYcTzQnLpiU`e(3sC)n++d$iz8Zl&q_nsq;j@kVgRYWeEc_)N|ez{$ED ziYd4GU_4kiZFSr9eBVzShyUC9=rb3nVNM@s7#`nKwRxZTX3uMZ>l>x0US zKN|_@Q_MX3=`X)qOG(2NuE3G}idY=3j zAb`F1o&z6%mt`8Z3Xi25wh7PrN`q(5)3sOl0Q^4T)8G$@eg=Gx@IBza2z~|nEE6Fq1U1sf(}uTxtFOrzPwN@s8J}~a$Gy^*Z##N`jNMc8lVnbaQWv2k*PfO`r-3?` zAz!5>N?jE`0G^aJSA+kF@U-=T;p>>M0FCsxwTzak=PmY2%x7F_`&ZztpEJVK#_xp> zz}vW`!INOCnYZyrW*v14J&*lu^dRe39cvK(Rx-Cj^{kuy%XFR_$R7gF{gI9cAAmnD zd!Uy2537-M~GvRx|PYXW|{TkFhr!~9RG)my#$`R zE6l;D`MgoET{~s4ty@kZAp%OZqovSTFWW`W*3PT4&Sx;jNnZd)esxL$U=py}tU1FU zgAV3vosA;rdgd3`*xBOfZ*_Fuz+BkTb2}V)=-AyPa>oDD4zI$4<#JI~qg+0$l&eV? zM@dpB4aTY~iN+GuwIiILYR;uN(^IiQa0G-VhN26eW&{r7G(l zjilO3eW6vaEk>DScs2Uf_C6$z8v?MG>B~{3)fog6rEDsE*Bq=KR zo0R(z$^N0Je^`a#=w3BiiVK6KC?E6><*E?%McsQig1@z)cnHow7>|xXoXh8{v5MKL zy#C8syLNIG{?=v{N{JfHYX{@-H66kFP zL1zI85|LRs=tOVR!34P}*&n2Xx8DwG;RAM63zh8$wEw|o6WL;FXZNo3E9pJEckSwM zHDP1>0ka3{+U%GjFd+*{(YY`mC$<_)rslnoRO*#<^pVZ9DHdk|l(AhvYcgO)$#r4B zHfD95ajIio*Oh%bz$1BND#__TVN^0G9El*pF{(y!92N!&rF=A|s$=1q(P%UZ4U86! z_9tTsXNNh$-Uhk2oIXOFJZ8{QunFXZk$ljJ`8KcvQ7##UX*j&p5w$ALsk9>ZOx9#_ zEYFq;FvSgXuUJ22Bps<@xir`TADDPEYN9tkgEz<0GmJLtjPrw+gShbZs61eI86`tF zjvbH4b@@IjMozW*fxvpxaY5fJG=R0X9~fekld>()ICc0$PnZplWEYet3t8*oXjAn_ zHhcWoiDTi(Ls=VauGneg6to8eMO+JlYsZhukKTSYk(9l%*~&+_RnAu;#oLmBQ2N?4 z>_~15!B47%DAo(mV9e6tD!io^T)VuPAmW{;teHh!tT|)%!X<0GTw$*sKmO{Q;g^nl z<#2f7&FtYYRAI5uAC=-rb@j#g(WsUgaAk$qbF80j1NqJ2n+e-){J8NMm&9kK7tr{d zNa7Fd3>sgzuA1i!a7KsE4b-*2X94(bwB`(+W6=j_@1ivw!n3U4%jjEAD3x(=3-Bqy zAE0kOt<<<+J-=6}3BmtDzq+W@6$dv1PYLFGR^6EKIe_dhz+CJ$x&UnpHrDi>QgaSo z4;(;`0w|%AbyLu zg7+EG|1R+U@0q&bIB^p&meA|n2VOO4>V)IOO~4Z}{yd&9n-Pag4rW|fH`G6X@um;) z4&vZvfv-832SvQk{ZQWdRFze>JU|Tm+4rbllbTH#HBiPo>J;AnaY+c#9c_8DqZW_J1Vcjv_ zx^8YE2>*qa)oz`6rB!NS< zZ2x7^NBA|_p6<~f>&LkG`wK0%vz5LNI1xuq`)U2N{RJ(5T7P4po}J0}eEM%awRZoY z<$UuqWc4kk8~k?BA_C%-Xc-6dJHsglQ-8?8w0*_FdryR`M;j;#b z`rPkP!R+|WCGTMRTX8VswGE9#F(`!`D1K<>46*-}dl5 z4?plQe?Q#)+VA<+|CBe*=Mq;x>tN=;*TWSLk9+u%hbKLJ-NQFMJnP}R9-i~CJ_zuW z`=h@t&iNt^Je>A$#=}_;_jJx2zt1fGPS#b8it-_j+`$vCb@8j*6TC|5*D?M2waB+U zJHPgN=f3LECO+E~@B)81z|o>2Q}3cI&=7+$k`vQ}l`c+`()qg$3*UQ9iiZ-!VHV*Mx=odoN|o zp-UM7J#x|{#PNUPyYP7qUh`U ze%`{|#q2LNpSmvbchi?~{9`eE$NnduO!EEtJLHSSe@^0mN#-&BT*{CCPRJIsf0GkH z -#include // ETH_P_IP -#include -#include -#include -#include -#include -#include "wg_proxy.c" - -const __u16 flag_feature_wg_proxy = 0b01; - -const __u32 map_key_features = 0; -struct bpf_map_def SEC("maps") nb_features = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(__u16), - .max_entries = 10, -}; - -SEC("xdp") -int nb_xdp_prog(struct xdp_md *ctx) { - __u16 *features; - features = bpf_map_lookup_elem(&nb_features, &map_key_features); - if (!features) { - return XDP_PASS; - } - - void *data = (void *)(long)ctx->data; - void *data_end = (void *)(long)ctx->data_end; - struct ethhdr *eth = data; - struct iphdr *ip = (data + sizeof(struct ethhdr)); - struct udphdr *udp = (data + sizeof(struct ethhdr) + sizeof(struct iphdr)); - - // return early if not enough data - if (data + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr) > data_end){ - return XDP_PASS; - } - - // skip non IPv4 packages - if (eth->h_proto != htons(ETH_P_IP)) { - return XDP_PASS; - } - - // skip non UPD packages - if (ip->protocol != IPPROTO_UDP) { - return XDP_PASS; - } - - if (*features & flag_feature_wg_proxy) { - xdp_wg_proxy(ip, udp); - } - return XDP_PASS; -} -char _license[] SEC("license") = "GPL"; diff --git a/client/internal/ebpf/ebpf/src/readme.md b/client/internal/ebpf/ebpf/src/readme.md deleted file mode 100644 index aa47847da..000000000 --- a/client/internal/ebpf/ebpf/src/readme.md +++ /dev/null @@ -1,27 +0,0 @@ -# XDP programs - -`prog.c` is attached to the `lo` device and dispatches to the features enabled in the -`nb_features` map. The only feature is the WireGuard proxy (`wg_proxy.c`): it rewrites -loopback UDP sent from the WireGuard listen port so it reaches the userspace relay proxy -port instead, and swaps the peer endpoint port into the source so the proxy can tell -peers apart. - -Maps use the legacy `struct bpf_map_def` form, defined in `bpf_map_def.h` because libbpf -1.0 removed it. They load on kernels built without BTF, which BTF-style (`SEC(".maps")`) -definitions do not. - -Regenerate the objects with `go generate ./client/internal/ebpf/ebpf/`; it needs -`clang-14`. Loading a regenerated object needs root, attaching it needs `bpf_link` -(kernel >= 5.7), and only one XDP program can own `lo` at a time. - -# Debug - -The CONFIG_BPF_EVENTS kernel module is required for bpf_printk. -Apply this code to use bpf_printk -``` -#define bpf_printk(fmt, ...) \ - ({ \ - char ____fmt[] = fmt; \ - bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ - }) -``` diff --git a/client/internal/ebpf/ebpf/src/wg_proxy.c b/client/internal/ebpf/ebpf/src/wg_proxy.c deleted file mode 100644 index 5e7474928..000000000 --- a/client/internal/ebpf/ebpf/src/wg_proxy.c +++ /dev/null @@ -1,60 +0,0 @@ -const __u32 map_key_proxy_port = 0; -const __u32 map_key_wg_port = 1; - -struct bpf_map_def SEC("maps") nb_wg_proxy_settings_map = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(__u16), - .max_entries = 10, -}; - -__u16 proxy_port = 0; -__u16 wg_port = 0; - -bool read_port_settings() { - __u16 *value; - value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_proxy_port); - if (!value) { - return false; - } - - proxy_port = *value; - - value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_wg_port); - if (!value) { - return false; - } - wg_port = htons(*value); - - return true; -} - -int xdp_wg_proxy(struct iphdr *ip, struct udphdr *udp) { - if (proxy_port == 0 || wg_port == 0) { - if (!read_port_settings()){ - return XDP_PASS; - } - // bpf_printk("proxy port: %d, wg port: %d", proxy_port, wg_port); - } - - // 2130706433 = 127.0.0.1 - if (ip->daddr != htonl(2130706433)) { - return XDP_PASS; - } - - if (udp->source != wg_port){ - return XDP_PASS; - } - - __be16 new_src_port = udp->dest; - __be16 new_dst_port = htons(proxy_port); - udp->dest = new_dst_port; - udp->source = new_src_port; - - // The ports are covered by the UDP checksum. This is an IPv4 loopback hop - // and the payload is already integrity-protected, so clear the checksum (a - // zero UDP checksum means "not computed" for IPv4) rather than leave a - // stale value the kernel would drop as UDP_CSUM. - udp->check = 0; - return XDP_PASS; -} diff --git a/client/internal/ebpf/ebpf/wg_proxy_linux.go b/client/internal/ebpf/ebpf/wg_proxy_linux.go deleted file mode 100644 index 4e0df7329..000000000 --- a/client/internal/ebpf/ebpf/wg_proxy_linux.go +++ /dev/null @@ -1,41 +0,0 @@ -package ebpf - -import log "github.com/sirupsen/logrus" - -const ( - mapKeyProxyPort uint32 = 0 - mapKeyWgPort uint32 = 1 -) - -func (tf *GeneralManager) LoadWgProxy(proxyPort, wgPort int) error { - log.Debugf("load ebpf WG proxy") - tf.lock.Lock() - defer tf.lock.Unlock() - - err := tf.loadXdp() - if err != nil { - return err - } - - err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyProxyPort, uint16(proxyPort)) - if err != nil { - return err - } - - err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyWgPort, uint16(wgPort)) - if err != nil { - return err - } - - tf.setFeatureFlag(featureFlagWGProxy) - err = tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags) - if err != nil { - return err - } - return nil -} - -func (tf *GeneralManager) FreeWGProxy() error { - log.Debugf("free ebpf WG proxy") - return tf.unsetFeatureFlag(featureFlagWGProxy) -} diff --git a/client/internal/ebpf/instantiater_linux.go b/client/internal/ebpf/instantiater_linux.go deleted file mode 100644 index 20d8145b4..000000000 --- a/client/internal/ebpf/instantiater_linux.go +++ /dev/null @@ -1,15 +0,0 @@ -//go:build !android - -package ebpf - -import ( - "github.com/netbirdio/netbird/client/internal/ebpf/ebpf" - "github.com/netbirdio/netbird/client/internal/ebpf/manager" -) - -// GetEbpfManagerInstance is a wrapper function. This encapsulation is required because if the code import the internal -// ebpf package the Go compiler will include the object files. But it is not supported on Android. It can cause instant -// panic on older Android version. -func GetEbpfManagerInstance() manager.Manager { - return ebpf.GetEbpfManagerInstance() -} diff --git a/client/internal/ebpf/instantiater_nonlinux.go b/client/internal/ebpf/instantiater_nonlinux.go deleted file mode 100644 index b7c38733a..000000000 --- a/client/internal/ebpf/instantiater_nonlinux.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !linux || android - -package ebpf - -import "github.com/netbirdio/netbird/client/internal/ebpf/manager" - -// GetEbpfManagerInstance return error because ebpf is not supported on all os -func GetEbpfManagerInstance() manager.Manager { - panic("unsupported os") -} diff --git a/client/internal/ebpf/manager/manager.go b/client/internal/ebpf/manager/manager.go deleted file mode 100644 index fdc5d8d82..000000000 --- a/client/internal/ebpf/manager/manager.go +++ /dev/null @@ -1,7 +0,0 @@ -package manager - -// Manager is used to load multiple eBPF programs. E.g., the WireGuard proxy -type Manager interface { - LoadWgProxy(proxyPort, wgPort int) error - FreeWGProxy() error -} diff --git a/client/internal/engine.go b/client/internal/engine.go index 1613bbbc9..fc7dce869 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -661,10 +661,6 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) } e.wgDevice.Store(e.wgInterface.GetWGDevice()) - // Set up notrack rules immediately after proxy is listening to prevent - // conntrack entries from being created before the rules are in place - e.setupWGProxyNoTrack() - // Start after interface is up since port may have been resolved from 0 or changed if occupied e.shutdownWg.Add(1) go func() { @@ -802,23 +798,6 @@ func (e *Engine) initFirewall() error { return nil } -// setupWGProxyNoTrack configures connection tracking exclusion for WireGuard proxy traffic. -// This prevents conntrack/MASQUERADE from affecting loopback traffic between WireGuard and the eBPF proxy. -func (e *Engine) setupWGProxyNoTrack() { - if e.firewall == nil { - return - } - - proxyPort := e.wgInterface.GetProxyPort() - if proxyPort == 0 { - return - } - - if err := e.firewall.SetupEBPFProxyNoTrack(proxyPort, uint16(e.config.WgPort)); err != nil { - log.Warnf("failed to setup ebpf proxy notrack: %v", err) - } -} - func (e *Engine) blockLanAccess() { if e.config.BlockInbound { // no need to set up extra deny rules if inbound is already blocked in general diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index ec388ac94..2a7ecd652 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -65,7 +65,6 @@ type MockWGIface struct { GetStatsFunc func() (map[string]configurer.WGStats, error) GetInterfaceGUIDStringFunc func() (string, error) GetProxyFunc func() wgproxy.Proxy - GetProxyPortFunc func() uint16 GetNetFunc func() *netstack.Net LastActivitiesFunc func() map[string]monotime.Time } @@ -162,12 +161,6 @@ func (m *MockWGIface) GetProxy() wgproxy.Proxy { return m.GetProxyFunc() } -func (m *MockWGIface) GetProxyPort() uint16 { - if m.GetProxyPortFunc != nil { - return m.GetProxyPortFunc() - } - return 0 -} func (m *MockWGIface) GetNet() *netstack.Net { return m.GetNetFunc() diff --git a/client/internal/iface_common.go b/client/internal/iface_common.go index 8ffa0b102..d772a3a03 100644 --- a/client/internal/iface_common.go +++ b/client/internal/iface_common.go @@ -28,7 +28,6 @@ type wgIfaceBase interface { Up() (*udpmux.UniversalUDPMuxDefault, error) UpdateAddr(newAddr wgaddr.Address) error GetProxy() wgproxy.Proxy - GetProxyPort() uint16 UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error RemoveEndpointAddress(key string) error RemovePeer(peerKey string) error diff --git a/docs/testing-privileged.md b/docs/testing-privileged.md index 72e8a0f8f..939770de9 100644 --- a/docs/testing-privileged.md +++ b/docs/testing-privileged.md @@ -1,7 +1,7 @@ # Privileged tests Some tests in this repo need `root` or mutate host network state: they create -TUN/WireGuard interfaces, open netlink/raw sockets, run eBPF programs, or shell +TUN/WireGuard interfaces, open netlink/raw sockets, or shell out to `ip`/`iptables`/`nft`/`ifconfig`/`route`. Running them on a developer machine would require `sudo` and could leave stray interfaces or routes behind. @@ -44,7 +44,6 @@ A test is privileged if it does any of: - creates a real interface via `iface.NewWGIFace(...).Create()`, - opens a netlink or raw socket that hard-fails without `CAP_NET_ADMIN`, -- runs an eBPF program (`ebpf.*.Listen()`), - shells out to `ip`, `iptables`, `nft`, `ifconfig`, or `route` to change state. Add the tag to the **top** of the file, combined with any existing platform diff --git a/go.mod b/go.mod index 2d0653ec0..35e254f8e 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,6 @@ require ( github.com/c-robinson/iplib v1.0.3 github.com/caarlos0/env/v11 v11.4.1 github.com/caddyserver/certmagic v0.21.3 - github.com/cilium/ebpf v0.19.0 github.com/coder/websocket v1.8.14 github.com/coreos/go-iptables v0.7.0 github.com/coreos/go-oidc/v3 v3.18.0 @@ -190,6 +189,7 @@ require ( github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cilium/ebpf v0.19.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect diff --git a/go.sum b/go.sum index e65c32f26..c193a388f 100644 --- a/go.sum +++ b/go.sum @@ -407,8 +407,6 @@ github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbd github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= -github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= From 8ab34fcf8ba693de2f504154a2fa24e003fa8e17 Mon Sep 17 00:00:00 2001 From: Brad Ison Date: Wed, 30 Sep 2026 11:05:45 +0200 Subject: [PATCH 65/75] [proxy] Apply the upstream HTTP version before cloning transports (#7806) createClientEntry and NewMultiTransport clone the secure transport into its insecure variant before newUpstreamTransport applies the configured HTTP version. http.Transport.Clone runs the source's one-time protocol setup, and at that point ForceAttemptHTTP2 is still false while a custom DialContext is set, so net/http disables HTTP/2 on the source for good. Setting ForceAttemptHTTP2 afterwards has no effect. As a result every TLS-verified upstream has been served over HTTP/1.1 since the upstream HTTP version became configurable, whatever NB_PROXY_UPSTREAM_HTTP_VERSION says, while skip-TLS-verify upstreams kept HTTP/2. gRPC upstreams break outright: unary calls get a 502 and streaming calls hang until the client gives up. Apply the version to the base transport before cloning it, and add a test that the direct and insecure transports both offer h2. --- proxy/internal/roundtrip/clone_http2_test.go | 40 ++++++++++++++++++++ proxy/internal/roundtrip/multi.go | 3 ++ proxy/internal/roundtrip/netbird.go | 3 ++ 3 files changed, 46 insertions(+) create mode 100644 proxy/internal/roundtrip/clone_http2_test.go diff --git a/proxy/internal/roundtrip/clone_http2_test.go b/proxy/internal/roundtrip/clone_http2_test.go new file mode 100644 index 000000000..f2707ff19 --- /dev/null +++ b/proxy/internal/roundtrip/clone_http2_test.go @@ -0,0 +1,40 @@ +package roundtrip + +import ( + "net/http" + "testing" + + log "github.com/sirupsen/logrus" +) + +// offersHTTP2 reports whether t will negotiate h2 with a TLS upstream. +// Clone forces t's one-time protocol setup, which registers an "h2" +// handler in t.TLSNextProto only when HTTP/2 ended up enabled. +func offersHTTP2(t *http.Transport) bool { + _ = t.Clone() + _, ok := t.TLSNextProto["h2"] + return ok +} + +func TestNewMultiTransportKeepsHTTP2AcrossClone(t *testing.T) { + for _, tc := range []struct { + version string + want bool + }{ + {string(upstreamHTTPAuto), true}, + {string(upstreamHTTP2), true}, + {string(upstreamHTTP11), false}, + } { + t.Run(tc.version, func(t *testing.T) { + t.Setenv(EnvUpstreamHTTPVersion, tc.version) + m := NewMultiTransport(noEmbeddedRoundTripper{}, log.New()) + + if got := offersHTTP2(m.direct.primary); got != tc.want { + t.Errorf("direct transport offers h2 = %v, want %v", got, tc.want) + } + if got := offersHTTP2(m.insecure.primary); got != tc.want { + t.Errorf("insecure transport offers h2 = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/proxy/internal/roundtrip/multi.go b/proxy/internal/roundtrip/multi.go index 1abf54a8d..d50ad1fc9 100644 --- a/proxy/internal/roundtrip/multi.go +++ b/proxy/internal/roundtrip/multi.go @@ -64,6 +64,9 @@ func NewMultiTransport(embedded http.RoundTripper, logger *log.Logger) *MultiTra ReadBufferSize: cfg.readBufferSize, DisableCompression: cfg.disableCompression, } + // Clone runs the transport's one-time protocol setup, so the HTTP + // version must be applied first or the source loses HTTP/2 for good. + applyUpstreamHTTPVersion(direct, cfg.upstreamHTTPVersion) insecure := direct.Clone() insecure.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // matches the embedded NetBird transport's per-target opt-in diff --git a/proxy/internal/roundtrip/netbird.go b/proxy/internal/roundtrip/netbird.go index d7b464182..07b497c46 100644 --- a/proxy/internal/roundtrip/netbird.go +++ b/proxy/internal/roundtrip/netbird.go @@ -425,6 +425,9 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account ReadBufferSize: n.transportCfg.readBufferSize, DisableCompression: n.transportCfg.disableCompression, } + // Clone runs the transport's one-time protocol setup, so the HTTP + // version must be applied first or the source loses HTTP/2 for good. + applyUpstreamHTTPVersion(transport, n.transportCfg.upstreamHTTPVersion) insecureTransport := transport.Clone() insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec From 96bfcc360008b0c12e41b39cb3f7c575ab49ac42 Mon Sep 17 00:00:00 2001 From: Edward <43848523+thomashacker@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:10:14 +0200 Subject: [PATCH 66/75] [client] Classify Windows local accounts by NetBIOS name (#7628) --- client/ssh/server/executor_windows.go | 35 ++++++++++++---- client/ssh/server/executor_windows_test.go | 48 ++++++++++++++++++++++ 2 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 client/ssh/server/executor_windows_test.go diff --git a/client/ssh/server/executor_windows.go b/client/ssh/server/executor_windows.go index 51c995ec3..9c2969d5d 100644 --- a/client/ssh/server/executor_windows.go +++ b/client/ssh/server/executor_windows.go @@ -6,7 +6,6 @@ import ( "context" "errors" "fmt" - "os" "os/exec" "os/user" "strings" @@ -506,15 +505,37 @@ func userExists(fullUsername, username, domain string) error { return nil } -// isLocalUser determines if this is a local user vs domain user +// isLocalUser reports whether domain refers to this machine rather than to a +// Windows domain. func (pd *PrivilegeDropper) isLocalUser(domain string) bool { - hostname, err := os.Hostname() - if err != nil { - hostname = "localhost" + return isLocalDomain(domain, netbiosComputerName) +} + +// isLocalDomain compares against the NetBIOS name because Windows qualifies local +// accounts with it, and it is the DNS host name truncated to 15 characters. +// An unknown name falls back to the domain path: treating it as local could +// authenticate a same named local account instead. +// https://learn.microsoft.com/en-us/windows/win32/sysinfo/computer-names +func isLocalDomain(domain string, machineName func() (string, error)) bool { + if domain == "" || domain == "." { + return true } - return domain == "" || domain == "." || - strings.EqualFold(domain, hostname) + name, err := machineName() + if err != nil { + log.Debugf("read NetBIOS computer name: %v", err) + return false + } + return strings.EqualFold(domain, name) +} + +func netbiosComputerName() (string, error) { + buf := make([]uint16, windows.MAX_COMPUTERNAME_LENGTH+1) + size := uint32(len(buf)) + if err := windows.GetComputerNameEx(windows.ComputerNamePhysicalNetBIOS, &buf[0], &size); err != nil { + return "", fmt.Errorf("GetComputerNameEx: %w", err) + } + return windows.UTF16ToString(buf[:size]), nil } // authenticateLocalUser handles authentication for local users diff --git a/client/ssh/server/executor_windows_test.go b/client/ssh/server/executor_windows_test.go new file mode 100644 index 000000000..678ca22b7 --- /dev/null +++ b/client/ssh/server/executor_windows_test.go @@ -0,0 +1,48 @@ +//go:build windows + +package server + +import ( + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +// Past 15 characters the DNS host name and the NetBIOS name differ, and Windows +// qualifies local accounts with the NetBIOS one. +func TestIsLocalDomain(t *testing.T) { + const dnsHostname = "WINTESTMACHINE01XYZ" // 19 characters + netbios := dnsHostname[:windows.MAX_COMPUTERNAME_LENGTH] + require.NotEqual(t, strings.ToLower(dnsHostname), strings.ToLower(netbios), + "a 19 character name must not equal its 15 character truncation") + + name := func() (string, error) { return netbios, nil } + unreadable := func() (string, error) { return "", errors.New("name unavailable") } + + tests := []struct { + name string + domain string + machineName func() (string, error) + want bool + }{ + {"empty_domain", "", unreadable, true}, + {"dot_domain", ".", unreadable, true}, + {"truncated_netbios_name", netbios, name, true}, + {"netbios_name_lowercase", strings.ToLower(netbios), name, true}, + {"untruncated_dns_host_name", dnsHostname, name, false}, + {"real_domain", "CORP", name, false}, + // Must not resolve to local: that could authenticate the wrong account. + {"unreadable_machine_name", netbios, unreadable, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isLocalDomain(tt.domain, tt.machineName), + "classification of domain %q", tt.domain) + }) + } +} From 39de33ceca93c2a0ea07756b5ca9ffd6bb4eb496 Mon Sep 17 00:00:00 2001 From: Pascal Fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:35:18 +0200 Subject: [PATCH 67/75] [management] handle db conn close on errpr (#7740) --- management/internals/shared/db/open.go | 22 +++++++++++++++++++--- management/server/store/store.go | 21 +++++---------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/management/internals/shared/db/open.go b/management/internals/shared/db/open.go index 020aff422..48a7bf330 100644 --- a/management/internals/shared/db/open.go +++ b/management/internals/shared/db/open.go @@ -93,7 +93,12 @@ func OpenSqliteFile(ctx context.Context, dataDir, storeFile string) (*Conn, erro if err != nil { return nil, err } - return NewConn(ctx, gormDB, SqliteStoreEngine, nil) + conn, err := NewConn(ctx, gormDB, SqliteStoreEngine, nil) + if err != nil { + closeGorm(gormDB) + return nil, err + } + return conn, nil } // OpenPostgres opens a Postgres database through gorm and a pgx pool sized by pool. @@ -107,7 +112,13 @@ func OpenPostgres(ctx context.Context, dsn string, pool PoolConfig) (*Conn, erro closeGorm(gormDB) return nil, err } - return NewConn(ctx, gormDB, PostgresStoreEngine, pgxPool) + conn, err := NewConn(ctx, gormDB, PostgresStoreEngine, pgxPool) + if err != nil { + pgxPool.Close() + closeGorm(gormDB) + return nil, err + } + return conn, nil } // MysqlDSN adds the connection parameters every MySQL handle needs, keeping @@ -126,7 +137,12 @@ func OpenMysql(ctx context.Context, dsn string) (*Conn, error) { if err != nil { return nil, err } - return NewConn(ctx, gormDB, MysqlStoreEngine, nil) + conn, err := NewConn(ctx, gormDB, MysqlStoreEngine, nil) + if err != nil { + closeGorm(gormDB) + return nil, err + } + return conn, nil } func newPgxPool(ctx context.Context, dsn string, cfg PoolConfig) (*pgxpool.Pool, error) { diff --git a/management/server/store/store.go b/management/server/store/store.go index bd55e3176..6d810643c 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -13,7 +13,6 @@ import ( "path" "path/filepath" "regexp" - "runtime" "slices" "strings" "sync" @@ -23,7 +22,6 @@ import ( log "github.com/sirupsen/logrus" "gorm.io/driver/mysql" "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" "gorm.io/gorm" "github.com/netbirdio/netbird/dns" @@ -716,29 +714,19 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) ( kind = types.SqliteStoreEngine } - storeStr := fmt.Sprintf("%s?cache=shared", db.SqliteFileName) - if runtime.GOOS == "windows" { - // Vo avoid `The process cannot access the file because it is being used by another process` on Windows - storeStr = db.SqliteFileName - } - - file := filepath.Join(dataDir, storeStr) - gormDB, err := gorm.Open(sqlite.Open(file), db.GormConfig()) + conn, err := db.OpenSqliteFile(ctx, dataDir, db.SqliteFileName) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("failed to create test store: %v", err) } if filename != "" { - err = LoadSQL(gormDB, filename) + err = LoadSQL(conn.DB(nil), filename) if err != nil { + _ = conn.Close() return nil, nil, fmt.Errorf("failed to load SQL file: %v", err) } } - conn, err := db.NewConn(ctx, gormDB, db.SqliteStoreEngine, nil) - if err != nil { - return nil, nil, fmt.Errorf("failed to create test store: %v", err) - } store, err := NewSqlStore(ctx, conn, nil, false) if err != nil { _ = conn.Close() @@ -747,6 +735,7 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) ( err = addAllGroupToAccount(ctx, store) if err != nil { + _ = store.Close(ctx) return nil, nil, fmt.Errorf("failed to add all group to account: %v", err) } From e72be6698f7ad0d00c1c5c08e5b601599488e87f Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Wed, 30 Sep 2026 16:00:14 +0200 Subject: [PATCH 68/75] [client] Keep the advertised ICE session ID when following a remote restart (#7814) A worker that saw a new remote session ID rebuilt its agent and also picked a new local ID. On the answer path nothing carries that ID back, so the next offer made the remote see a changed session, rebuild, and answer with yet another ID. Two peers kept tearing down working ICE connections on every offer and answer; nearly every answer in the affected logs carried a new remote session ID. Only a local restart changes the local ID now: a failed negotiation, as before, and an explicit Close, which previously kept the old ID and left the remote answering from a negotiation this side had abandoned. Following a remote restart keeps the ID the remote already knows, so the pair settles after one rebuild, also against peers that still pick a new ID when following a restart. --- client/internal/peer/worker_ice.go | 27 +- .../internal/peer/worker_ice_session_test.go | 375 ++++++++++++++++++ 2 files changed, 392 insertions(+), 10 deletions(-) create mode 100644 client/internal/peer/worker_ice_session_test.go diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index d17f6e693..5979e9bdc 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -121,11 +121,8 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { } } - sessionID, err := NewICESessionID() - if err != nil { - w.log.Errorf("failed to create new session ID: %s", err) - } - w.sessionID = sessionID + // Keep the ID already advertised to the remote. Answers do not get a + // reply, so changing it here makes the next offer restart both sides. w.abandonNegotiation() } @@ -205,6 +202,9 @@ func (w *WorkerICE) Close() { w.muxAgent.Lock() defer w.muxAgent.Unlock() + if w.agent != nil || w.agentConnecting { + w.renewSessionID() + } if w.agent != nil { w.agentDialerCancel() if err := w.agent.Close(); err != nil { @@ -366,16 +366,23 @@ func (w *WorkerICE) closeAgent(agent *icemaker.ThreadSafeAgent, cancel context.C // Only the owner of the current session may reset its state: a stale dial // goroutine waking after a newer attempt must not clobber it. if w.agent == agent { - sessionID, err := NewICESessionID() - if err != nil { - w.log.Errorf("failed to create new session ID: %s", err) - } - w.sessionID = sessionID + w.renewSessionID() w.abandonNegotiation() } return sessionChanged } +// renewSessionID starts a new local session, so the remote treats our next offer +// or answer as a restart. Caller holds muxAgent. +func (w *WorkerICE) renewSessionID() { + sessionID, err := NewICESessionID() + if err != nil { + w.log.Errorf("failed to create new session ID: %s", err) + return + } + w.sessionID = sessionID +} + // abandonNegotiation drops all recorded ICE session state so the worker treats the // next offer as a fresh start instead of a duplicate of a dead negotiation. The // agent and agentConnecting flags must change together: leaving one stale wedges diff --git a/client/internal/peer/worker_ice_session_test.go b/client/internal/peer/worker_ice_session_test.go new file mode 100644 index 000000000..4858e0bc3 --- /dev/null +++ b/client/internal/peer/worker_ice_session_test.go @@ -0,0 +1,375 @@ +package peer + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + icemaker "github.com/netbirdio/netbird/client/internal/peer/ice" +) + +func TestWorkerICE_RemoteRestartPreservesAdvertisedSession(t *testing.T) { + w := newTestWorkerICE(t) + t.Cleanup(w.Close) + w.dialFunc = parkDial + advertised := w.SessionID() + remoteSession := ICESessionID("remote-first") + offer := OfferAnswer{ + IceCredentials: IceCredentials{UFrag: "remoteufrag", Pwd: "remote-password-long-enough"}, + SessionID: &remoteSession, + } + w.OnNewOffer(&offer) + require.True(t, w.InProgress(), "the first remote session must start ICE") + w.muxAgent.Lock() + firstAgent := w.agent + w.muxAgent.Unlock() + + // The same callback handles answers. A changed remote ID must not create + // an unannounced local ID that makes the remote restart on our next offer. + secondSession := ICESessionID("remote-restarted") + answer := offer + answer.SessionID = &secondSession + w.OnNewOffer(&answer) + assert.Equal(t, advertised, w.SessionID(), "following a remote restart must keep our advertised ID") + w.muxAgent.Lock() + secondAgent := w.agent + w.muxAgent.Unlock() + assert.NotSame(t, firstAgent, secondAgent, "the changed remote session must still rebuild ICE") + + w.OnNewOffer(&answer) + w.muxAgent.Lock() + defer w.muxAgent.Unlock() + assert.Same(t, secondAgent, w.agent, "a repeated answer must keep the replacement agent") +} + +func TestWorkerICE_LocalCloseChangesAdvertisedSession(t *testing.T) { + w := newTestWorkerICE(t) + dialStarted := make(chan struct{}) + dialDone := make(chan struct{}) + w.dialFunc = func(ctx context.Context, _ *icemaker.ThreadSafeAgent, _ *OfferAnswer) (net.Conn, error) { + close(dialStarted) + defer close(dialDone) + <-ctx.Done() + return nil, ctx.Err() + } + session := ICESessionID("remote-session") + w.OnNewOffer(&OfferAnswer{ + IceCredentials: IceCredentials{UFrag: "remoteufrag", Pwd: "remote-password-long-enough"}, + SessionID: &session, + }) + <-dialStarted + advertised := w.SessionID() + w.Close() + assert.NotEqual(t, advertised, w.SessionID(), "a local teardown must tell the remote to restart") + closedSession := w.SessionID() + + // The abandoned dial goroutine cleans up after Close returned. + <-dialDone + assert.Never(t, func() bool { return w.SessionID() != closedSession }, 200*time.Millisecond, 10*time.Millisecond, + "the late cleanup of a closed negotiation must not restart again") + w.Close() + assert.Equal(t, closedSession, w.SessionID(), "closing an idle worker must not restart again") +} + +// parkDial stands in for the ICE dial. It never connects and returns once the +// negotiation is abandoned, so a test decides when a negotiation fails. +func parkDial(ctx context.Context, _ *icemaker.ThreadSafeAgent, _ *OfferAnswer) (net.Conn, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func newTestSessionID(t *testing.T) ICESessionID { + t.Helper() + sid, err := NewICESessionID() + require.NoError(t, err) + return sid +} + +// handshakeSide is one end of a simulated signaling exchange. +type handshakeSide interface { + // message builds the offer or answer the side would send now. + message() OfferAnswer + // receive hands a remote offer or answer to the side's ICE logic. + receive(msg OfferAnswer) + // teardowns counts negotiations the side tore down to follow a remote restart. + teardowns() int + // failAgent ends the side's current negotiation as an ICE failure does. + failAgent() +} + +// workerSide drives a real WorkerICE. +type workerSide struct { + t *testing.T + w *WorkerICE + replaced int +} + +func newWorkerSide(t *testing.T) *workerSide { + t.Helper() + w := newTestWorkerICE(t) + w.dialFunc = parkDial + t.Cleanup(w.Close) + return &workerSide{t: t, w: w} +} + +func (s *workerSide) message() OfferAnswer { + sid := s.w.SessionID() + ufrag, pwd := s.w.GetLocalUserCredentials() + return OfferAnswer{IceCredentials: IceCredentials{UFrag: ufrag, Pwd: pwd}, SessionID: &sid} +} + +func (s *workerSide) receive(msg OfferAnswer) { + before := s.agent() + s.w.OnNewOffer(&msg) + if after := s.agent(); before != nil && after != before { + s.replaced++ + } +} + +func (s *workerSide) teardowns() int { return s.replaced } + +func (s *workerSide) agent() *icemaker.ThreadSafeAgent { + s.w.muxAgent.Lock() + defer s.w.muxAgent.Unlock() + return s.w.agent +} + +// failAgent runs the cleanup the dial goroutine or the Failed state callback +// performs when the current negotiation dies. +func (s *workerSide) failAgent() { + s.t.Helper() + s.w.muxAgent.Lock() + agent, cancel := s.w.agent, s.w.agentDialerCancel + s.w.muxAgent.Unlock() + require.NotNil(s.t, agent, "failing requires a running negotiation") + s.w.closeAgent(agent, cancel) +} + +// legacySide models a remote peer running a release from before this change: +// when it follows a remote restart it also picks a new session ID of its own, +// which it announces only with its next offer or answer. +type legacySide struct { + t *testing.T + sessionID ICESessionID + remoteID ICESessionID + hasAgent bool + replaced int +} + +func newLegacySide(t *testing.T) *legacySide { + return &legacySide{t: t, sessionID: newTestSessionID(t)} +} + +func (s *legacySide) message() OfferAnswer { + sid := s.sessionID + return OfferAnswer{ + IceCredentials: IceCredentials{UFrag: "legacyufrag", Pwd: "legacy-password-long-enough"}, + SessionID: &sid, + } +} + +func (s *legacySide) receive(msg OfferAnswer) { + if msg.SessionID == nil { + s.hasAgent = true + return + } + if s.hasAgent { + if *msg.SessionID == s.remoteID { + return + } + s.replaced++ + s.sessionID = newTestSessionID(s.t) + } + s.hasAgent = true + s.remoteID = *msg.SessionID +} + +func (s *legacySide) teardowns() int { return s.replaced } + +func (s *legacySide) failAgent() { + s.hasAgent = false + s.remoteID = "" + s.sessionID = newTestSessionID(s.t) +} + +// exchange runs one guard-driven round in the order Handshaker.Listen uses: the +// answerer handles the offer and answers with the session ID it holds +// afterwards, and the offerer handles the answer without replying. +func exchange(offerer, answerer handshakeSide) { + answerer.receive(offerer.message()) + offerer.receive(answerer.message()) +} + +// offerPattern decides which side's guard sends the offer in a round. +type offerPattern struct { + name string + picker func(round int, local, remote handshakeSide) (offerer, answerer handshakeSide) +} + +var offerPatterns = []offerPattern{ + { + // A routing peer whose relay is down keeps offering on its own. + name: "local peer offers", + picker: func(_ int, local, remote handshakeSide) (handshakeSide, handshakeSide) { + return local, remote + }, + }, + { + name: "both peers offer", + picker: func(round int, local, remote handshakeSide) (handshakeSide, handshakeSide) { + if round%2 == 0 { + return local, remote + } + return remote, local + }, + }, +} + +// assertSettles runs guard rounds and requires the pair to stop restarting +// each other: at most maxTeardowns in total, and none once half the rounds ran. +func assertSettles(t *testing.T, pattern offerPattern, local, remote handshakeSide, maxTeardowns int) { + t.Helper() + const rounds = 10 + + total := func() int { return local.teardowns() + remote.teardowns() } + start := total() + var halfway int + for round := range rounds { + if round == rounds/2 { + halfway = total() + } + offerer, answerer := pattern.picker(round, local, remote) + exchange(offerer, answerer) + } + + assert.LessOrEqual(t, total()-start, maxTeardowns, "the peers must not keep restarting each other") + assert.Equal(t, halfway, total(), "the negotiation must be stable in the later rounds") +} + +// establish runs the first offer and answer, so both sides negotiate. +func establish(t *testing.T, local, remote handshakeSide) { + t.Helper() + exchange(local, remote) + require.Zero(t, local.teardowns()+remote.teardowns(), "the first exchange must not restart anything") +} + +func TestICESession_SettlesAfterAgentFailure(t *testing.T) { + sides := []struct { + name string + remote func(t *testing.T) handshakeSide + }{ + {name: "current remote", remote: func(t *testing.T) handshakeSide { return newWorkerSide(t) }}, + {name: "legacy remote", remote: func(t *testing.T) handshakeSide { return newLegacySide(t) }}, + } + failures := []struct { + name string + fail func(local, remote handshakeSide) + }{ + {name: "remote agent fails", fail: func(_, remote handshakeSide) { remote.failAgent() }}, + {name: "local agent fails", fail: func(local, _ handshakeSide) { local.failAgent() }}, + {name: "both agents fail", fail: func(local, remote handshakeSide) { + local.failAgent() + remote.failAgent() + }}, + } + + for _, side := range sides { + for _, failure := range failures { + for _, pattern := range offerPatterns { + t.Run(fmt.Sprintf("%s/%s/%s", side.name, failure.name, pattern.name), func(t *testing.T) { + local := newWorkerSide(t) + remote := side.remote(t) + establish(t, local, remote) + + failure.fail(local, remote) + assertSettles(t, pattern, local, remote, 2) + }) + } + } + } +} + +// TestICESession_LocalCloseRestartsRemote covers an explicit teardown, as on a +// WireGuard handshake timeout. The remote must start over as well, or it keeps +// answering from the negotiation this side just abandoned. +func TestICESession_LocalCloseRestartsRemote(t *testing.T) { + for _, pattern := range offerPatterns { + t.Run(pattern.name, func(t *testing.T) { + local := newWorkerSide(t) + remote := newWorkerSide(t) + establish(t, local, remote) + + local.w.Close() + assertSettles(t, pattern, local, remote, 1) + assert.Equal(t, 1, remote.teardowns(), "the remote must restart its negotiation exactly once") + }) + } +} + +func TestICESession_DuplicateMessagesKeepNegotiation(t *testing.T) { + local := newWorkerSide(t) + remote := newWorkerSide(t) + + offer := local.message() + remote.receive(offer) + answer := remote.message() + local.receive(answer) + + // Signaling may deliver the same message again, and a peer answers every + // offer, including repeats of one it already handled. + remote.receive(offer) + local.receive(answer) + local.receive(remote.message()) + + assert.Zero(t, local.teardowns(), "a repeated answer must not restart the negotiation") + assert.Zero(t, remote.teardowns(), "a repeated offer must not restart the negotiation") +} + +// TestICESession_RemoteWithoutSessionIDKeepsNegotiation covers remote peers +// too old to send session IDs: once negotiating, their messages cannot tell a +// restart from a repeat, so they must not tear anything down. +func TestICESession_RemoteWithoutSessionIDKeepsNegotiation(t *testing.T) { + local := newWorkerSide(t) + unversioned := OfferAnswer{IceCredentials: IceCredentials{UFrag: "oldufrag", Pwd: "old-password-long-enough"}} + + local.receive(unversioned) + require.NotNil(t, local.agent(), "a message without a session ID must still start ICE") + advertised := local.w.SessionID() + + for range 3 { + local.receive(unversioned) + } + assert.Zero(t, local.teardowns(), "messages without a session ID must not restart the negotiation") + assert.Equal(t, advertised, local.w.SessionID(), "the advertised session must not change") +} + +// TestWorkerICE_StaleCleanupKeepsAdvertisedSession covers the cleanup of a +// replaced negotiation finishing late, from its dial goroutine or its Closed +// state callback. It must neither pick a new session ID, an unannounced local +// restart, nor disturb the negotiation that replaced it. +func TestWorkerICE_StaleCleanupKeepsAdvertisedSession(t *testing.T) { + local := newWorkerSide(t) + remote := newWorkerSide(t) + establish(t, local, remote) + + local.w.muxAgent.Lock() + oldAgent, oldCancel := local.w.agent, local.w.agentDialerCancel + local.w.muxAgent.Unlock() + + remote.failAgent() + exchange(local, remote) + require.Equal(t, 1, local.teardowns(), "the local side must follow the remote restart") + advertised := local.w.SessionID() + current := local.agent() + + local.w.closeAgent(oldAgent, oldCancel) + + assert.Equal(t, advertised, local.w.SessionID(), "a stale cleanup must not change the advertised session") + assert.Same(t, current, local.agent(), "a stale cleanup must keep the current negotiation") + assertSettles(t, offerPatterns[1], local, remote, 0) +} From 0dc729c4ead3dce2839cdc34738a7a99bdef71ad Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:03:04 +0900 Subject: [PATCH 69/75] [client] Cache the box shared key per remote peer in the Signal client (#7807) --- encryption/sharedkey.go | 147 +++++++++++++++++++++ encryption/sharedkey_limit.go | 8 ++ encryption/sharedkey_limit_mobile.go | 8 ++ encryption/sharedkey_test.go | 184 +++++++++++++++++++++++++++ shared/signal/client/grpc.go | 7 +- 5 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 encryption/sharedkey.go create mode 100644 encryption/sharedkey_limit.go create mode 100644 encryption/sharedkey_limit_mobile.go create mode 100644 encryption/sharedkey_test.go diff --git a/encryption/sharedkey.go b/encryption/sharedkey.go new file mode 100644 index 000000000..1509632c4 --- /dev/null +++ b/encryption/sharedkey.go @@ -0,0 +1,147 @@ +package encryption + +import ( + "fmt" + "sync" + + pb "github.com/golang/protobuf/proto" //nolint + "golang.org/x/crypto/nacl/box" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +// SharedKeyCache encrypts and decrypts messages for one local private key, deriving +// the box shared key once per remote public key instead of once per message. +// +// The shared key is a pure function of the two keys, so a cached entry never goes +// stale: a different remote key is a different entry, and a different local key +// needs a different cache. Entries are only dropped to stay under maxSharedKeys. +// Every message still uses its own random nonce. +// +// The cached values are secret key material, as sensitive as the private key. +type SharedKeyCache struct { + privateKey wgtypes.Key + limit int + + mu sync.RWMutex + keys map[wgtypes.Key]*[32]byte + closed bool +} + +// NewSharedKeyCache returns a cache for messages sent and received with privateKey. +func NewSharedKeyCache(privateKey wgtypes.Key) *SharedKeyCache { + return &SharedKeyCache{ + privateKey: privateKey, + limit: maxSharedKeys, + keys: make(map[wgtypes.Key]*[32]byte), + } +} + +// Encrypt encrypts msg for peerPublicKey. It is safe for concurrent use. +func (c *SharedKeyCache) Encrypt(msg []byte, peerPublicKey wgtypes.Key) ([]byte, error) { + nonce, err := genNonce() + if err != nil { + return nil, err + } + return box.SealAfterPrecomputation(nonce[:], msg, nonce, c.sharedKey(peerPublicKey)), nil +} + +// Decrypt decrypts a message that peerPublicKey encrypted for this cache's private +// key. It is safe for concurrent use. +func (c *SharedKeyCache) Decrypt(encryptedMsg []byte, peerPublicKey wgtypes.Key) ([]byte, error) { + if len(encryptedMsg) < nonceSize { + return nil, fmt.Errorf("invalid encrypted message length") + } + + var nonce [nonceSize]byte + copy(nonce[:], encryptedMsg[:nonceSize]) + + shared, cached := c.cached(peerPublicKey) + if !cached { + shared = c.derive(peerPublicKey) + } + + opened, ok := box.OpenAfterPrecomputation(nil, encryptedMsg[nonceSize:], &nonce, shared) + if !ok { + return nil, fmt.Errorf("failed to decrypt message from peer %s", peerPublicKey.String()) + } + + // The sender key of an incoming message is not authenticated until it opens, so + // only a key that produced a valid message is cached. Forged senders cannot fill + // the cache or evict real peers. + if !cached { + c.store(peerPublicKey, shared) + } + return opened, nil +} + +// EncryptMessage marshals message and encrypts it for peerPublicKey. +func (c *SharedKeyCache) EncryptMessage(peerPublicKey wgtypes.Key, message pb.Message) ([]byte, error) { + body, err := pb.Marshal(message) + if err != nil { + return nil, fmt.Errorf("marshal message: %w", err) + } + return c.Encrypt(body, peerPublicKey) +} + +// DecryptMessage decrypts a message from peerPublicKey and unmarshals it into message. +func (c *SharedKeyCache) DecryptMessage(peerPublicKey wgtypes.Key, encryptedMessage []byte, message pb.Message) error { + body, err := c.Decrypt(encryptedMessage, peerPublicKey) + if err != nil { + return err + } + if err := pb.Unmarshal(body, message); err != nil { + return fmt.Errorf("unmarshal message from peer %s: %w", peerPublicKey.String(), err) + } + return nil +} + +// Close drops every cached shared key and stops caching new ones. Encrypt and +// Decrypt keep working afterwards by deriving the key for each message. +func (c *SharedKeyCache) Close() { + c.mu.Lock() + defer c.mu.Unlock() + c.closed = true + clear(c.keys) +} + +func (c *SharedKeyCache) sharedKey(peerPublicKey wgtypes.Key) *[32]byte { + if shared, ok := c.cached(peerPublicKey); ok { + return shared + } + + shared := c.derive(peerPublicKey) + c.store(peerPublicKey, shared) + return shared +} + +func (c *SharedKeyCache) cached(peerPublicKey wgtypes.Key) (*[32]byte, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + shared, ok := c.keys[peerPublicKey] + return shared, ok +} + +// derive computes the shared key outside the lock: two goroutines racing on a new +// peer compute the same value, and holding the lock would serialise the x25519 work +// this cache avoids. +func (c *SharedKeyCache) derive(peerPublicKey wgtypes.Key) *[32]byte { + shared := new([32]byte) + box.Precompute(shared, toByte32(peerPublicKey), toByte32(c.privateKey)) + return shared +} + +func (c *SharedKeyCache) store(peerPublicKey wgtypes.Key, shared *[32]byte) { + c.mu.Lock() + defer c.mu.Unlock() + if c.closed { + return + } + if len(c.keys) >= c.limit { + // Map iteration order is random, so this evicts an arbitrary entry. + for k := range c.keys { + delete(c.keys, k) + break + } + } + c.keys[peerPublicKey] = shared +} diff --git a/encryption/sharedkey_limit.go b/encryption/sharedkey_limit.go new file mode 100644 index 000000000..b141492f6 --- /dev/null +++ b/encryption/sharedkey_limit.go @@ -0,0 +1,8 @@ +//go:build !ios && !android + +package encryption + +// maxSharedKeys bounds the cache so peers that come and go (ephemeral peers get a +// new key on every registration) cannot grow it for the lifetime of the process. +// An entry costs about 130 bytes, so a full cache is around 8 MB. +const maxSharedKeys = 1 << 16 diff --git a/encryption/sharedkey_limit_mobile.go b/encryption/sharedkey_limit_mobile.go new file mode 100644 index 000000000..f36181f94 --- /dev/null +++ b/encryption/sharedkey_limit_mobile.go @@ -0,0 +1,8 @@ +//go:build ios || android + +package encryption + +// maxSharedKeys is small on mobile, where the process runs under a tight memory +// limit. A miss only costs a fresh key derivation. An entry costs about 130 bytes, +// so a full cache is around 130 KB. +const maxSharedKeys = 1 << 10 diff --git a/encryption/sharedkey_test.go b/encryption/sharedkey_test.go new file mode 100644 index 000000000..003ccb627 --- /dev/null +++ b/encryption/sharedkey_test.go @@ -0,0 +1,184 @@ +package encryption + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +func newKeyPair(t testing.TB) (wgtypes.Key, wgtypes.Key) { + t.Helper() + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + return priv, priv.PublicKey() +} + +// The cache must stay wire compatible with peers that use the uncached functions, +// in both directions. +func TestSharedKeyCache_InteropWithUncached(t *testing.T) { + alicePriv, alicePub := newKeyPair(t) + bobPriv, bobPub := newKeyPair(t) + alice := NewSharedKeyCache(alicePriv) + msg := []byte("offer") + + enc, err := alice.Encrypt(msg, bobPub) + require.NoError(t, err) + dec, err := Decrypt(enc, alicePub, bobPriv) + require.NoError(t, err) + assert.Equal(t, msg, dec, "uncached peer must read a cached sender's message") + + enc, err = Encrypt(msg, alicePub, bobPriv) + require.NoError(t, err) + dec, err = alice.Decrypt(enc, bobPub) + require.NoError(t, err) + assert.Equal(t, msg, dec, "cached peer must read an uncached sender's message") +} + +// Two messages to the same peer share the derived key but never the nonce, so the +// ciphertexts differ. +func TestSharedKeyCache_FreshNoncePerMessage(t *testing.T) { + priv, _ := newKeyPair(t) + _, peerPub := newKeyPair(t) + c := NewSharedKeyCache(priv) + + a, err := c.Encrypt([]byte("same"), peerPub) + require.NoError(t, err) + b, err := c.Encrypt([]byte("same"), peerPub) + require.NoError(t, err) + assert.NotEqual(t, a, b, "ciphertexts of identical plaintext must differ") + assert.Len(t, c.keys, 1, "the shared key must be derived once per peer") +} + +// A message from one peer must not decrypt under another peer's cached key. +func TestSharedKeyCache_DoesNotMixPeers(t *testing.T) { + alicePriv, alicePub := newKeyPair(t) + bobPriv, _ := newKeyPair(t) + _, carolPub := newKeyPair(t) + alice := NewSharedKeyCache(alicePriv) + + enc, err := Encrypt([]byte("hi"), alicePub, bobPriv) + require.NoError(t, err) + _, err = alice.Decrypt(enc, carolPub) + assert.Error(t, err, "a message from Bob must not open with Carol's key") +} + +func TestSharedKeyCache_RejectsShortMessage(t *testing.T) { + priv, _ := newKeyPair(t) + _, peerPub := newKeyPair(t) + _, err := NewSharedKeyCache(priv).Decrypt(make([]byte, nonceSize-1), peerPub) + assert.Error(t, err) +} + +func TestSharedKeyCache_StaysBounded(t *testing.T) { + priv, _ := newKeyPair(t) + c := NewSharedKeyCache(priv) + c.limit = 4 + + for i := 0; i < 20; i++ { + _, peerPub := newKeyPair(t) + _, err := c.Encrypt([]byte("x"), peerPub) + require.NoError(t, err) + assert.LessOrEqual(t, len(c.keys), c.limit, "cache must not grow past its cap") + } + assert.Len(t, c.keys, c.limit, "a full cache keeps evicting one entry per new peer") + c.Close() + assert.Empty(t, c.keys, "Close must drop every entry") +} + +func TestSharedKeyCache_Concurrent(t *testing.T) { + alicePriv, alicePub := newKeyPair(t) + bobPriv, bobPub := newKeyPair(t) + alice := NewSharedKeyCache(alicePriv) + bob := NewSharedKeyCache(bobPriv) + + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + enc, err := alice.Encrypt([]byte("m"), bobPub) + if !assert.NoError(t, err) { + return + } + dec, err := bob.Decrypt(enc, alicePub) + if !assert.NoError(t, err) || !assert.Equal(t, []byte("m"), dec) { + return + } + } + }() + } + wg.Wait() +} + +func BenchmarkEncryptDecryptUncached(b *testing.B) { + alicePriv, alicePub := newKeyPair(b) + bobPriv, bobPub := newKeyPair(b) + msg := make([]byte, 512) + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + enc, err := Encrypt(msg, bobPub, alicePriv) + require.NoError(b, err) + _, err = Decrypt(enc, alicePub, bobPriv) + require.NoError(b, err) + } +} + +func BenchmarkEncryptDecryptCached(b *testing.B) { + alicePriv, alicePub := newKeyPair(b) + bobPriv, bobPub := newKeyPair(b) + alice := NewSharedKeyCache(alicePriv) + bob := NewSharedKeyCache(bobPriv) + msg := make([]byte, 512) + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + enc, err := alice.Encrypt(msg, bobPub) + require.NoError(b, err) + _, err = bob.Decrypt(enc, alicePub) + require.NoError(b, err) + } +} + +// A forged sender key must not populate the cache: the key of an incoming message +// is only trusted once the message opens. +func TestSharedKeyCache_FailedDecryptDoesNotCache(t *testing.T) { + alicePriv, alicePub := newKeyPair(t) + bobPriv, bobPub := newKeyPair(t) + _, forgedPub := newKeyPair(t) + alice := NewSharedKeyCache(alicePriv) + + enc, err := Encrypt([]byte("hi"), alicePub, bobPriv) + require.NoError(t, err) + + _, err = alice.Decrypt(enc, forgedPub) + require.Error(t, err) + assert.Empty(t, alice.keys, "a message that fails to open must not add a cache entry") + + _, err = alice.Decrypt(enc, bobPub) + require.NoError(t, err) + assert.Len(t, alice.keys, 1, "a message that opens caches its sender's key") +} + +// After Close the cache still works but no longer keeps key material. +func TestSharedKeyCache_ClosedDoesNotRepopulate(t *testing.T) { + alicePriv, alicePub := newKeyPair(t) + bobPriv, bobPub := newKeyPair(t) + alice := NewSharedKeyCache(alicePriv) + + _, err := alice.Encrypt([]byte("x"), bobPub) + require.NoError(t, err) + alice.Close() + assert.Empty(t, alice.keys) + + enc, err := alice.Encrypt([]byte("y"), bobPub) + require.NoError(t, err) + dec, err := Decrypt(enc, alicePub, bobPriv) + require.NoError(t, err) + assert.Equal(t, []byte("y"), dec, "a closed cache must still encrypt correctly") + assert.Empty(t, alice.keys, "a closed cache must not cache new keys") +} diff --git a/shared/signal/client/grpc.go b/shared/signal/client/grpc.go index a0bb2f080..92be57b30 100644 --- a/shared/signal/client/grpc.go +++ b/shared/signal/client/grpc.go @@ -53,6 +53,7 @@ type ConnStateNotifier interface { // GrpcClient Wraps the Signal Exchange Service gRpc client type GrpcClient struct { key wgtypes.Key + sharedKeys *encryption.SharedKeyCache realClient proto.SignalExchangeClient signalConn *grpc.ClientConn ctx context.Context @@ -107,6 +108,7 @@ func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled boo c := &GrpcClient{ ctx: ctx, key: key, + sharedKeys: encryption.NewSharedKeyCache(key), mux: sync.Mutex{}, status: StreamDisconnected, connStateCallbackLock: sync.RWMutex{}, @@ -158,6 +160,7 @@ func (c *GrpcClient) Close() error { } c.decryptionWg.Wait() c.decryptionWorker = nil + c.sharedKeys.Close() return c.signalConn.Close() } @@ -418,7 +421,7 @@ func (c *GrpcClient) decryptMessage(msg *proto.EncryptedMessage) (*proto.Message } body := &proto.Body{} - err = encryption.DecryptMessage(remoteKey, c.key, msg.GetBody(), body) + err = c.sharedKeys.DecryptMessage(remoteKey, msg.GetBody(), body) if err != nil { return nil, err } @@ -438,7 +441,7 @@ func (c *GrpcClient) encryptMessage(msg *proto.Message) (*proto.EncryptedMessage return nil, err } - encryptedBody, err := encryption.EncryptMessage(remoteKey, c.key, msg.Body) + encryptedBody, err := c.sharedKeys.EncryptMessage(remoteKey, msg.Body) if err != nil { return nil, err } From 6c453a0f979806cd93a230073ae0e127b6ce01f0 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:37:49 +0900 Subject: [PATCH 70/75] [client] Add debug cpu start and stop commands (#7749) * Add debug cpu start and stop commands to profile the daemon without a restart * Restore test globals on every exit and stop the daemon in the cpu profile test * Add a no-updown flag to debug for * Enable sync response persistence with --no-updown and reset flags between debug test runs * Reset flags of every command between debug test runs * Reset slice flags with Replace in the debug test helper * Explain a running CPU profile in debug for and document cpu start and no-updown limits --- client/cmd/debug.go | 84 ++++++++++++------ client/cmd/debug_cpu.go | 83 ++++++++++++++++++ client/cmd/debug_cpu_test.go | 164 +++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+), 26 deletions(-) create mode 100644 client/cmd/debug_cpu.go create mode 100644 client/cmd/debug_cpu_test.go diff --git a/client/cmd/debug.go b/client/cmd/debug.go index 98fe53626..c4d5ad6d7 100644 --- a/client/cmd/debug.go +++ b/client/cmd/debug.go @@ -23,7 +23,10 @@ import ( "github.com/netbirdio/netbird/version" ) -const errCloseConnection = "Failed to close connection: %v" +const ( + errCloseConnection = "Failed to close connection: %v" + noUpDownFlag = "no-updown" +) var ( logFileCount uint32 @@ -257,13 +260,14 @@ func runForDuration(cmd *cobra.Command, args []string) error { } stateWasDown := stat.Status != string(internal.StatusConnected) && stat.Status != string(internal.StatusConnecting) + noUpDown, _ := cmd.Flags().GetBool(noUpDownFlag) initialLogLevel, err := client.GetLogLevel(cmd.Context(), &proto.GetLogLevelRequest{}) if err != nil { return fmt.Errorf("failed to get log level: %v", status.Convert(err).Message()) } - if stateWasDown { + if stateWasDown && !noUpDown { if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) } else { @@ -284,34 +288,20 @@ func runForDuration(cmd *cobra.Command, args []string) error { } needsRestoreUp := false - if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { - cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message()) + if noUpDown { + enableSyncResponsePersistence(cmd, client) } else { - needsRestoreUp = !stateWasDown - cmd.Println("netbird down") + needsRestoreUp = restartDaemon(cmd, client, stateWasDown) } - time.Sleep(1 * time.Second) - - // Enable sync response persistence before bringing the service up - if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{ - Enabled: true, - }); err != nil { - cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message()) - } - - if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { - cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) - } else { - needsRestoreUp = false - cmd.Println("netbird up") - } - - time.Sleep(3 * time.Second) - cpuProfilingStarted := false if _, err := client.StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil { - cmd.PrintErrf("Failed to start CPU profiling: %v\n", err) + if msg := status.Convert(err).Message(); strings.Contains(msg, "already in progress") { + cmd.PrintErrln("CPU profiling is already running (started with `netbird debug cpu start`). " + + "It is left running and is included in a bundle created after `netbird debug cpu stop`.") + } else { + cmd.PrintErrf("Failed to start CPU profiling: %v\n", msg) + } } else { cpuProfilingStarted = true defer func() { @@ -401,7 +391,7 @@ func runForDuration(cmd *cobra.Command, args []string) error { } } - if stateWasDown { + if stateWasDown && !noUpDown { if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { cmd.PrintErrf("Failed to restore service down state: %v\n", status.Convert(err).Message()) } else { @@ -458,6 +448,47 @@ func setSyncResponsePersistence(cmd *cobra.Command, args []string) error { return nil } +// enableSyncResponsePersistence asks the daemon to keep the latest sync +// response so the bundle carries the network map. With a running daemon only +// syncs received after the call are kept. +func enableSyncResponsePersistence(cmd *cobra.Command, client proto.DaemonServiceClient) { + if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{ + Enabled: true, + }); err != nil { + cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message()) + } +} + +// restartDaemon cycles the daemon down and up with sync response persistence +// enabled so the bundle carries the network map. It reports whether the +// daemon was left down although it was running before, so the caller can +// bring it back up. +func restartDaemon(cmd *cobra.Command, client proto.DaemonServiceClient, stateWasDown bool) bool { + needsRestoreUp := false + if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { + cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message()) + } else { + needsRestoreUp = !stateWasDown + cmd.Println("netbird down") + } + + time.Sleep(1 * time.Second) + + // Enable sync response persistence before bringing the service up + enableSyncResponsePersistence(cmd, client) + + if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { + cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) + } else { + needsRestoreUp = false + cmd.Println("netbird up") + } + + time.Sleep(3 * time.Second) + + return needsRestoreUp +} + func waitForDurationOrCancel(ctx context.Context, duration time.Duration, cmd *cobra.Command) error { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() @@ -546,4 +577,5 @@ func init() { forCmd.Flags().StringVar(&uploadBundleURLFlag, "upload-bundle-url", types.DefaultBundleURL, "Service URL to get an URL to upload the debug bundle") forCmd.Flags().BoolVar(&uploadBundleInsecureFlag, "upload-bundle-insecure", false, "Allow uploading to an http or untrusted-TLS upload server (self-hosted); requires root") forCmd.Flags().Bool("capture", false, "Capture packets during the debug duration and include in bundle") + forCmd.Flags().Bool(noUpDownFlag, false, "Keep the daemon running instead of bringing it down and up before collecting. The bundle only includes the network map if a sync arrives during the run") } diff --git a/client/cmd/debug_cpu.go b/client/cmd/debug_cpu.go new file mode 100644 index 000000000..a01b845cf --- /dev/null +++ b/client/cmd/debug_cpu.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "google.golang.org/grpc/status" + + "github.com/netbirdio/netbird/client/proto" +) + +var debugCPUCmd = &cobra.Command{ + Use: "cpu", + Short: "Profile the daemon's CPU usage", + Long: `Starts and stops CPU profiling in the running daemon without restarting it. +The profile is included in the next debug bundle as cpu.prof. + +Profiling is not time limited: it keeps running, and keeps costing CPU, until +"netbird debug cpu stop" is run.`, +} + +var debugCPUStartCmd = &cobra.Command{ + Use: "start", + Short: "Start CPU profiling in the daemon", + Example: " netbird debug cpu start", + Args: cobra.NoArgs, + RunE: debugCPUStart, +} + +var debugCPUStopCmd = &cobra.Command{ + Use: "stop", + Short: "Stop CPU profiling in the daemon", + Long: `Stops CPU profiling. The captured profile stays in the daemon until the next +debug bundle is created, which includes it as cpu.prof.`, + Example: " netbird debug cpu stop && netbird debug bundle", + Args: cobra.NoArgs, + RunE: debugCPUStop, +} + +func debugCPUStart(cmd *cobra.Command, _ []string) error { + conn, err := getClient(cmd) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + log.Errorf(errCloseConnection, err) + } + }() + + if _, err := proto.NewDaemonServiceClient(conn).StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil { + return fmt.Errorf("start CPU profiling: %v", status.Convert(err).Message()) + } + + cmd.Println("CPU profiling started and runs until stopped. Run `netbird debug cpu stop` and then `netbird debug bundle` to collect it.") + return nil +} + +func debugCPUStop(cmd *cobra.Command, _ []string) error { + conn, err := getClient(cmd) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + log.Errorf(errCloseConnection, err) + } + }() + + if _, err := proto.NewDaemonServiceClient(conn).StopCPUProfile(cmd.Context(), &proto.StopCPUProfileRequest{}); err != nil { + return fmt.Errorf("stop CPU profiling: %v", status.Convert(err).Message()) + } + + cmd.Println("CPU profiling stopped. Run `netbird debug bundle` to include cpu.prof.") + return nil +} + +func init() { + debugCPUCmd.AddCommand(debugCPUStartCmd) + debugCPUCmd.AddCommand(debugCPUStopCmd) + debugCmd.AddCommand(debugCPUCmd) +} diff --git a/client/cmd/debug_cpu_test.go b/client/cmd/debug_cpu_test.go new file mode 100644 index 000000000..85fffd462 --- /dev/null +++ b/client/cmd/debug_cpu_test.go @@ -0,0 +1,164 @@ +package cmd + +import ( + "bytes" + "context" + "os/user" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/profilemanager" +) + +// startDebugTestDaemon starts an in-process daemon with an isolated profile +// directory and returns the address the CLI should dial. +func startDebugTestDaemon(t *testing.T) string { + t.Helper() + + tempDir := t.TempDir() + origDefaultProfileDir := profilemanager.DefaultConfigPathDir + origActiveProfileStatePath := profilemanager.ActiveProfileStatePath + origConfigDirOverride := profilemanager.ConfigDirOverride + origDaemonAddr := daemonAddr + t.Cleanup(func() { + profilemanager.DefaultConfigPathDir = origDefaultProfileDir + profilemanager.ActiveProfileStatePath = origActiveProfileStatePath + profilemanager.ConfigDirOverride = origConfigDirOverride + daemonAddr = origDaemonAddr + }) + + profilemanager.DefaultConfigPathDir = tempDir + profilemanager.ActiveProfileStatePath = tempDir + "/active_profile.json" + profilemanager.ConfigDirOverride = tempDir + + currUser, err := user.Current() + require.NoError(t, err) + sm := profilemanager.ServiceManager{} + created, err := sm.AddProfile("test1", currUser.Username) + require.NoError(t, err) + require.NoError(t, sm.SetActiveProfileState(&profilemanager.ActiveProfileState{ + ID: created.ID, + Username: currUser.Username, + })) + + ctx, cancel := context.WithCancel(internal.CtxInitState(context.Background())) + srv, lis := startClientDaemon(t, ctx, "", tempDir+"/config.json") + t.Cleanup(func() { + cancel() + srv.Stop() + }) + + return "tcp://" + lis.Addr().String() +} + +// runDebugCmd runs `netbird debug ` against the daemon at addr and +// returns everything the command printed. +func runDebugCmd(addr string, args ...string) (string, error) { + daemonAddr = addr + var out bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&out) + rootCmd.SetArgs(append(append([]string{"debug"}, args...), "--daemon-addr", addr, "--log-file", "")) + err := rootCmd.Execute() + rootCmd.SetOut(nil) + rootCmd.SetErr(nil) + rootCmd.SetArgs(nil) + resetFlags(rootCmd) + return out.String(), err +} + +// resetFlags puts every flag of the command and its subcommands back to its +// default so a value parsed in one run does not leak into the next in-process +// execution. +func resetFlags(cmd *cobra.Command) { + reset := func(f *pflag.Flag) { + // Set appends to a slice flag and would parse the "[a,b]" default + // text as elements, so slices are replaced instead. + if sv, ok := f.Value.(pflag.SliceValue); ok { + var def []string + if trimmed := strings.Trim(f.DefValue, "[]"); trimmed != "" { + def = strings.Split(trimmed, ",") + } + _ = sv.Replace(def) + } else { + _ = f.Value.Set(f.DefValue) + } + f.Changed = false + } + cmd.Flags().VisitAll(reset) + cmd.PersistentFlags().VisitAll(reset) + // Commands pin their writers to the buffer of the run that first used + // them, so a later run would print into the old buffer. + cmd.SetOut(nil) + cmd.SetErr(nil) + for _, sub := range cmd.Commands() { + resetFlags(sub) + } +} + +// TestResetFlagsSliceDefault guards against Set("[]") on slice flags, which +// stores a literal "[]" element instead of the empty default. +func TestResetFlagsSliceDefault(t *testing.T) { + cmd := &cobra.Command{Use: "x"} + var env, withDefault []string + cmd.Flags().StringSliceVar(&env, "env", nil, "") + cmd.Flags().StringSliceVar(&withDefault, "names", []string{"a", "b"}, "") + require.NoError(t, cmd.Flags().Parse([]string{"--env", "K=V", "--names", "c"})) + + resetFlags(cmd) + + assert.Empty(t, env, "slice flag with no default must reset to empty") + assert.Equal(t, []string{"a", "b"}, withDefault, "slice flag must reset to its default") +} + +func TestDebugCPUStartStop(t *testing.T) { + addr := startDebugTestDaemon(t) + + run := func(args ...string) error { + _, err := runDebugCmd(addr, append([]string{"cpu"}, args...)...) + return err + } + + require.Error(t, run("stop"), "stop without a running profile must fail") + require.NoError(t, run("start")) + assert.Error(t, run("start"), "second start must be rejected while profiling") + require.NoError(t, run("stop")) + assert.Error(t, run("stop"), "second stop must be rejected") + assert.NoError(t, run("start"), "profiling can be started again after a stop") + assert.NoError(t, run("stop")) +} + +// TestDebugForKeepsRunningCPUProfile covers `debug for` started while a +// profile from `debug cpu start` is running: it must say so, leave the +// profile alone, and still create the bundle. +func TestDebugForKeepsRunningCPUProfile(t *testing.T) { + addr := startDebugTestDaemon(t) + + _, err := runDebugCmd(addr, "cpu", "start") + require.NoError(t, err) + + out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown") + require.NoError(t, err, "output: %s", out) + assert.Contains(t, out, "CPU profiling is already running", "the conflict must be explained") + assert.NotContains(t, out, "rpc error", "the raw RPC error must not reach the user") + assert.Contains(t, out, "Local file:", "the bundle must still be created") + + _, err = runDebugCmd(addr, "cpu", "stop") + assert.NoError(t, err, "the profile started by the user must still be running") +} + +func TestDebugForNoUpDown(t *testing.T) { + addr := startDebugTestDaemon(t) + + out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown") + require.NoError(t, err, "output: %s", out) + assert.NotContains(t, out, "netbird down", "--no-updown must not bring the daemon down") + assert.NotContains(t, out, "netbird up", "--no-updown must not bring the daemon up") + assert.Contains(t, out, "Local file:", "the bundle must still be created") +} From fd1a0203c7d9dcf0df81bb16d344edb7bd85a416 Mon Sep 17 00:00:00 2001 From: Brad Ison Date: Wed, 30 Sep 2026 18:25:38 +0200 Subject: [PATCH 71/75] [management] Clean up after account deletion (#7812) Deleting an account left state behind that DeleteAccount's store associations don't reach. The Agent Network tables outlived the account, keeping its gateway domain claimed and its provider API keys stored. The proxies kept serving its gateway until they next resynced. Cloud-side state, such as managed proxy deployments, had no way to be cleaned up at all. Account deletion now runs registered hooks after the permission check and before any users or data are removed. A failing hook aborts the deletion. Agent Network registers one that tells the proxies to drop the account's gateway mappings. The account's settings, providers, policies, guardrails and budget rules are deleted in the account's transaction. Consumption counters, and the access logs of deleted accounts, are left to the background cleanup; usage records are kept. --- e2e/agentnetwork/account_delete_test.go | 292 ++++++++++++++++++ e2e/harness/agentnetwork.go | 5 + e2e/harness/combined.go | 27 ++ e2e/harness/proxy.go | 39 +++ .../accesslog_cleanup_realstore_test.go | 76 +++++ .../internals/modules/agentnetwork/manager.go | 73 +++-- .../modules/agentnetwork/reconcile.go | 68 +++- .../modules/agentnetwork/reconcile_test.go | 154 +++++++++ management/internals/server/modules.go | 1 + management/server/account.go | 33 ++ management/server/account/deletion_hook.go | 14 + management/server/account_test.go | 95 ++++++ management/server/store/sql_store_account.go | 28 ++ .../server/store/sql_store_account_test.go | 46 +++ .../sql_store_agent_network_access_log.go | 17 + .../server/store/sql_store_agentnetwork.go | 15 + .../sql_store_agentnetwork_accesslog_test.go | 35 +++ management/server/store/store.go | 2 + management/server/store/store_mock.go | 30 ++ 19 files changed, 1023 insertions(+), 27 deletions(-) create mode 100644 e2e/agentnetwork/account_delete_test.go create mode 100644 management/internals/modules/agentnetwork/accesslog_cleanup_realstore_test.go create mode 100644 management/server/account/deletion_hook.go diff --git a/e2e/agentnetwork/account_delete_test.go b/e2e/agentnetwork/account_delete_test.go new file mode 100644 index 000000000..a8b52766f --- /dev/null +++ b/e2e/agentnetwork/account_delete_test.go @@ -0,0 +1,292 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// accountDeleteModel is a made-up model id the provider enumerates and prices, +// so the chat routes to the mock upstream and is metered deterministically. +const accountDeleteModel = "e2e-account-delete-model" + +// agentNetworkConfigTables are deleted with the account, in its transaction. +var agentNetworkConfigTables = []string{ + "agent_network_settings", + "agent_network_providers", + "agent_network_policies", + "agent_network_guardrails", + "agent_network_budget_rules", +} + +// TestAccountDelete_RemovesAgentNetworkState deletes an account that has a full +// Agent Network setup and has served traffic, and checks what that leaves +// behind, end to end: +// +// - the proxy stops running the account's gateway, instead of keeping its +// mappings and provider API keys in memory until it next resyncs; +// - the configuration rows go with the account, while access logs and usage +// records stay for retention; +// - the account's consumption counters are swept once the cleanup runs; +// - the gateway domain is free for another account to claim. +// +// It runs on a dedicated server, since deleting the shared account would take +// every other test down with it. +func TestAccountDelete_RemovesAgentNetworkState(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + + fresh, err := harnessStartFresh(ctx, t) + require.NoError(t, err, "start dedicated combined server") + + accounts, err := fresh.API().Accounts.List(ctx) + require.NoError(t, err, "list accounts") + require.Len(t, accounts, 1, "a fresh server has exactly the bootstrapped account") + accountID := accounts[0].Id + + cluster := harness.AgentNetworkCluster + settings, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ProxyAddress: &cluster}) + require.NoError(t, err, "bootstrap agent-network endpoint") + require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned at bootstrap") + + env := provisionAccountDeleteEnv(t, ctx, fresh, settings.Endpoint) + chatThrough(t, ctx, env) + + // Preconditions: the proxy runs the account's gateway, and the request left + // the traffic-driven rows the rest of the test expects to outlive the delete. + requireEventually(t, ctx, 60*time.Second, "proxy should run a client for the account", func() bool { + return proxyRunsAccount(t, ctx, env.proxy, accountID) + }) + requireEventually(t, ctx, accessLogIngestWindow, "the request should leave consumption, usage and access-log rows", func() bool { + counts := accountRowCounts(t, fresh, accountID, + "agent_network_consumption", "agent_network_request_usage", "agent_network_access_log") + return counts["agent_network_consumption"] > 0 && + counts["agent_network_request_usage"] > 0 && + counts["agent_network_access_log"] > 0 + }) + + require.NoError(t, fresh.API().Accounts.Delete(ctx, accountID), "delete account") + + // The proxy is told to drop the gateway. A proxy that only learns on its + // next resync keeps serving the deleted account with its provider API keys. + if !eventually(ctx, 60*time.Second, func() bool { return proxyDroppedAccount(t, ctx, env.proxy, accountID) }) { + t.Errorf("proxy still runs a client for deleted account %s\n=== proxy logs ===\n%s", + accountID, env.proxy.Logs(context.Background())) + } + + counts := accountRowCounts(t, fresh, accountID, append(slices.Clone(agentNetworkConfigTables), + "agent_network_request_usage", "agent_network_access_log")...) + for _, table := range agentNetworkConfigTables { + assert.Zero(t, counts[table], "%s rows should be deleted with the account", table) + } + assert.NotZero(t, counts["agent_network_request_usage"], "usage records should be kept") + assert.NotZero(t, counts["agent_network_access_log"], "access logs should be left for retention") + + // The cleanup's first pass runs at startup, and whether instance setup is + // open again is only re-evaluated then. + require.NoError(t, fresh.Restart(ctx), "restart combined server") + requireEventually(t, ctx, 60*time.Second, "the cleanup should sweep the deleted account's consumption counters", func() bool { + return accountRowCounts(t, fresh, accountID, "agent_network_consumption")["agent_network_consumption"] == 0 + }) + + // A new account can claim the deleted account's gateway domain: its + // settings row no longer holds the global unique index. + _, err = fresh.Bootstrap(ctx) + require.NoError(t, err, "bootstrap a second account once the first is gone") + claimed, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{Endpoint: &settings.Endpoint}) + require.NoError(t, err, "a new account should be able to claim the deleted account's gateway domain") + assert.Equal(t, settings.Endpoint, claimed.Endpoint, "the new account should hold the released domain") +} + +// accountDeleteEnv is a connected gateway for one account: a proxy running the +// debug endpoint, a client peer, and the resolved endpoint. +type accountDeleteEnv struct { + endpoint string + proxyIP string + client *harness.Client + proxy *harness.Proxy +} + +// provisionAccountDeleteEnv gives the server's account one of every Agent +// Network configuration row (provider, guardrail, policy, budget rule; the +// settings row is the caller's) and brings up a proxy and a client. The policy +// and budget rule switch on usage metering, so a request records consumption. +func provisionAccountDeleteEnv(t *testing.T, ctx context.Context, srv *harness.Combined, endpoint string) accountDeleteEnv { + t.Helper() + + vllm, err := harness.StartVLLM(ctx, srv) + require.NoError(t, err, "start mock vLLM upstream") + t.Cleanup(func() { _ = vllm.Terminate(context.Background()) }) + + grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-account-delete"}) + require.NoError(t, err, "create group") + + ephemeral := false + sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{ + Name: "e2e-account-delete-client", + Type: "reusable", + ExpiresIn: 86400, + AutoGroups: []string{grp.Id}, + Ephemeral: &ephemeral, + }) + require.NoError(t, err, "mint setup key") + + apiKey := "sk-account-delete-e2e" + models := []api.AgentNetworkProviderModel{{Id: accountDeleteModel, InputPer1k: 0.01, OutputPer1k: 0.02}} + prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: "account-delete", + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &apiKey, + Enabled: ptr(true), + Models: &models, + }) + require.NoError(t, err, "create provider") + + var gr api.AgentNetworkGuardrailRequest + gr.Name = "e2e-account-delete" + gr.Checks.ModelAllowlist.Enabled = true + gr.Checks.ModelAllowlist.Models = []string{accountDeleteModel} + guard, err := srv.CreateGuardrail(ctx, gr) + require.NoError(t, err, "create guardrail") + + limits := api.AgentNetworkPolicyLimits{ + TokenLimit: api.AgentNetworkPolicyTokenLimit{ + Enabled: true, + GroupCap: 10_000_000, + UserCap: 10_000_000, + WindowSeconds: 60, + }, + } + _, err = srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{ + Name: "e2e-account-delete", + Enabled: ptr(true), + SourceGroups: []string{grp.Id}, + DestinationProviderIds: []string{prov.Id}, + GuardrailIds: &[]string{guard.Id}, + Limits: &limits, + }) + require.NoError(t, err, "create policy") + + _, err = srv.CreateBudgetRule(ctx, api.AgentNetworkBudgetRuleRequest{ + Name: "e2e-account-delete", + Limits: limits, + TargetGroups: &[]string{grp.Id}, + }) + require.NoError(t, err, "create budget rule") + + proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-account-delete-proxy") + require.NoError(t, err, "mint proxy token") + px, err := harness.StartProxy(ctx, srv, proxyToken, map[string]string{"NB_PROXY_DEBUG_ENDPOINT": "true"}) + require.NoError(t, err, "start proxy") + t.Cleanup(func() { _ = px.Terminate(context.Background()) }) + + cl, err := harness.StartClient(ctx, srv, sk.Key) + require.NoError(t, err, "start client") + t.Cleanup(func() { _ = cl.Terminate(context.Background()) }) + + require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management") + proxyIP, err := cl.ResolveProxyIP(ctx, endpoint) + require.NoError(t, err, "resolve endpoint to proxy IP") + if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil { + t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background())) + } + + return accountDeleteEnv{endpoint: endpoint, proxyIP: proxyIP, client: cl, proxy: px} +} + +// chatThrough drives one chat through the gateway, retrying to absorb +// first-call tunnel and DNS jitter. +func chatThrough(t *testing.T, ctx context.Context, env accountDeleteEnv) { + t.Helper() + var code int + var body string + ok := eventually(ctx, 90*time.Second, func() bool { + c, b, err := env.client.Chat(ctx, env.endpoint, env.proxyIP, harness.WireChat, + accountDeleteModel, "Reply with exactly: pong", "e2e-session-account-delete") + code, body = c, b + return err == nil && c == 200 + }) + require.True(t, ok, "chat must return 200, last got %d: %s\n=== proxy logs ===\n%s", + code, body, env.proxy.Logs(context.Background())) +} + +// proxyRunsAccount reports whether a lookup succeeded and shows the proxy +// running a client for the account. +func proxyRunsAccount(t *testing.T, ctx context.Context, px *harness.Proxy, accountID string) bool { + t.Helper() + runs, ok := lookupProxyAccount(t, ctx, px, accountID) + return ok && runs +} + +// proxyDroppedAccount reports whether a lookup succeeded and shows the proxy +// no longer running a client for the account. A failed lookup confirms +// nothing, so it keeps the caller polling rather than passing the check. +func proxyDroppedAccount(t *testing.T, ctx context.Context, px *harness.Proxy, accountID string) bool { + t.Helper() + runs, ok := lookupProxyAccount(t, ctx, px, accountID) + return ok && !runs +} + +// lookupProxyAccount asks the proxy whether it runs a client for the account; +// ok is false when the lookup itself failed. +func lookupProxyAccount(t *testing.T, ctx context.Context, px *harness.Proxy, accountID string) (runs, ok bool) { + t.Helper() + clients, err := px.DebugClients(ctx) + if err != nil { + t.Logf("proxy debug clients: %v", err) + return false, false + } + return slices.ContainsFunc(clients, func(c harness.ProxyDebugClient) bool { return c.AccountID == accountID }), true +} + +// accountRowCounts counts the account's rows in each table, read from a +// snapshot of the management store. +func accountRowCounts(t *testing.T, srv *harness.Combined, accountID string, tables ...string) map[string]int64 { + t.Helper() + dbPath, err := srv.SnapshotStoreDB(t.TempDir()) + require.NoError(t, err, "snapshot management sqlite store") + db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{}) + require.NoError(t, err, "open store snapshot") + sqlDB, err := db.DB() + require.NoError(t, err) + defer func() { _ = sqlDB.Close() }() + + counts := make(map[string]int64, len(tables)) + for _, table := range tables { + var n int64 + require.NoError(t, db.Table(table).Where("account_id = ?", accountID).Count(&n).Error, "count %s rows", table) + counts[table] = n + } + return counts +} + +// eventually polls cond every two seconds until it holds or timeout passes. +func eventually(ctx context.Context, timeout time.Duration, cond func() bool) bool { + deadline := time.Now().Add(timeout) + for { + if cond() { + return true + } + if time.Now().After(deadline) || !waitBeforeRetry(ctx, 2*time.Second) { + return false + } + } +} + +// requireEventually fails the test now if cond does not hold within timeout. +func requireEventually(t *testing.T, ctx context.Context, timeout time.Duration, msg string, cond func() bool) { + t.Helper() + require.True(t, eventually(ctx, timeout, cond), msg) +} diff --git a/e2e/harness/agentnetwork.go b/e2e/harness/agentnetwork.go index e51f2dd7a..8f688836d 100644 --- a/e2e/harness/agentnetwork.go +++ b/e2e/harness/agentnetwork.go @@ -135,6 +135,11 @@ func (c *Combined) DeleteGuardrail(ctx context.Context, id string) error { return anDelete(ctx, c, "/api/agent-network/guardrails/"+id) } +// CreateBudgetRule creates an account-level agent-network budget rule. +func (c *Combined) CreateBudgetRule(ctx context.Context, req api.AgentNetworkBudgetRuleRequest) (api.AgentNetworkBudgetRule, error) { + return anRequest[api.AgentNetworkBudgetRule](ctx, c, http.MethodPost, "/api/agent-network/budget-rules", req) +} + // CreateSettings bootstraps the account's agent-network settings row, // assigning the immutable endpoint. Exactly one of req.ProxyAddress (labeled // endpoint beneath that cluster) and req.Endpoint (self-addressed dedicated diff --git a/e2e/harness/combined.go b/e2e/harness/combined.go index e03f9f256..ea451e3ad 100644 --- a/e2e/harness/combined.go +++ b/e2e/harness/combined.go @@ -305,6 +305,33 @@ func (c *Combined) SnapshotStoreDB(dstDir string) (string, error) { return dst, nil } +// Restart stops and starts the combined container, keeping its bind-mounted +// data dir, and waits for the API again. The host port can change across a +// restart, so BaseURL and the authenticated client are refreshed. Work that +// management only does at startup (such as the agent-network cleanup's first +// pass, or re-evaluating whether instance setup is required) runs again. +func (c *Combined) Restart(ctx context.Context) error { + if err := c.container.Stop(ctx, nil); err != nil { + return fmt.Errorf("stop combined container: %w", err) + } + if err := c.container.Start(ctx); err != nil { + return fmt.Errorf("start combined container: %w", err) + } + host, err := c.container.Host(ctx) + if err != nil { + return fmt.Errorf("container host: %w", err) + } + mapped, err := c.container.MappedPort(ctx, nat.Port(combinedHTTPPort)) + if err != nil { + return fmt.Errorf("mapped port: %w", err) + } + c.BaseURL = fmt.Sprintf("http://%s:%s", host, mapped.Port()) + if c.PAT != "" { + c.api = rest.New(c.BaseURL, c.PAT) + } + return nil +} + // Logs returns the combined server container logs, for diagnostics. func (c *Combined) Logs(ctx context.Context) string { return containerLogs(ctx, c.container) diff --git a/e2e/harness/proxy.go b/e2e/harness/proxy.go index 3d709b439..ee458908b 100644 --- a/e2e/harness/proxy.go +++ b/e2e/harness/proxy.go @@ -3,13 +3,17 @@ package harness import ( + "bytes" "context" + "encoding/json" "fmt" + "io" "os" "time" "github.com/docker/docker/api/types/container" "github.com/testcontainers/testcontainers-go" + tcexec "github.com/testcontainers/testcontainers-go/exec" "github.com/testcontainers/testcontainers-go/wait" ) @@ -114,6 +118,41 @@ func StartProxy(ctx context.Context, c *Combined, proxyToken string, envOverride return &Proxy{container: ctr, workDir: workDir}, nil } +// ProxyDebugClient is one per-account embedded client the proxy runs, as the +// proxy's debug endpoint reports it. +type ProxyDebugClient struct { + AccountID string `json:"account_id"` + ServiceCount int `json:"service_count"` + ServiceKeys []string `json:"service_keys"` +} + +// DebugClients lists the per-account clients the proxy is running, through +// the proxy's own debug CLI inside the container. The proxy must be started +// with NB_PROXY_DEBUG_ENDPOINT=true. +func (p *Proxy) DebugClients(ctx context.Context) ([]ProxyDebugClient, error) { + code, reader, err := p.container.Exec(ctx, + []string{"/usr/bin/netbird-proxy", "debug", "clients", "--json"}, tcexec.Multiplexed()) + if err != nil { + return nil, fmt.Errorf("exec debug clients: %w", err) + } + out, _ := io.ReadAll(reader) + if code != 0 { + return nil, fmt.Errorf("debug clients exited %d: %s", code, string(out)) + } + // stderr is multiplexed in; the JSON document starts at the first brace. + start := bytes.IndexByte(out, '{') + if start < 0 { + return nil, fmt.Errorf("no JSON in debug clients output: %s", string(out)) + } + var resp struct { + Clients []ProxyDebugClient `json:"clients"` + } + if err := json.NewDecoder(bytes.NewReader(out[start:])).Decode(&resp); err != nil { + return nil, fmt.Errorf("decode debug clients output: %w", err) + } + return resp.Clients, nil +} + // Logs returns the proxy container logs, for diagnostics on failure. func (p *Proxy) Logs(ctx context.Context) string { return containerLogs(ctx, p.container) diff --git a/management/internals/modules/agentnetwork/accesslog_cleanup_realstore_test.go b/management/internals/modules/agentnetwork/accesslog_cleanup_realstore_test.go new file mode 100644 index 000000000..0e294a560 --- /dev/null +++ b/management/internals/modules/agentnetwork/accesslog_cleanup_realstore_test.go @@ -0,0 +1,76 @@ +package agentnetwork + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" + "github.com/netbirdio/netbird/management/server/store" + nbtypes "github.com/netbirdio/netbird/management/server/types" +) + +// TestCleanupAccessLogs_RealStore_DeletedAccount covers a deleted account's access logs. +// The sweep is driven by settings rows, which go with the account, so without a fallback +// those logs would never expire. They get the default retention instead. A live account +// can delete its own settings row, so "no settings" must not be mistaken for "deleted": +// that account's logs are left alone, as are those of an account that keeps logs forever. +func TestCleanupAccessLogs_RealStore_DeletedAccount(t *testing.T) { + ctx := context.Background() + s, cleanup, err := store.NewTestStoreFromSQL(ctx, "", t.TempDir()) + require.NoError(t, err, "real sqlite test store must come up") + defer cleanup() + + const ( + deletedAccountID = "acc-deleted" + keepAccountID = "acc-keep-forever" + noSettingsAccountID = "acc-live-no-settings" + ) + old := time.Now().UTC().AddDate(0, 0, -(types.DefaultAccessLogRetentionDays + 10)) + recent := time.Now().UTC().AddDate(0, 0, -1) + + require.NoError(t, s.SaveAccount(ctx, &nbtypes.Account{Id: keepAccountID})) + require.NoError(t, s.SaveAccount(ctx, &nbtypes.Account{Id: noSettingsAccountID})) + + keepSettings := types.DefaultSettings(keepAccountID) + keepSettings.Domain = "keep.gw.example.com" + keepSettings.AccessLogRetentionDays = 0 + require.NoError(t, s.SaveAgentNetworkSettings(ctx, keepSettings)) + + mkLog := func(id, accountID string, ts time.Time) { + t.Helper() + entry := &types.AgentNetworkAccessLog{ + ID: id, AccountID: accountID, ServiceID: "svc", Timestamp: ts, StatusCode: 200, Model: "gpt-4o", + } + groups := []types.AgentNetworkAccessLogGroup{{LogID: id, GroupID: "grp-eng", AccountID: accountID}} + require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, entry, groups)) + } + mkLog("deleted-old", deletedAccountID, old) + mkLog("deleted-recent", deletedAccountID, recent) + mkLog("keep-old", keepAccountID, old) + mkLog("no-settings-old", noSettingsAccountID, old) + + m := &managerImpl{store: s} + m.cleanupAccessLogsOnce(ctx) + + logIDs := func(accountID string) []string { + t.Helper() + logs, _, err := s.GetAgentNetworkAccessLogs(ctx, store.LockingStrengthNone, accountID, + types.AgentNetworkAccessLogFilter{Page: 1, PageSize: 50}) + require.NoError(t, err) + ids := make([]string, 0, len(logs)) + for _, l := range logs { + ids = append(ids, l.ID) + } + return ids + } + assert.Equal(t, []string{"deleted-recent"}, logIDs(deletedAccountID), + "a deleted account should have logs past the default retention swept") + assert.Equal(t, []string{"keep-old"}, logIDs(keepAccountID), + "an account with retention disabled should keep its old logs") + assert.Equal(t, []string{"no-settings-old"}, logIDs(noSettingsAccountID), + "a live account without a settings row should keep its old logs") +} diff --git a/management/internals/modules/agentnetwork/manager.go b/management/internals/modules/agentnetwork/manager.go index d1a5ebd7b..71ff53214 100644 --- a/management/internals/modules/agentnetwork/manager.go +++ b/management/internals/modules/agentnetwork/manager.go @@ -80,6 +80,9 @@ type Manager interface { ListAccessLogSessions(ctx context.Context, accountID, userID string, filter types.AgentNetworkAccessLogFilter) ([]*types.AgentNetworkAccessLogSession, int64, error) GetUsageOverview(ctx context.Context, accountID, userID string, filter types.AgentNetworkAccessLogFilter, granularity types.UsageGranularity) ([]*types.AgentNetworkUsageBucket, error) StartAccessLogCleanup(ctx context.Context, cleanupIntervalHours int) + // RemoveAccountGateway drops the account's gateway mappings from the + // proxies. It runs as an account deletion hook. + RemoveAccountGateway(ctx context.Context, accountID string) error RecordConsumption(ctx context.Context, accountID string, kind types.ConsumptionDimension, dimID string, windowSeconds, tokensIn, tokensOut int64, costUSD float64) error RecordAccountBudgetUsage(ctx context.Context, accountID, userID string, groupIDs []string, tokensIn, tokensOut int64, costUSD float64) error RecordUsage(ctx context.Context, in RecordUsageInput) error @@ -1350,8 +1353,8 @@ func (m *managerImpl) scopeFilterToCaller(ctx context.Context, accountID, userID // StartAccessLogCleanup launches a background sweep that periodically deletes // each account's agent-network access-log rows older than that account's -// AccessLogRetentionDays. Usage records are never swept. A non-positive -// interval defaults to 24h. +// AccessLogRetentionDays, and the consumption counters of deleted accounts. +// Usage records are never swept. A non-positive interval defaults to 24h. func (m *managerImpl) StartAccessLogCleanup(ctx context.Context, cleanupIntervalHours int) { if cleanupIntervalHours <= 0 { cleanupIntervalHours = 24 @@ -1362,21 +1365,40 @@ func (m *managerImpl) StartAccessLogCleanup(ctx context.Context, cleanupInterval ticker := time.NewTicker(interval) defer ticker.Stop() - m.cleanupAccessLogsOnce(ctx) // run once on startup + m.cleanupOnce(ctx) // run once on startup for { select { case <-ctx.Done(): return case <-ticker.C: - m.cleanupAccessLogsOnce(ctx) + m.cleanupOnce(ctx) } } }() } +func (m *managerImpl) cleanupOnce(ctx context.Context) { + m.cleanupAccessLogsOnce(ctx) + m.cleanupDeletedAccountConsumption(ctx) +} + +// cleanupDeletedAccountConsumption deletes the consumption counters of accounts +// that no longer exist. Best-effort: a failure is logged and retried next sweep. +func (m *managerImpl) cleanupDeletedAccountConsumption(ctx context.Context) { + deleted, err := m.store.DeleteAgentNetworkConsumptionOfDeletedAccounts(ctx) + if err != nil { + log.WithContext(ctx).Warnf("agent-network consumption cleanup: %v", err) + return + } + if deleted > 0 { + log.WithContext(ctx).Infof("agent-network consumption cleanup: deleted %d counters of deleted accounts", deleted) + } +} + // cleanupAccessLogsOnce sweeps every account's expired access-log rows against -// its configured retention. Best-effort: a per-account failure is logged and -// the sweep continues. +// its configured retention. Deleted accounts, whose settings rows went with +// them, get the default retention. Best-effort: a per-account failure is +// logged and the sweep continues. func (m *managerImpl) cleanupAccessLogsOnce(ctx context.Context) { settings, err := m.store.GetAllAgentNetworkSettings(ctx, store.LockingStrengthNone) if err != nil { @@ -1384,18 +1406,31 @@ func (m *managerImpl) cleanupAccessLogsOnce(ctx context.Context) { return } for _, s := range settings { - if s.AccessLogRetentionDays <= 0 { - continue // keep indefinitely - } - cutoff := time.Now().UTC().AddDate(0, 0, -s.AccessLogRetentionDays) - deleted, err := m.store.DeleteOldAgentNetworkAccessLogs(ctx, s.AccountID, cutoff) - if err != nil { - log.WithContext(ctx).Warnf("agent-network access-log cleanup for account %s: %v", s.AccountID, err) - continue - } - if deleted > 0 { - log.WithContext(ctx).Infof("agent-network access-log cleanup: deleted %d rows for account %s (retention %d days)", deleted, s.AccountID, s.AccessLogRetentionDays) - } + m.cleanupAccountAccessLogs(ctx, s.AccountID, s.AccessLogRetentionDays) + } + + deleted, err := m.store.GetDeletedAccountIDsWithAgentNetworkAccessLogs(ctx) + if err != nil { + log.WithContext(ctx).Errorf("agent-network access-log cleanup: list deleted accounts: %v", err) + return + } + for _, accountID := range deleted { + m.cleanupAccountAccessLogs(ctx, accountID, types.DefaultAccessLogRetentionDays) + } +} + +func (m *managerImpl) cleanupAccountAccessLogs(ctx context.Context, accountID string, retentionDays int) { + if retentionDays <= 0 { + return // keep indefinitely + } + cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays) + deleted, err := m.store.DeleteOldAgentNetworkAccessLogs(ctx, accountID, cutoff) + if err != nil { + log.WithContext(ctx).Warnf("agent-network access-log cleanup for account %s: %v", accountID, err) + return + } + if deleted > 0 { + log.WithContext(ctx).Infof("agent-network access-log cleanup: deleted %d rows for account %s (retention %d days)", deleted, accountID, retentionDays) } } @@ -1545,6 +1580,8 @@ func (*mockManager) GetUsageOverview(_ context.Context, _, _ string, _ types.Age func (*mockManager) StartAccessLogCleanup(_ context.Context, _ int) {} +func (*mockManager) RemoveAccountGateway(_ context.Context, _ string) error { return nil } + func (*mockManager) RecordConsumption(_ context.Context, _ string, _ types.ConsumptionDimension, _ string, _, _, _ int64, _ float64) error { return nil } diff --git a/management/internals/modules/agentnetwork/reconcile.go b/management/internals/modules/agentnetwork/reconcile.go index 69e684014..20d0bb42a 100644 --- a/management/internals/modules/agentnetwork/reconcile.go +++ b/management/internals/modules/agentnetwork/reconcile.go @@ -2,8 +2,10 @@ package agentnetwork import ( "context" + "fmt" log "github.com/sirupsen/logrus" + goproto "google.golang.org/protobuf/proto" rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service" "github.com/netbirdio/netbird/management/server/types" @@ -81,18 +83,66 @@ func (m *managerImpl) reconcile(ctx context.Context, accountID string) { } m.reconcileMu.Unlock() - for _, entry := range creates { - entry.mapping.Type = proto.ProxyMappingUpdateType_UPDATE_TYPE_CREATED - m.proxyController.SendServiceUpdateToCluster(ctx, accountID, entry.mapping, entry.cluster) + m.sendMappings(ctx, accountID, creates, proto.ProxyMappingUpdateType_UPDATE_TYPE_CREATED) + m.sendMappings(ctx, accountID, updates, proto.ProxyMappingUpdateType_UPDATE_TYPE_MODIFIED) + m.sendMappings(ctx, accountID, deletes, proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED) +} + +// sendMappings sends each entry as updateType. It sends a copy: the entries' +// mappings are shared with reconcileCache, which another reconcile or +// RemoveAccountGateway may be reading, so they are never written. +func (m *managerImpl) sendMappings(ctx context.Context, accountID string, entries []syntheticMapping, updateType proto.ProxyMappingUpdateType) { + for _, entry := range entries { + update := goproto.Clone(entry.mapping).(*proto.ProxyMapping) + update.Type = updateType + m.proxyController.SendServiceUpdateToCluster(ctx, accountID, update, entry.cluster) } - for _, entry := range updates { - entry.mapping.Type = proto.ProxyMappingUpdateType_UPDATE_TYPE_MODIFIED - m.proxyController.SendServiceUpdateToCluster(ctx, accountID, entry.mapping, entry.cluster) +} + +// RemoveAccountGateway tells the proxies to drop every mapping of the account's +// gateway, so a deleted account's proxy config, provider API keys included, does +// not linger in proxy memory until the next resync. It is an account deletion +// hook: it runs before the account's data is removed, the last point at which +// the mappings can be synthesised from the store. The cache alone would miss +// them, since it is per instance and empty after a restart. If the deletion +// then fails, the gateway stays down until the account's next change reconciles +// it back. +func (m *managerImpl) RemoveAccountGateway(ctx context.Context, accountID string) error { + if m.proxyController == nil { + return nil } - for _, entry := range deletes { - entry.mapping.Type = proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED - m.proxyController.SendServiceUpdateToCluster(ctx, accountID, entry.mapping, entry.cluster) + + services, err := SynthesizeServices(ctx, m.store, accountID) + if err != nil { + return fmt.Errorf("synthesise agent network services: %w", err) } + oidcCfg := m.proxyController.GetOIDCValidationConfig() + removed := make(map[string]syntheticMapping, len(services)) + for _, svc := range services { + if svc == nil || svc.ID == "" { + continue + } + removed[svc.ID] = syntheticMapping{ + mapping: svc.ToProtoMapping(rpservice.Delete, "", oidcCfg), + cluster: svc.ProxyCluster, + } + } + + m.reconcileMu.Lock() + for id, entry := range m.reconcileCache[accountID] { + if _, ok := removed[id]; !ok { + removed[id] = entry + } + } + delete(m.reconcileCache, accountID) + m.reconcileMu.Unlock() + + entries := make([]syntheticMapping, 0, len(removed)) + for _, entry := range removed { + entries = append(entries, entry) + } + m.sendMappings(ctx, accountID, entries, proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED) + return nil } // diffMappings classifies the previous→current transition for a single diff --git a/management/internals/modules/agentnetwork/reconcile_test.go b/management/internals/modules/agentnetwork/reconcile_test.go index ab3b08481..2cfea9828 100644 --- a/management/internals/modules/agentnetwork/reconcile_test.go +++ b/management/internals/modules/agentnetwork/reconcile_test.go @@ -2,6 +2,8 @@ package agentnetwork import ( "context" + "sync" + "sync/atomic" "testing" "go.uber.org/mock/gomock" @@ -12,6 +14,7 @@ import ( "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" "github.com/netbirdio/netbird/management/server/store" "github.com/netbirdio/netbird/shared/management/proto" + "github.com/netbirdio/netbird/shared/management/status" ) func newReconcileMgr(t *testing.T, ctrl *gomock.Controller) (*managerImpl, *store.MockStore, *proxy.MockController) { @@ -287,3 +290,154 @@ func TestDiffMappings_RemovedServiceIsDeletedOnItsOwnCluster(t *testing.T) { assert.Equal(t, "brave-otter.gateway.example.com", deletes[0].cluster) } } + +// TestRemoveAccountGateway_EmitsRemovedFromStore — account deletion runs on an +// instance that may never have reconciled the account, so its cache is empty. +// The mappings are synthesised from the store, still intact before the delete, +// and each is sent as REMOVED to the cluster that serves it. +func TestRemoveAccountGateway_EmitsRemovedFromStore(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mgr, mockStore, mockProxy := newReconcileMgr(t, ctrl) + provider := newReconcileTestProvider() + policy := newReconcileTestPolicy(provider.ID, "grp-eng") + + expectReconcileSynthInputs(mockStore, ctx, []*types.Provider{provider}, []*types.Policy{policy}, []*types.Guardrail{}) + mockProxy.EXPECT().GetOIDCValidationConfig().Return(proxy.OIDCValidationConfig{}) + + var sent []*proto.ProxyMapping + mockProxy.EXPECT(). + SendServiceUpdateToCluster(ctx, "acct-1", gomock.Any(), "eu.proxy.netbird.io"). + Do(func(_ context.Context, _ string, m *proto.ProxyMapping, _ string) { + sent = append(sent, m) + }) + + require.NoError(t, mgr.RemoveAccountGateway(ctx, "acct-1")) + + require.Len(t, sent, 1, "the account's one gateway mapping must be removed") + assert.Equal(t, proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED, sent[0].Type, "the update must be a removal") + assert.Equal(t, "agent-net-svc-acct-1", sent[0].Id, "the removal must name the account's gateway service") +} + +// TestRemoveAccountGateway_AlsoRemovesCachedMappings — a mapping this instance +// last sent but the store no longer synthesises (here, one on another cluster) +// is removed too, and the account's cache entry is cleared. +func TestRemoveAccountGateway_AlsoRemovesCachedMappings(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mgr, mockStore, mockProxy := newReconcileMgr(t, ctrl) + mgr.reconcileCache["acct-1"] = map[string]syntheticMapping{ + "stale-svc": {mapping: &proto.ProxyMapping{Id: "stale-svc"}, cluster: "us.proxy.netbird.io"}, + } + + // Settings but no providers: the store synthesises nothing. + mockStore.EXPECT(). + GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "acct-1"). + Return(newReconcileTestSettings(), nil) + mockStore.EXPECT(). + GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, "acct-1"). + Return([]*types.Provider{}, nil) + mockProxy.EXPECT().GetOIDCValidationConfig().Return(proxy.OIDCValidationConfig{}) + + var sent []*proto.ProxyMapping + mockProxy.EXPECT(). + SendServiceUpdateToCluster(ctx, "acct-1", gomock.Any(), "us.proxy.netbird.io"). + Do(func(_ context.Context, _ string, m *proto.ProxyMapping, _ string) { + sent = append(sent, m) + }) + + require.NoError(t, mgr.RemoveAccountGateway(ctx, "acct-1")) + + require.Len(t, sent, 1, "the cached mapping must be removed from its own cluster") + assert.Equal(t, "stale-svc", sent[0].Id) + assert.Equal(t, proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED, sent[0].Type) + mgr.reconcileMu.Lock() + _, present := mgr.reconcileCache["acct-1"] + mgr.reconcileMu.Unlock() + assert.False(t, present, "the deleted account's cache entry must be cleared") +} + +// TestRemoveAccountGateway_SynthFailureAbortsDeletion — if the mappings cannot +// be read, nothing is sent and the error is returned, which as an account +// deletion hook keeps the account rather than leaving its gateway running. +func TestRemoveAccountGateway_SynthFailureAbortsDeletion(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mgr, mockStore, _ := newReconcileMgr(t, ctrl) + mockStore.EXPECT(). + GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "acct-1"). + Return(nil, status.Errorf(status.Internal, "store unavailable")) + + assert.Error(t, mgr.RemoveAccountGateway(ctx, "acct-1"), "a failed synthesis must fail the hook") +} + +func TestRemoveAccountGateway_NilProxyController_NoOp(t *testing.T) { + mgr := &managerImpl{reconcileCache: make(map[string]map[string]syntheticMapping)} + // Must not panic and must not query the store. + assert.NoError(t, mgr.RemoveAccountGateway(context.Background(), "acct-1")) +} + +// TestReconcile_ConcurrentWithGatewayChanges — while an account's gateway +// flaps (its policy is removed and re-added between reads), concurrent +// reconciles and RemoveAccountGateway share the cached mappings: one caches a +// mapping and sends it, another finds it gone and sends its removal. Run under +// -race: neither path may write a cached mapping, only copies of it. +func TestReconcile_ConcurrentWithGatewayChanges(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mgr, mockStore, mockProxy := newReconcileMgr(t, ctrl) + // gomock serialises every call on the controller's mutex, which would give + // the race detector the ordering the code under test lacks. The sends go + // through a fake that takes no lock. + mgr.proxyController = unsyncedSender{MockController: mockProxy} + provider := newReconcileTestProvider() + policy := newReconcileTestPolicy(provider.ID, "grp-eng") + + var reads atomic.Int64 + mockStore.EXPECT().GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "acct-1").Return(newReconcileTestSettings(), nil).AnyTimes() + mockStore.EXPECT().GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, "acct-1").Return([]*types.Provider{provider}, nil).AnyTimes() + mockStore.EXPECT().GetAccountAgentNetworkPolicies(ctx, store.LockingStrengthNone, "acct-1"). + DoAndReturn(func(context.Context, store.LockingStrength, string) ([]*types.Policy, error) { + if reads.Add(1)%2 == 0 { + return []*types.Policy{}, nil + } + return []*types.Policy{policy}, nil + }).AnyTimes() + mockStore.EXPECT().GetAccountAgentNetworkGuardrails(ctx, store.LockingStrengthNone, "acct-1").Return([]*types.Guardrail{}, nil).AnyTimes() + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(remove bool) { + defer wg.Done() + for j := 0; j < 50; j++ { + if remove && j%10 == 0 { + _ = mgr.RemoveAccountGateway(ctx, "acct-1") + continue + } + mgr.reconcile(ctx, "acct-1") + } + }(i == 0) + } + wg.Wait() +} + +// unsyncedSender answers the calls reconcile makes on every pass without any +// locking, so concurrent callers are not ordered by the fake itself. +type unsyncedSender struct { + *proxy.MockController +} + +func (unsyncedSender) GetOIDCValidationConfig() proxy.OIDCValidationConfig { + return proxy.OIDCValidationConfig{} +} + +func (unsyncedSender) SendServiceUpdateToCluster(context.Context, string, *proto.ProxyMapping, string) {} diff --git a/management/internals/server/modules.go b/management/internals/server/modules.go index 6b1365f3b..4840e40ad 100644 --- a/management/internals/server/modules.go +++ b/management/internals/server/modules.go @@ -103,6 +103,7 @@ func (s *BaseServer) AccountManager() account.Manager { s.AfterInit(func(s *BaseServer) { accountManager.SetServiceManager(s.ServiceManager()) + accountManager.AddAccountDeletionHook(s.AgentNetworkManager().RemoveAccountGateway) }) return accountManager diff --git a/management/server/account.go b/management/server/account.go index 6ccf673f5..340bcc84b 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -112,6 +112,9 @@ type DefaultAccountManager struct { permissionsManager permissions.Manager disableDefaultPolicy bool + + deletionHooksMu sync.RWMutex + deletionHooks []account.DeletionHook } var _ account.Manager = (*DefaultAccountManager)(nil) @@ -120,6 +123,32 @@ func (am *DefaultAccountManager) SetServiceManager(serviceManager service.Manage am.serviceManager = serviceManager } +// AddAccountDeletionHook registers hook to run on every account deletion. Hooks run in +// registration order, and the first one to fail stops the rest and aborts the deletion. +// It panics on a nil hook: dropping one silently would skip that hook's cleanup on every +// deletion, so the wiring bug surfaces at startup instead. +func (am *DefaultAccountManager) AddAccountDeletionHook(hook account.DeletionHook) { + if hook == nil { + panic("nil account deletion hook") + } + am.deletionHooksMu.Lock() + defer am.deletionHooksMu.Unlock() + am.deletionHooks = append(am.deletionHooks, hook) +} + +func (am *DefaultAccountManager) runAccountDeletionHooks(ctx context.Context, accountID string) error { + am.deletionHooksMu.RLock() + hooks := slices.Clone(am.deletionHooks) + am.deletionHooksMu.RUnlock() + + for _, hook := range hooks { + if err := hook(ctx, accountID); err != nil { + return fmt.Errorf("account deletion hook: %w", err) + } + } + return nil +} + func isUniqueConstraintError(err error) bool { switch { case strings.Contains(err.Error(), "(SQLSTATE 23505)"), @@ -889,6 +918,10 @@ func (am *DefaultAccountManager) DeleteAccount(ctx context.Context, accountID, u return status.Errorf(status.Internal, "failed to build user infos for account %s: %v", accountID, err) } + if err = am.runAccountDeletionHooks(ctx, accountID); err != nil { + return err + } + if err = am.deleteAccountUsers(ctx, accountID, userID, account.Users, userInfosMap); err != nil { return err } diff --git a/management/server/account/deletion_hook.go b/management/server/account/deletion_hook.go new file mode 100644 index 000000000..17c817444 --- /dev/null +++ b/management/server/account/deletion_hook.go @@ -0,0 +1,14 @@ +package account + +import "context" + +// DeletionHook runs when an account is deleted, after the caller's permission to delete +// it has been checked and before any of its users or data are removed. It lets code that +// keeps per-account state outside the store tear that state down while the account still +// exists. +// +// A hook that returns an error aborts the deletion and the account is kept. The caller +// sees the error, so a hook that wants a specific response returns a status error. A +// retried deletion runs every hook again, and a later step can still fail after the hooks +// succeed, so a hook must be idempotent and must tolerate the account surviving it. +type DeletionHook func(ctx context.Context, accountID string) error diff --git a/management/server/account_test.go b/management/server/account_test.go index c63782ca8..8c735b28e 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -958,6 +958,101 @@ func TestAccountManager_DeleteAccount(t *testing.T) { assert.Len(t, pats, 0) } +func TestAccountManager_DeleteAccount_RunsDeletionHooks(t *testing.T) { + manager, _, err := createManager(t) + require.NoError(t, err) + + ownerID := "account_creator" + account, err := createAccount(manager, "test_account", ownerID, "") + require.NoError(t, err) + + // Each hook records its call and checks the account is still in the store, which is + // the point of running before deletion: a hook must be able to read what it cleans up. + var calls []string + hook := func(name string) nbAccount.DeletionHook { + return func(ctx context.Context, accountID string) error { + calls = append(calls, name+":"+accountID) + _, err := manager.Store.GetAccount(ctx, accountID) + assert.NoError(t, err, "account should still exist while hook %s runs", name) + return nil + } + } + manager.AddAccountDeletionHook(hook("first")) + manager.AddAccountDeletionHook(hook("second")) + + require.NoError(t, manager.DeleteAccount(context.Background(), account.Id, ownerID)) + + assert.Equal(t, []string{"first:" + account.Id, "second:" + account.Id}, calls, + "hooks should run once each, in registration order, with the deleted account's ID") + _, err = manager.Store.GetAccount(context.Background(), account.Id) + assert.Error(t, err, "account should be deleted after the hooks succeed") +} + +func TestAccountManager_DeleteAccount_DeletionHookErrorAbortsDeletion(t *testing.T) { + manager, _, err := createManager(t) + require.NoError(t, err) + + ownerID := "account_creator" + account, err := createAccount(manager, "test_account", ownerID, "") + require.NoError(t, err) + + manager.AddAccountDeletionHook(func(context.Context, string) error { + return status.Errorf(status.PreconditionFailed, "teardown refused") + }) + secondCalled := false + manager.AddAccountDeletionHook(func(context.Context, string) error { + secondCalled = true + return nil + }) + + err = manager.DeleteAccount(context.Background(), account.Id, ownerID) + require.Error(t, err) + + // The hook's status type has to survive the wrapping, since the HTTP layer maps it + // to the response code. + sErr, ok := status.FromError(err) + require.True(t, ok, "error should carry the hook's status error, got %v", err) + assert.Equal(t, status.PreconditionFailed, sErr.Type(), "status type should be the hook's") + assert.False(t, secondCalled, "hooks after a failing one should not run") + + _, err = manager.Store.GetAccount(context.Background(), account.Id) + assert.NoError(t, err, "account should survive a failing hook") + _, err = manager.Store.GetUserByUserID(context.Background(), store.LockingStrengthNone, ownerID) + assert.NoError(t, err, "account owner should survive a failing hook") +} + +func TestAccountManager_AddAccountDeletionHook_RejectsNil(t *testing.T) { + manager, _, err := createManager(t) + require.NoError(t, err) + + assert.PanicsWithValue(t, "nil account deletion hook", func() { + manager.AddAccountDeletionHook(nil) + }, "registering a nil hook should panic instead of breaking a later deletion") +} + +func TestAccountManager_DeleteAccount_DeletionHooksSkippedWithoutPermission(t *testing.T) { + manager, _, err := createManager(t) + require.NoError(t, err) + + ownerID := "account_creator" + account, err := createAccount(manager, "test_account", ownerID, "") + require.NoError(t, err) + + adminID := "regular_admin" + account.Users[adminID] = types.NewAdminUser(adminID) + require.NoError(t, manager.Store.SaveAccount(context.Background(), account)) + + called := false + manager.AddAccountDeletionHook(func(context.Context, string) error { + called = true + return nil + }) + + err = manager.DeleteAccount(context.Background(), account.Id, adminID) + require.Error(t, err, "only the owner may delete the account") + assert.False(t, called, "hooks should not run for a caller who may not delete the account") +} + func BenchmarkTest_GetAccountWithclaims(b *testing.B) { claims := auth.UserAuth{ Domain: "example.com", diff --git a/management/server/store/sql_store_account.go b/management/server/store/sql_store_account.go index 8729c1f5a..43fe09861 100644 --- a/management/server/store/sql_store_account.go +++ b/management/server/store/sql_store_account.go @@ -18,6 +18,7 @@ import ( "gorm.io/gorm/clause" nbdns "github.com/netbirdio/netbird/dns" + agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" nbpeer "github.com/netbirdio/netbird/management/server/peer" "github.com/netbirdio/netbird/management/server/types" "github.com/netbirdio/netbird/route" @@ -162,6 +163,10 @@ func (s *SqlStore) DeleteAccount(ctx context.Context, account *types.Account) er return result.Error } + if err := deleteAgentNetworkAccountConfig(tx, account.Id); err != nil { + return err + } + result = tx.Select(clause.Associations).Delete(account) if result.Error != nil { return result.Error @@ -179,6 +184,29 @@ func (s *SqlStore) DeleteAccount(ctx context.Context, account *types.Account) er return err } +// deleteAgentNetworkAccountConfig removes the account's agent network configuration. These +// tables are not account associations, so deleting the account does not reach them. The +// settings row holds the account's globally unique gateway domain and the provider rows +// hold its upstream API keys. Tables that grow with traffic are left out: consumption +// counters and access logs are swept in the background, and usage records are kept. +func deleteAgentNetworkAccountConfig(tx *gorm.DB, accountID string) error { + // Dependents first: policies point at providers and guardrails, and settings + // go last, as DeleteSettings refuses while providers exist. + models := []any{ + &agentNetworkTypes.Policy{}, + &agentNetworkTypes.Provider{}, + &agentNetworkTypes.Guardrail{}, + &agentNetworkTypes.AccountBudgetRule{}, + &agentNetworkTypes.Settings{}, + } + for _, model := range models { + if err := tx.Delete(model, "account_id = ?", accountID).Error; err != nil { + return fmt.Errorf("delete %T rows: %w", model, err) + } + } + return nil +} + func (s *SqlStore) UpdateAccountDomainAttributes(ctx context.Context, accountID string, domain string, category string, isPrimaryDomain bool) error { accountCopy := types.Account{ Domain: domain, diff --git a/management/server/store/sql_store_account_test.go b/management/server/store/sql_store_account_test.go index 4c3b3f5fd..ec5976982 100644 --- a/management/server/store/sql_store_account_test.go +++ b/management/server/store/sql_store_account_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/require" nbdns "github.com/netbirdio/netbird/dns" + agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" proxydomain "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/domain" rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service" resourceTypes "github.com/netbirdio/netbird/management/server/networks/resources/types" @@ -401,6 +402,25 @@ func TestSqlite_DeleteAccount(t *testing.T) { require.NoError(t, err) require.Equal(t, o.AccountID, account.Id) + err = store.CreateAgentNetworkSettings(context.Background(), &agentNetworkTypes.Settings{ + AccountID: account.Id, + Domain: "gw.example.com", + ProxyAddress: "gw.example.com", + }) + require.NoError(t, err) + + agentNetworkConfig := []any{ + &agentNetworkTypes.Provider{ID: "an_provider", AccountID: account.Id, APIKey: "sk-test"}, + &agentNetworkTypes.Policy{ID: "an_policy", AccountID: account.Id}, + &agentNetworkTypes.Guardrail{ID: "an_guardrail", AccountID: account.Id}, + &agentNetworkTypes.AccountBudgetRule{ID: "an_budget_rule", AccountID: account.Id}, + } + for _, row := range agentNetworkConfig { + require.NoError(t, store.(*SqlStore).db.Create(row).Error, "creating %T", row) + } + otherProvider := &agentNetworkTypes.Provider{ID: "other_provider", AccountID: "other_account"} + require.NoError(t, store.(*SqlStore).db.Create(otherProvider).Error) + err = store.DeleteAccount(context.Background(), account) require.NoError(t, err) @@ -465,6 +485,32 @@ func TestSqlite_DeleteAccount(t *testing.T) { err = store.(*SqlStore).db.Model(&rpservice.Target{}).Find(&targets, "account_id = ?", account.Id).Error require.NoError(t, err, "expecting no error after DeleteAccount when searching for service targets") require.Len(t, targets, 0, "expecting no service targets to be found after DeleteAccount") + + _, err = store.GetAgentNetworkSettings(context.Background(), LockingStrengthNone, account.Id) + require.Error(t, err, "expecting agent network settings to be deleted with the account") + sErr, ok := status.FromError(err) + require.True(t, ok, "expecting a status error when getting agent network settings, got %v", err) + require.Equal(t, status.NotFound, sErr.Type(), "expecting agent network settings to be deleted with the account") + + // The domain is globally unique, so a leftover row would keep it from another account. + err = store.CreateAgentNetworkSettings(context.Background(), &agentNetworkTypes.Settings{ + AccountID: "other_account", + Domain: "gw.example.com", + ProxyAddress: "gw.example.com", + }) + require.NoError(t, err, "expecting the deleted account's gateway domain to be free for another account") + + for _, row := range agentNetworkConfig { + var count int64 + err = store.(*SqlStore).db.Model(row).Where("account_id = ?", account.Id).Count(&count).Error + require.NoError(t, err, "counting %T rows after DeleteAccount", row) + assert.Zero(t, count, "expecting no %T rows to be found after DeleteAccount", row) + } + + var otherProviders int64 + err = store.(*SqlStore).db.Model(&agentNetworkTypes.Provider{}).Where("account_id = ?", "other_account").Count(&otherProviders).Error + require.NoError(t, err) + assert.Equal(t, int64(1), otherProviders, "expecting another account's agent network provider to survive DeleteAccount") } func Test_GetAccount(t *testing.T) { diff --git a/management/server/store/sql_store_agent_network_access_log.go b/management/server/store/sql_store_agent_network_access_log.go index a1ae0150d..479dfa45e 100644 --- a/management/server/store/sql_store_agent_network_access_log.go +++ b/management/server/store/sql_store_agent_network_access_log.go @@ -69,6 +69,23 @@ func (s *SqlStore) DeleteOldAgentNetworkAccessLogs(ctx context.Context, accountI return deleted, nil } +// GetDeletedAccountIDsWithAgentNetworkAccessLogs returns the IDs of accounts that no +// longer exist but still have access-log rows. The retention sweep is driven by settings +// rows, which are deleted with the account, so it uses this to find logs it would +// otherwise never expire. +func (s *SqlStore) GetDeletedAccountIDsWithAgentNetworkAccessLogs(ctx context.Context) ([]string, error) { + var accountIDs []string + err := s.db.Model(&agentNetworkTypes.AgentNetworkAccessLog{}). + Distinct("account_id"). + Where("NOT EXISTS (SELECT 1 FROM accounts WHERE accounts.id = agent_network_access_log.account_id)"). + Pluck("account_id", &accountIDs).Error + if err != nil { + log.WithContext(ctx).Errorf("failed to get deleted accounts with agent-network access logs: %v", err) + return nil, status.Errorf(status.Internal, "failed to get deleted accounts with agent-network access logs") + } + return accountIDs, nil +} + // GetAgentNetworkAccessLogs retrieves flattened agent-network access logs for // an account with server-side pagination, filtering and sorting. Authorising // group ids are hydrated from the group child table for the returned page. diff --git a/management/server/store/sql_store_agentnetwork.go b/management/server/store/sql_store_agentnetwork.go index 4fe77d994..bdb1f97d6 100644 --- a/management/server/store/sql_store_agentnetwork.go +++ b/management/server/store/sql_store_agentnetwork.go @@ -663,6 +663,21 @@ func (s *SqlStore) IncrementAgentNetworkConsumptionBatch( return nil } +// DeleteAgentNetworkConsumptionOfDeletedAccounts deletes every consumption counter whose +// account no longer exists and returns the number of rows deleted. Counters grow with +// traffic, so they are swept in the background instead of in the account-deletion +// transaction, and the sweep also catches counters a proxy writes after the deletion. +func (s *SqlStore) DeleteAgentNetworkConsumptionOfDeletedAccounts(ctx context.Context) (int64, error) { + res := s.db. + Where("NOT EXISTS (SELECT 1 FROM accounts WHERE accounts.id = agent_network_consumption.account_id)"). + Delete(&agentNetworkTypes.Consumption{}) + if res.Error != nil { + log.WithContext(ctx).Errorf("failed to delete agent-network consumption of deleted accounts: %v", res.Error) + return 0, status.Errorf(status.Internal, "failed to delete agent-network consumption of deleted accounts") + } + return res.RowsAffected, nil +} + // ListAgentNetworkConsumption returns every consumption row recorded // for the account, ordered by window_start descending. Backs the // dashboard's basic counter view. diff --git a/management/server/store/sql_store_agentnetwork_accesslog_test.go b/management/server/store/sql_store_agentnetwork_accesslog_test.go index 8ba79a062..b8a3560e0 100644 --- a/management/server/store/sql_store_agentnetwork_accesslog_test.go +++ b/management/server/store/sql_store_agentnetwork_accesslog_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types" + "github.com/netbirdio/netbird/management/server/types" ) // TestAgentNetworkUsage_RealStore_RoundTrip drives CreateAgentNetworkUsage and @@ -300,3 +301,37 @@ func TestDeleteOldAgentNetworkAccessLogs(t *testing.T) { require.NoError(t, err) require.Len(t, usage, 1, "usage record for the deleted log must survive") } + +// TestDeleteAgentNetworkConsumptionOfDeletedAccounts verifies that the sweep removes the +// consumption counters of accounts that no longer exist and leaves live accounts' counters, +// including those of a live account without a settings row. +func TestDeleteAgentNetworkConsumptionOfDeletedAccounts(t *testing.T) { + runTestForAllEngines(t, "", func(t *testing.T, s Store) { + ctx := context.Background() + const ( + liveAccountID = "acc-anet-consumption-live" + deletedAccountID = "acc-anet-consumption-deleted" + ) + require.NoError(t, s.SaveAccount(ctx, &types.Account{Id: liveAccountID})) + + windowStart := time.Now().UTC().Truncate(time.Hour) + for _, accountID := range []string{liveAccountID, deletedAccountID} { + for _, dimID := range []string{"user-1", "user-2"} { + require.NoError(t, s.IncrementAgentNetworkConsumption(ctx, accountID, + agentNetworkTypes.DimensionUser, dimID, 3600, windowStart, 10, 5, 0.01)) + } + } + + deleted, err := s.DeleteAgentNetworkConsumptionOfDeletedAccounts(ctx) + require.NoError(t, err) + assert.Equal(t, int64(2), deleted, "both of the deleted account's counters should be removed") + + rows, err := s.ListAgentNetworkConsumption(ctx, LockingStrengthNone, deletedAccountID) + require.NoError(t, err) + assert.Empty(t, rows, "the deleted account should have no consumption counters left") + + rows, err = s.ListAgentNetworkConsumption(ctx, LockingStrengthNone, liveAccountID) + require.NoError(t, err) + assert.Len(t, rows, 2, "the live account's consumption counters should survive") + }) +} diff --git a/management/server/store/store.go b/management/server/store/store.go index 6d810643c..465f84413 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -320,6 +320,7 @@ type Store interface { GetAgentNetworkAccessLogSessions(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLogSession, int64, error) GetAgentNetworkUsageRows(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkUsage, error) DeleteOldAgentNetworkAccessLogs(ctx context.Context, accountID string, olderThan time.Time) (int64, error) + GetDeletedAccountIDsWithAgentNetworkAccessLogs(ctx context.Context) ([]string, error) GetServiceTargetByTargetID(ctx context.Context, lockStrength LockingStrength, accountID string, targetID string) (*rpservice.Target, error) GetTargetsByServiceID(ctx context.Context, lockStrength LockingStrength, accountID string, serviceID string) ([]*rpservice.Target, error) DeleteTarget(ctx context.Context, accountID string, serviceID string, targetID uint) error @@ -388,6 +389,7 @@ type Store interface { GetAgentNetworkConsumption(ctx context.Context, lockStrength LockingStrength, accountID string, kind agentNetworkTypes.ConsumptionDimension, dimID string, windowSeconds int64, windowStart time.Time) (*agentNetworkTypes.Consumption, error) GetAgentNetworkConsumptionBatch(ctx context.Context, lockStrength LockingStrength, accountID string, keys []agentNetworkTypes.ConsumptionKey) (map[agentNetworkTypes.ConsumptionKey]*agentNetworkTypes.Consumption, error) ListAgentNetworkConsumption(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*agentNetworkTypes.Consumption, error) + DeleteAgentNetworkConsumptionOfDeletedAccounts(ctx context.Context) (int64, error) GetAccountAgentNetworkBudgetRules(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*agentNetworkTypes.AccountBudgetRule, error) GetAgentNetworkBudgetRuleByID(ctx context.Context, lockStrength LockingStrength, accountID, ruleID string) (*agentNetworkTypes.AccountBudgetRule, error) SaveAgentNetworkBudgetRule(ctx context.Context, rule *agentNetworkTypes.AccountBudgetRule) error diff --git a/management/server/store/store_mock.go b/management/server/store/store_mock.go index 4dd7f9286..cd9e7334d 100644 --- a/management/server/store/store_mock.go +++ b/management/server/store/store_mock.go @@ -457,6 +457,21 @@ func (mr *MockStoreMockRecorder) DeleteAgentNetworkBudgetRule(ctx, accountID, ru return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAgentNetworkBudgetRule", reflect.TypeOf((*MockStore)(nil).DeleteAgentNetworkBudgetRule), ctx, accountID, ruleID) } +// DeleteAgentNetworkConsumptionOfDeletedAccounts mocks base method. +func (m *MockStore) DeleteAgentNetworkConsumptionOfDeletedAccounts(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAgentNetworkConsumptionOfDeletedAccounts", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteAgentNetworkConsumptionOfDeletedAccounts indicates an expected call of DeleteAgentNetworkConsumptionOfDeletedAccounts. +func (mr *MockStoreMockRecorder) DeleteAgentNetworkConsumptionOfDeletedAccounts(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAgentNetworkConsumptionOfDeletedAccounts", reflect.TypeOf((*MockStore)(nil).DeleteAgentNetworkConsumptionOfDeletedAccounts), ctx) +} + // DeleteAgentNetworkGuardrail mocks base method. func (m *MockStore) DeleteAgentNetworkGuardrail(ctx context.Context, accountID, guardrailID string) error { m.ctrl.T.Helper() @@ -1972,6 +1987,21 @@ func (mr *MockStoreMockRecorder) GetDNSRecordByID(ctx, lockStrength, accountID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDNSRecordByID", reflect.TypeOf((*MockStore)(nil).GetDNSRecordByID), ctx, lockStrength, accountID, zoneID, recordID) } +// GetDeletedAccountIDsWithAgentNetworkAccessLogs mocks base method. +func (m *MockStore) GetDeletedAccountIDsWithAgentNetworkAccessLogs(ctx context.Context) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDeletedAccountIDsWithAgentNetworkAccessLogs", ctx) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDeletedAccountIDsWithAgentNetworkAccessLogs indicates an expected call of GetDeletedAccountIDsWithAgentNetworkAccessLogs. +func (mr *MockStoreMockRecorder) GetDeletedAccountIDsWithAgentNetworkAccessLogs(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeletedAccountIDsWithAgentNetworkAccessLogs", reflect.TypeOf((*MockStore)(nil).GetDeletedAccountIDsWithAgentNetworkAccessLogs), ctx) +} + // GetEmbeddedProxyPeerIDsByCluster mocks base method. func (m *MockStore) GetEmbeddedProxyPeerIDsByCluster(ctx context.Context, accountID string) (map[string][]string, error) { m.ctrl.T.Helper() From 82e5428c2f75469fdbfb133b52d8d5d284734a21 Mon Sep 17 00:00:00 2001 From: Nicolas Frati Date: Wed, 30 Sep 2026 23:41:30 +0200 Subject: [PATCH 72/75] [management] Let usage_viewer read Agent Network access logs (#7750) usage_viewer saw account-wide usage but only its own request logs, so the people reviewing cost could not drill into the requests behind it. The role now also holds Read on agent_network.logs, which makes the access-log and session endpoints return every caller's rows instead of self-scoping. Logs can contain captured prompts, so this widens what the role exposes; policies, guardrails, budgets and settings stay hidden. Co-authored-by: Misha Bragin --- agent-network/README.md | 12 ++++----- .../permissions/agent_network_roles_test.go | 14 +++++------ .../server/permissions/roles/usage_viewer.go | 25 +++++++++++-------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/agent-network/README.md b/agent-network/README.md index 029ada299..35b9c6668 100644 --- a/agent-network/README.md +++ b/agent-network/README.md @@ -110,12 +110,12 @@ Two roles delegate Agent Network access without account-admin rights: read-only users, groups, peers, and account info (needed to build policies). Nothing else in the account. - **`usage_viewer`** — the regular User baseline plus read on - `agent_network.usage` (the aggregated usage and cost overview) and read-only - access to the resources the usage filters resolve against: users, groups, - peers, and the provider list (connection config redacted — no upstream URLs - or operator-supplied header values). No policies, and no account-wide - request-level access logs; like any caller, it still reads its own requests - through the self-scoped endpoints below. + `agent_network.usage` (the aggregated usage and cost overview) and + `agent_network.logs` (the account-wide request-level access logs, which can + contain captured prompts), and read-only access to the resources those + filters resolve against: users, groups, peers, and the provider list + (connection config redacted — no upstream URLs or operator-supplied header + values). No policies, guardrails, budgets, or settings. Every authenticated user, regardless of role, can read the caller-scoped self-service endpoint `GET /api/agent-network/agent-config` (the endpoint, providers, diff --git a/management/server/permissions/agent_network_roles_test.go b/management/server/permissions/agent_network_roles_test.go index 9ab708bd7..f5ad2000d 100644 --- a/management/server/permissions/agent_network_roles_test.go +++ b/management/server/permissions/agent_network_roles_test.go @@ -62,11 +62,11 @@ func TestAgentNetworkAdminRole(t *testing.T) { } } -// TestUsageViewerRole pins the least-privilege cost role: read on the -// aggregated usage overview plus read-only on the resources its filters -// and display columns resolve against (users, groups, peers, the provider -// list) — no policies, no request-level logs (which can contain captured -// prompts), nothing else in the account. +// TestUsageViewerRole pins the read-only usage role: read on the aggregated +// usage overview and the account-wide request-level logs, plus read-only on +// the resources their filters and display columns resolve against (users, +// groups, peers, the provider list) — no policies, guardrails, budgets, or +// settings, nothing else in the account. func TestUsageViewerRole(t *testing.T) { manager := NewManager(nil) ctx := context.Background() @@ -76,6 +76,7 @@ func TestUsageViewerRole(t *testing.T) { readOnly := []modules.Module{ modules.AgentNetworkUsage, + modules.AgentNetworkLogs, modules.AgentNetworkProviders, modules.Users, modules.Groups, @@ -83,7 +84,7 @@ func TestUsageViewerRole(t *testing.T) { } for _, m := range readOnly { assert.True(t, manager.ValidateRoleModuleAccess(ctx, "account", role, m, operations.Read), - "usage_viewer must read %s for the usage view and its filters", m) + "usage_viewer must read %s for the usage and log views and their filters", m) for _, op := range []operations.Operation{operations.Create, operations.Update, operations.Delete} { assert.False(t, manager.ValidateRoleModuleAccess(ctx, "account", role, m, op), "usage_viewer must not have %s on %s", op, m) @@ -95,7 +96,6 @@ func TestUsageViewerRole(t *testing.T) { modules.AgentNetworkPolicies, modules.AgentNetworkGuardrails, modules.AgentNetworkBudgets, - modules.AgentNetworkLogs, modules.AgentNetworkSettings, modules.Networks, modules.SetupKeys, diff --git a/management/server/permissions/roles/usage_viewer.go b/management/server/permissions/roles/usage_viewer.go index e480ae478..ab35a24db 100644 --- a/management/server/permissions/roles/usage_viewer.go +++ b/management/server/permissions/roles/usage_viewer.go @@ -7,16 +7,15 @@ import ( ) // UsageViewer is the regular User baseline plus read access to the -// aggregated Agent Network usage and cost overview, and read-only access -// to the resources the usage filters and display columns resolve against: -// users and groups (identity filters and name resolution), peers (agent -// principals in the caller column), and the provider list (provider and -// model filter options — the manager redacts connection config such as -// upstream URLs and operator-supplied header values for callers holding -// read without update). It sees no policies and no account-wide -// request-level access logs (which can contain captured prompts); its own -// requests remain readable through the self-scoped endpoints, like any -// caller's. +// aggregated Agent Network usage and cost overview and to the account-wide +// request-level access logs (which can contain captured prompts), and +// read-only access to the resources the usage and log filters and display +// columns resolve against: users and groups (identity filters and name +// resolution), peers (agent principals in the caller column), and the +// provider list (provider and model filter options — the manager redacts +// connection config such as upstream URLs and operator-supplied header +// values for callers holding read without update). It sees no policies, +// guardrails, budgets, or Agent Network settings. var UsageViewer = RolePermissions{ Role: types.UserRoleUsageViewer, AutoAllowNew: map[operations.Operation]bool{ @@ -32,6 +31,12 @@ var UsageViewer = RolePermissions{ operations.Update: false, operations.Delete: false, }, + modules.AgentNetworkLogs: { + operations.Read: true, + operations.Create: false, + operations.Update: false, + operations.Delete: false, + }, modules.AgentNetworkProviders: { operations.Read: true, operations.Create: false, From 6425b042002f0891e8d3142fd9371b87512b75bb Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:01:10 +0900 Subject: [PATCH 73/75] [client] Only treat LocalSystem as a privileged identity by SID on Windows (#7889) --- client/internal/ipcauth/forward_test.go | 2 +- client/internal/ipcauth/identity.go | 12 ++-- ...tity_sameuser_test.go => identity_test.go} | 55 +++++++++++++++++++ client/internal/ipcauth/privileged.go | 10 +++- client/internal/ipcauth/privileged_test.go | 27 ++++++++- 5 files changed, 97 insertions(+), 9 deletions(-) rename client/internal/ipcauth/{identity_sameuser_test.go => identity_test.go} (56%) diff --git a/client/internal/ipcauth/forward_test.go b/client/internal/ipcauth/forward_test.go index d9adf05da..d80c293be 100644 --- a/client/internal/ipcauth/forward_test.go +++ b/client/internal/ipcauth/forward_test.go @@ -38,7 +38,7 @@ func asDaemon(t *testing.T, id Identity) { prevID, prevKnown, prevDelegate := selfIdentity, selfKnown, selfMayDelegate t.Cleanup(func() { selfIdentity, selfKnown, selfMayDelegate = prevID, prevKnown, prevDelegate }) selfIdentity, selfKnown = id, true - selfMayDelegate = !id.IsPrivileged() + selfMayDelegate = mayDelegate(id) } func TestCallerIdentity_DirectConnections(t *testing.T) { diff --git a/client/internal/ipcauth/identity.go b/client/internal/ipcauth/identity.go index d7d10f57d..255585821 100644 --- a/client/internal/ipcauth/identity.go +++ b/client/internal/ipcauth/identity.go @@ -18,7 +18,8 @@ import ( "google.golang.org/grpc/peer" ) -// Well-known Windows SIDs that identify a fully privileged principal. +// Well-known Windows SIDs. Only LocalSystem and BUILTIN\Administrators identify a +// privileged principal; the service accounts are shared by unrelated services. const ( sidLocalSystem = "S-1-5-18" // NT AUTHORITY\SYSTEM sidLocalService = "S-1-5-19" // NT AUTHORITY\LOCAL SERVICE @@ -67,9 +68,9 @@ func (i Identity) IsWindows() bool { // user-to-root boundary. // // On Windows the decision comes from the caller's token rather than from -// account names or group RIDs: an elevated token, one of the service accounts -// the daemon itself may run as, or a token with BUILTIN\Administrators -// enabled. A UAC-filtered administrator has that group marked deny-only, and +// account names or group RIDs: an elevated token, the LocalSystem SID, or a +// token with BUILTIN\Administrators enabled. LocalService and NetworkService +// are not privileged by SID. A UAC-filtered administrator has that group marked deny-only, and // deny-only groups are dropped when the identity is captured, so such a // caller is correctly reported as unprivileged. Domain group memberships // (Domain Admins and friends) are deliberately not consulted: they say @@ -83,8 +84,7 @@ func (i Identity) IsPrivileged() bool { return true } - switch i.SID { - case sidLocalSystem, sidLocalService, sidNetworkService: + if i.SID == sidLocalSystem { return true } diff --git a/client/internal/ipcauth/identity_sameuser_test.go b/client/internal/ipcauth/identity_test.go similarity index 56% rename from client/internal/ipcauth/identity_sameuser_test.go rename to client/internal/ipcauth/identity_test.go index c98f583db..57be1b94e 100644 --- a/client/internal/ipcauth/identity_sameuser_test.go +++ b/client/internal/ipcauth/identity_test.go @@ -64,3 +64,58 @@ func TestIdentitySameUser(t *testing.T) { }) } } + +func TestIdentityIsPrivileged(t *testing.T) { + tests := []struct { + name string + id Identity + want bool + }{ + { + name: "Root", + id: Identity{UID: 0, GID: 0}, + want: true, + }, + { + name: "Non-root", + id: Identity{UID: 1000, GID: 1000}, + want: false, + }, + { + name: "Local system windows", + id: Identity{SID: sidLocalSystem}, + want: true, + }, + { + name: "Windows elevated", + id: Identity{SID: "S-1-5-21-1927267129-3959769253-3036563910-1001", Elevated: true}, + want: true, + }, + { + name: "Admin group windows", + id: Identity{SID: "S-1-5-21-1927267129-3959769253-3036563910-1001", Groups: []string{sidAdministrators}}, + want: true, + }, + { + name: "Regular user windows", + id: Identity{SID: "S-1-5-21-1927267129-3959769253-3036563910-1001"}, + want: false, + }, + { + name: "Network service windows", + id: Identity{SID: sidNetworkService}, + want: false, + }, + { + name: "Local service windows", + id: Identity{SID: sidLocalService}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.id.IsPrivileged()) + }) + } +} diff --git a/client/internal/ipcauth/privileged.go b/client/internal/ipcauth/privileged.go index 3c2e68432..54c66a5d2 100644 --- a/client/internal/ipcauth/privileged.go +++ b/client/internal/ipcauth/privileged.go @@ -45,7 +45,15 @@ func init() { // matching there would let a non-elevated shell of an administrator account // act as an administrator, which is the boundary the token check exists to // keep. - selfMayDelegate = !id.IsPrivileged() + selfMayDelegate = mayDelegate(id) +} + +// mayDelegate reports whether a daemon running as id may extend its authority to +// callers sharing its identity. The shared service accounts are excluded: their +// SID is held by unrelated services, so matching on it would grant them the +// daemon's authority. +func mayDelegate(id Identity) bool { + return !id.IsPrivileged() && id.SID != sidLocalService && id.SID != sidNetworkService } // IsDaemonSelf reports whether an identity is this very process. The JSON gateway diff --git a/client/internal/ipcauth/privileged_test.go b/client/internal/ipcauth/privileged_test.go index c1c7c1543..a6bbcf44b 100644 --- a/client/internal/ipcauth/privileged_test.go +++ b/client/internal/ipcauth/privileged_test.go @@ -98,7 +98,7 @@ func TestIsPrivilegedCaller_SelfRule(t *testing.T) { t.Cleanup(func() { selfIdentity, selfKnown, selfMayDelegate = prevID, prevKnown, prevDelegate }) selfIdentity, selfKnown = tt.self, tt.selfKnown - selfMayDelegate = tt.selfKnown && !tt.self.IsPrivileged() + selfMayDelegate = tt.selfKnown && mayDelegate(tt.self) if got := IsPrivilegedCaller(tt.caller); got != tt.want { t.Fatalf("IsPrivilegedCaller(%v) with daemon %v = %t, want %t", @@ -132,3 +132,28 @@ func TestIsPrivilegedCaller_ThisProcess(t *testing.T) { t.Errorf("an unrelated identity %v was treated as privileged", other) } } + +// The shared service accounts are held by unrelated services, so a daemon running +// as one of them must not extend its authority to every process with that SID. +func TestMayDelegate(t *testing.T) { + tests := []struct { + name string + self Identity + want bool + }{ + {name: "unprivileged unix user", self: Identity{UID: 1000}, want: true}, + {name: "root", self: Identity{UID: 0}, want: false}, + {name: "unprivileged windows user", self: Identity{SID: "S-1-5-21-1-2-3-1001"}, want: true}, + {name: "elevated windows user", self: Identity{SID: "S-1-5-21-1-2-3-1001", Elevated: true}, want: false}, + {name: "local system", self: Identity{SID: sidLocalSystem}, want: false}, + {name: "local service", self: Identity{SID: sidLocalService}, want: false}, + {name: "network service", self: Identity{SID: sidNetworkService}, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := mayDelegate(tt.self); got != tt.want { + t.Errorf("mayDelegate(%+v) = %v, want %v", tt.self, got, tt.want) + } + }) + } +} From 3906295446d33466c1579993dbf48ced23980c1d Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 1 Oct 2026 21:00:08 +0200 Subject: [PATCH 74/75] [client] Add Homebrew cask e2e test (#7618) * [client] Migrate macOS cask template to Homebrew install steps Homebrew deprecated the postflight and uninstall_preflight cask stanzas in favour of the declarative *_steps DSL, so every brew command that evaluates netbirdio/tap now prints deprecation warnings. Once the deprecation becomes a disable the generated cask stops loading and netbird-ui can no longer be installed or upgraded through Homebrew. The *_steps blocks take JSON-serialisable steps run in a sandbox rather than arbitrary Ruby, so system_command is re-expressed as run/remove. The two postflight blocks merge into one because a cask carries only a single instance, preserving the original order. set_permissions moves from a hardcoded /Applications to base: :appdir, matching what the installer invocation already did. The launchctl fallbacks keep their tolerant semantics through must_succeed: false, and remove is a no-op when the plist is absent. (cherry picked from commit df3756151f3af5c378aa74fc56dedc70e4adb052) * [client] Test the macOS Homebrew cask on a disposable runner The cask template only runs on real macOS with Homebrew, sudo and launchd, so changes to it have never been exercised before merge. This job installs the rendered cask on a GitHub macOS runner, walks the uninstall through a running, stopped and missing daemon, and reinstalls over the tap's published legacy cask, which is the path every existing user takes on their next upgrade. The fixture is the published cask itself rather than a pinned version and checksums, so the test follows each release instead of breaking at the next one. The installer scripts inside the signed archives are not under test, which is why their paths are left out of the trigger. * [client] Address SonarCloud findings in the Homebrew cask test Positional parameters move into local variables and the scenario switch gains an explicit default, so an unknown scenario fails instead of silently running the plain install and uninstall path. * [client] Make the Homebrew cask test deterministic with a stub bundle The released installer script opens the UI as root, which never returns on a headless runner, so a test that installs the published archive hangs until the job timeout. The cask itself never looks past two script paths and a version argument, so the test now builds a stub bundle on the runner, serves it from a local HTTP server and renders the template against it. The scripts ship without the executable bit, which turns the 0755 check into proof that set_permissions ran, and the stub records the version and uid it received. The published archives are still downloaded to assert the two script paths exist, and the published cask still supplies the legacy stanzas for the reinstall scenario. * [client] Drop the launchctl stderr check from the Homebrew cask test The test asserts what the cask template promises: install, uninstall and no deprecation warnings. Whether the uninstall steps print launchctl errors is a review remark on the template, not part of that contract. * [client] Retry the daemon start in the Homebrew cask test stub A reinstall runs the previous cask's bootout and the new postflight within a second of each other. launchd is still tearing the old daemon down at that point, so loading the same label again fails with EIO. The stub now retries the start for up to fifteen seconds, and the test still verifies afterwards that the daemon reached the running state. --------- Co-authored-by: Daniele Casciani --- .github/scripts/test-homebrew-cask.sh | 338 +++++++++++++++++++++++ .github/workflows/test-homebrew-cask.yml | 46 +++ client/ui/netbird-ui.rb.tmpl | 26 +- 3 files changed, 393 insertions(+), 17 deletions(-) create mode 100755 .github/scripts/test-homebrew-cask.sh create mode 100644 .github/workflows/test-homebrew-cask.yml diff --git a/.github/scripts/test-homebrew-cask.sh b/.github/scripts/test-homebrew-cask.sh new file mode 100755 index 000000000..9c1041bdd --- /dev/null +++ b/.github/scripts/test-homebrew-cask.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +if [[ ${RUNNER_ENVIRONMENT:-} != github-hosted || ${RUNNER_OS:-} != macOS || $(uname -s) != Darwin ]]; then + fail "This test installs a system daemon and must run on a disposable GitHub macOS runner." +fi +if [[ $EUID == 0 ]]; then + fail "Run this script as the Homebrew user, not root." +fi + +readonly test_dir="${RUNNER_TEMP:?}/homebrew-cask" +readonly results_dir="$test_dir/results" +readonly app='/Applications/Netbird UI.app' +readonly plist='/Library/LaunchDaemons/netbird.plist' +readonly cask='netbirdio/tap/netbird-ui' +readonly formula='netbirdio/tap/netbird' +readonly published_cask="$test_dir/published-netbird-ui.rb" +readonly legacy_cask="$test_dir/legacy-netbird-ui.rb" +readonly rendered_cask="$test_dir/rendered-netbird-ui.rb" +readonly fixture_dir="$test_dir/fixture" +readonly serve_dir="$test_dir/serve" +readonly fixture_zip="$serve_dir/netbird-ui.zip" +readonly fixture_port=18080 +readonly fixture_url="http://127.0.0.1:$fixture_port/netbird-ui.zip" +readonly marker="$test_dir/installer.marker" + +mkdir -p "$results_dir" "$fixture_dir/netbird_ui_darwin" "$serve_dir" "$test_dir/downloads" +exec > >(tee "$results_dir/test.log") 2>&1 + +sudo -n true +if command -v netbird || [[ -e "$app" || -e "$plist" ]] || pgrep -x netbird-ui; then + fail "The runner already has NetBird installed or running." +fi +if sudo launchctl print system/netbird > "$results_dir/initial-service.log" 2>&1; then + fail "The runner already has a NetBird service loaded." +fi + +install_attempted=false +server_pid='' +daemon_pid='' +version='' + +stop_ui() { + local status=0 + sudo pkill -x netbird-ui || status=$? + # pkill returns 1 when the UI is already closed. + [[ $status == 0 || $status == 1 ]] +} + +cleanup() { + local status=$? + trap - EXIT + set +e + + if [[ $install_attempted == true ]]; then + stop_ui || status=1 + if [[ -S /var/run/netbird.sock ]]; then + sudo netbird down || status=1 + fi + if brew list --cask "$cask" >/dev/null 2>&1 || [[ -e "$app" ]]; then + brew uninstall --cask --force "$cask" || status=1 + fi + # A failed cask install can leave a daemon even after Homebrew rolls back the app. + if sudo launchctl print system/netbird > "$results_dir/cleanup-service.log" 2>&1; then + sudo netbird service stop || status=1 + fi + if [[ -e "$plist" ]]; then + sudo netbird service uninstall || status=1 + fi + fi + if [[ -f /var/log/netbird/client.log ]]; then + sudo cat /var/log/netbird/client.log > "$results_dir/client.log" || status=1 + fi + if command -v netbird >/dev/null; then + brew uninstall --formula "$formula" || status=1 + fi + if [[ -n $server_pid ]]; then + kill "$server_pid" 2>/dev/null || true + fi + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +run_logged() { + local name=$1 + shift + "$@" 2>&1 | tee "$results_dir/$name.log" +} + +cask_field() { + local stanza=$1 file=$2 + sed -nE "s/^[[:space:]]*$stanza \"([^\"]+)\".*/\\1/p" "$file" +} + +release_fields() { + local file=$1 + grep -E '^[[:space:]]*(version|url|sha256|app) ' "$file" +} + +use_cask() { + local file=$1 + cp "$file" "$tap_dir/Casks/netbird-ui.rb" +} + +# The released installer opens the UI as root, which never returns on a headless +# runner. The cask only needs two script paths and a version argument, so the test +# ships a stub bundle that records what it received and starts the daemon. +build_fixture() { + local bundle="$fixture_dir/netbird_ui_darwin" + printf '#!/bin/sh\nexit 0\n' > "$bundle/netbird-ui" + chmod 755 "$bundle/netbird-ui" + # After a bootout launchd keeps tearing the previous daemon down for a couple of + # seconds, and loading the same label again fails until that finishes. + cat > "$bundle/installer.sh" < '$marker' +netbird service install +attempt=0 +until netbird service start; do + attempt=\$((attempt + 1)) + [ "\$attempt" -lt 15 ] || exit 1 + sleep 1 +done +EOF + printf '#!/bin/sh\nexit 0\n' > "$bundle/uninstaller.sh" + # Shipped without the executable bit so the 0755 seen after install can only come from the cask. + chmod 644 "$bundle/installer.sh" "$bundle/uninstaller.sh" + rm -f "$fixture_zip" + (cd "$fixture_dir" && zip -qr "$fixture_zip" netbird_ui_darwin) +} + +start_fixture_server() { + python3 -m http.server "$fixture_port" --bind 127.0.0.1 --directory "$serve_dir" \ + > "$results_dir/fixture-server.log" 2>&1 & + server_pid=$! + local attempt + for attempt in {1..20}; do + if curl --silent --fail --output /dev/null "$fixture_url"; then + return + fi + sleep 0.5 + done + fail "The fixture HTTP server did not come up on port $fixture_port." +} + +assert_published_layout() { + local url archive script + while read -r url; do + archive="$test_dir/downloads/${url##*/}" + curl --fail --location --silent --retry 3 --output "$archive" "$url" + for script in installer.sh uninstaller.sh; do + unzip -l "$archive" | grep -q " netbird_ui_darwin/$script\$" || + fail "The published archive ${url##*/} has no netbird_ui_darwin/$script." + done + done < <(cask_field url "$published_cask") +} + +assert_no_deprecations() { + if grep -Ei '(postflight|uninstall_preflight).*deprecated|deprecated.*(postflight|uninstall_preflight)' "$@"; then + fail "Homebrew reported a deprecated cask lifecycle hook." + fi +} + +wait_for_daemon() { + local attempt + for attempt in {1..30}; do + if sudo launchctl print system/netbird > "$results_dir/service.log" 2>&1 && + grep -Eq '^[[:space:]]*state = running$' "$results_dir/service.log"; then + return + fi + sleep 1 + done + cat "$results_dir/service.log" + fail "The installed daemon did not reach the running state." +} + +wait_for_exit() { + local pid=$1 attempt + for attempt in {1..30}; do + if ! sudo kill -0 "$pid" 2>/dev/null; then + return + fi + sleep 1 + done + fail "Daemon process $pid is still running after removal." +} + +assert_service_absent() { + if sudo launchctl print system/netbird > "$results_dir/removed-service.log" 2>&1; then + fail "The NetBird service is still loaded after removal." + fi +} + +assert_installed() { + local script + [[ -f $marker ]] || fail "The cask did not run installer.sh." + grep -qx "version=$version" "$marker" || fail "installer.sh did not receive the cask version: $(cat "$marker")" + grep -qx 'uid=0' "$marker" || fail "installer.sh did not run as root: $(cat "$marker")" + [[ -d "$app" && -x "$app/netbird-ui" ]] || fail "The UI was not installed." + for script in installer.sh uninstaller.sh; do + [[ $(stat -f '%Lp' "$app/$script") == 755 ]] || fail "Incorrect permissions on $script." + done + [[ -f "$plist" ]] || fail "The installer did not create the daemon plist." + wait_for_daemon + daemon_pid=$(awk '/^[[:space:]]*pid = / { print $3; exit }' "$results_dir/service.log") + [[ $daemon_pid =~ ^[0-9]+$ ]] || fail "The running daemon has no PID." + sudo kill -0 "$daemon_pid" +} + +assert_uninstalled() { + local log=$1 + assert_no_deprecations "$log" + [[ ! -e "$app" ]] || fail "The UI app remains after uninstall." + [[ ! -e "$plist" ]] || fail "The daemon plist remains after uninstall." + assert_service_absent + wait_for_exit "$daemon_pid" + [[ $(netbird version) == "$version" ]] || fail "Cask uninstall removed the CLI dependency." +} + +installed_caskfiles() { + local extension=$1 + find "$(brew --caskroom)/netbird-ui/.metadata" -name "netbird-ui.$extension" 2>/dev/null +} + +assert_legacy_metadata() { + installed_caskfiles rb | grep -q . || fail "The legacy cask did not leave a Ruby caskfile behind." +} + +assert_steps_metadata() { + if installed_caskfiles rb | grep -q .; then + fail "Homebrew still keeps the legacy Ruby caskfile after reinstall." + fi + installed_caskfiles json | grep -q . || fail "Homebrew did not save the reinstalled cask as JSON." +} + +brew --version +sw_vers +brew tap netbirdio/tap "${GITHUB_WORKSPACE:?}/.homebrew-cask-tap" +tap_dir=$(brew --repository netbirdio/tap) +readonly tap_dir + +[[ -f "$tap_dir/Casks/netbird-ui.rb" ]] || fail "The tap has no Casks/netbird-ui.rb." +cp "$tap_dir/Casks/netbird-ui.rb" "$published_cask" +cp "$published_cask" "$results_dir/published-netbird-ui.rb" + +version=$(brew info --json=v2 --formula "$formula" | jq -r '.formulae[0].versions.stable') +readonly version +[[ -n $version && $version != null ]] || fail "Could not read the formula version from the tap." + +assert_published_layout + +build_fixture +fixture_sha=$(shasum -a 256 "$fixture_zip" | cut -d' ' -f1) +readonly fixture_sha +start_fixture_server + +export PROJECT=netbird-ui VERSION="$version" +export AMD="$fixture_zip" ARM="$fixture_zip" AMD_URL="$fixture_url" ARM_URL="$fixture_url" +gomplate -f "$GITHUB_WORKSPACE/client/ui/netbird-ui.rb.tmpl" -o "$rendered_cask" +cp "$rendered_cask" "$results_dir/rendered-netbird-ui.rb" + +sed -E "s|^([[:space:]]*version) \"[^\"]+\"|\\1 \"$version\"|; s|^([[:space:]]*url) \"[^\"]+\"|\\1 \"$fixture_url\"|; s|^([[:space:]]*sha256) \"[^\"]+\"|\\1 \"$fixture_sha\"|" \ + "$published_cask" > "$legacy_cask" +cp "$legacy_cask" "$results_dir/legacy-netbird-ui.rb" +if ! diff <(release_fields "$legacy_cask") <(release_fields "$rendered_cask"); then + fail "The rendered cask changes release data, not only lifecycle stanzas." +fi + +use_cask "$rendered_cask" +brew info --json=v2 --cask "$cask" > "$results_dir/cask.json" 2> "$results_dir/load.log" +cat "$results_dir/load.log" +assert_no_deprecations "$results_dir/load.log" +run_logged style brew style --cask --only-cops=Cask/InstallSteps "$cask" + +run_logged install-cli brew install --formula "$formula" +[[ $(netbird version) == "$version" ]] || fail "The installed CLI does not report the formula version." + +for scenario in running stopped missing; do + echo "::group::Uninstall with $scenario service" + install_attempted=true + sudo rm -f "$marker" + run_logged "install-$scenario" brew install --cask "$cask" + assert_no_deprecations "$results_dir/install-$scenario.log" + assert_installed + stop_ui + + case "$scenario" in + running) ;; + stopped) + run_logged stop-daemon sudo netbird service stop + wait_for_exit "$daemon_pid" + [[ -f "$plist" ]] || fail "Stopping the daemon unexpectedly removed its plist." + ;; + missing) + run_logged stop-missing-daemon sudo netbird service stop + run_logged remove-daemon sudo netbird service uninstall + wait_for_exit "$daemon_pid" + [[ ! -e "$plist" ]] || fail "The missing-service scenario still has a plist." + assert_service_absent + ;; + *) fail "Unknown uninstall scenario: $scenario" ;; + esac + + run_logged "uninstall-$scenario" brew uninstall --cask "$cask" + assert_uninstalled "$results_dir/uninstall-$scenario.log" + echo "::endgroup::" +done + +# Every existing user first meets the new cask through an upgrade of the published +# one, whose legacy flight blocks Homebrew replays from the saved Ruby caskfile. +echo "::group::Reinstall over the published legacy cask" +install_attempted=true +use_cask "$legacy_cask" +sudo rm -f "$marker" +run_logged install-legacy brew install --cask "$cask" +assert_installed +assert_legacy_metadata +stop_ui + +use_cask "$rendered_cask" +sudo rm -f "$marker" +run_logged reinstall-legacy brew reinstall --cask "$cask" +assert_installed +assert_steps_metadata +stop_ui + +run_logged uninstall-legacy brew uninstall --cask "$cask" +assert_uninstalled "$results_dir/uninstall-legacy.log" +echo "::endgroup::" diff --git a/.github/workflows/test-homebrew-cask.yml b/.github/workflows/test-homebrew-cask.yml new file mode 100644 index 000000000..a75951425 --- /dev/null +++ b/.github/workflows/test-homebrew-cask.yml @@ -0,0 +1,46 @@ +name: Test Homebrew cask + +on: + pull_request: + paths: + - "client/ui/netbird-ui.rb.tmpl" + - ".github/scripts/test-homebrew-cask.sh" + - ".github/workflows/test-homebrew-cask.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + install-uninstall: + runs-on: macos-latest + timeout-minutes: 20 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Clone the Homebrew tap + run: git clone https://github.com/netbirdio/homebrew-tap.git .homebrew-cask-tap + + - name: Update Homebrew and install gomplate + # The runner image disables auto-update; the cask steps DSL needs Homebrew 6.0.20 or newer. + run: | + brew update + brew install gomplate + + - name: Install and uninstall the cask + run: .github/scripts/test-homebrew-cask.sh + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: homebrew-cask-results + path: ${{ runner.temp }}/homebrew-cask/results + if-no-files-found: ignore diff --git a/client/ui/netbird-ui.rb.tmpl b/client/ui/netbird-ui.rb.tmpl index 1c77e6717..a6ef59d3f 100644 --- a/client/ui/netbird-ui.rb.tmpl +++ b/client/ui/netbird-ui.rb.tmpl @@ -17,25 +17,17 @@ cask "{{ $projectName }}" do depends_on formula: "netbird" - postflight do - set_permissions "/Applications/Netbird UI.app/installer.sh", '0755' - set_permissions "/Applications/Netbird UI.app/uninstaller.sh", '0755' + postflight_steps do + set_permissions ["Netbird UI.app/installer.sh", "Netbird UI.app/uninstaller.sh"], "0755", + base: :appdir, recursive: false + run "Netbird UI.app/installer.sh", args: ["#{version}"], base: :appdir, sudo: true end - postflight do - system_command "#{appdir}/Netbird UI.app/installer.sh", - args: ["#{version}"], - sudo: true - end - - uninstall_preflight do - system_command "/bin/sh", - args: ["-c", <<~CMD], - launchctl bootout system/netbird 2>/dev/null || \ - launchctl unload /Library/LaunchDaemons/netbird.plist 2>/dev/null || true - rm -f /Library/LaunchDaemons/netbird.plist - CMD - sudo: true + uninstall_preflight_steps do + run "/bin/launchctl", args: ["bootout", "system/netbird"], sudo: true, must_succeed: false + run "/bin/launchctl", args: ["unload", "/Library/LaunchDaemons/netbird.plist"], + sudo: true, must_succeed: false + remove "/Library/LaunchDaemons/netbird.plist", sudo: true end name "Netbird UI" From 5ceca6e500fb453ddb55a917f55fb84929eabcbd Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 1 Oct 2026 21:01:56 +0200 Subject: [PATCH 75/75] [client] Report both peers' state when the connect test times out (#7944) Test_ConnectPeers fails every few weeks on the Linux runner with a bare "waiting for peer handshake timeout after 30s". The failing logs show both kernel devices up and both peers configured within a second, then nothing for 30 s, which is six retries of the 5 s handshake retransmit and so a condition that lasted the whole window rather than a race. The failure cannot be reproduced locally and the log cannot tell whether initiations were sent, whether they arrived, or whether only one direction worked. On timeout the test now prints each device's view of its peer, the endpoint, the byte counters and the last handshake, so the next failure says which of those it is. The comment also states that the peers are kernel devices on the runner and that the first initiation of each side is always lost to the other side not knowing the peer yet. --- client/iface/iface_test.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/client/iface/iface_test.go b/client/iface/iface_test.go index 89c8cd16e..fff0d4e30 100644 --- a/client/iface/iface_test.go +++ b/client/iface/iface_test.go @@ -568,11 +568,14 @@ func Test_ConnectPeers(t *testing.T) { if err != nil { t.Fatal(err) } - // The peers use userspace WireGuard (stdnet transport). A tight busy-loop - // here starves the wireguard-go goroutines that process the handshake, so - // poll on a ticker instead and yield the CPU between checks. WireGuard also - // only retries a lost handshake initiation every REKEY_TIMEOUT (5s), which - // is why the overall wait can occasionally stretch to tens of seconds. + // On Linux with the kernel module both peers are kernel devices, elsewhere + // they run on wireguard-go. A tight busy-loop here would starve the + // wireguard-go goroutines that process the handshake, so poll on a ticker + // instead and yield the CPU between checks. WireGuard also only retries a + // lost handshake initiation every REKEY_TIMEOUT (5s), which is why the + // overall wait can occasionally stretch to tens of seconds. Each side sends + // its first initiation when its peer is configured, and the first one leaves + // before the other device knows the peer, so that one is always wasted. timeout := 30 * time.Second timeoutChannel := time.After(timeout) ticker := time.NewTicker(500 * time.Millisecond) @@ -590,13 +593,26 @@ func Test_ConnectPeers(t *testing.T) { select { case <-timeoutChannel: - t.Fatalf("waiting for peer handshake timeout after %s", timeout.String()) + // The counters tell whether initiations were sent at all, whether they + // arrived, and whether only one direction is working. + t.Fatalf("waiting for peer handshake timeout after %s\n%s\n%s", timeout.String(), + describePeer(peer1ifaceName, peer2Key.PublicKey().String()), + describePeer(peer2ifaceName, peer1Key.PublicKey().String())) case <-ticker.C: } } } +func describePeer(ifaceName, peerPubKey string) string { + peer, err := getPeer(ifaceName, peerPubKey) + if err != nil { + return fmt.Sprintf("%s: peer %s: %v", ifaceName, peerPubKey, err) + } + return fmt.Sprintf("%s: peer %s endpoint=%v tx=%d rx=%d last_handshake=%v", + ifaceName, peerPubKey, peer.Endpoint, peer.TransmitBytes, peer.ReceiveBytes, peer.LastHandshakeTime) +} + func getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) { wg, err := wgctrl.New() if err != nil {