mirror of
https://github.com/netbirdio/netbird.git
synced 2026-09-26 08:39:06 +02:00
[client] Raise the daemon IPC receive limit above gRPC's 4 MB default (#7676)
Connections to the daemon were left on gRPC's own defaults, which cap a received message at 4 MB. A detailed status carries an entry per peer, so on a large deployment the response outgrows that cap and the command fails outright: netbird status -d Error: status failed: grpc: received message larger than max (4287609 vs. 4194304) The limit is raised where the daemon dial options are built, so every caller inherits it: the CLI, the desktop UI, the JSON gateway, and the SSH client and proxy. It is overridable through NB_DAEMON_GRPC_MAX_MSG_SIZE for a deployment that outgrows the new default too, mirroring what the management client already does with NB_MANAGEMENT_GRPC_MAX_MSG_SIZE, and reusing its 16 MB default. Only the receive direction needs raising. Requests to the daemon are small, and gRPC does not cap the send side by default, so the daemon could already send a response the caller then refused to read.
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
package daemonaddr
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvMaxRecvMsgSize overrides the default gRPC max receive message size for
|
||||
// connections to the daemon. Value is in bytes.
|
||||
EnvMaxRecvMsgSize = "NB_DAEMON_GRPC_MAX_MSG_SIZE"
|
||||
|
||||
// defaultMaxRecvMsgSize is the max gRPC receive message size used for daemon
|
||||
// connections when EnvMaxRecvMsgSize is unset or invalid. It overrides the
|
||||
// gRPC library default of 4 MB, which a detailed status already exceeds on a
|
||||
// network of a few thousand peers.
|
||||
defaultMaxRecvMsgSize = 1024 * 1024 * 16
|
||||
)
|
||||
|
||||
// MaxRecvMsgSize returns the max gRPC receive message size for daemon connections
|
||||
// from the environment, or defaultMaxRecvMsgSize (16 MB) if unset or invalid.
|
||||
func MaxRecvMsgSize() int {
|
||||
val := os.Getenv(EnvMaxRecvMsgSize)
|
||||
if val == "" {
|
||||
return defaultMaxRecvMsgSize
|
||||
}
|
||||
|
||||
size, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
log.Warnf("invalid %s value %q, using default: %v", EnvMaxRecvMsgSize, val, err)
|
||||
return defaultMaxRecvMsgSize
|
||||
}
|
||||
|
||||
if size <= 0 {
|
||||
log.Warnf("invalid %s value %d, must be positive, using default", EnvMaxRecvMsgSize, size)
|
||||
return defaultMaxRecvMsgSize
|
||||
}
|
||||
|
||||
return size
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package daemonaddr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
)
|
||||
|
||||
func TestMaxRecvMsgSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
envValue string
|
||||
expected int
|
||||
}{
|
||||
{name: "unset returns default", envValue: "", expected: defaultMaxRecvMsgSize},
|
||||
{name: "non-numeric returns default", envValue: "abc", expected: defaultMaxRecvMsgSize},
|
||||
{name: "negative returns default", envValue: "-1", expected: defaultMaxRecvMsgSize},
|
||||
{name: "zero returns default", envValue: "0", expected: defaultMaxRecvMsgSize},
|
||||
{name: "valid value is used", envValue: "33554432", expected: 33554432},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Set first so the previous value is restored on cleanup, then unset to
|
||||
// exercise the absent case.
|
||||
t.Setenv(EnvMaxRecvMsgSize, tc.envValue)
|
||||
if tc.envValue == "" {
|
||||
require.NoError(t, os.Unsetenv(EnvMaxRecvMsgSize), "unset the override")
|
||||
}
|
||||
|
||||
assert.Equal(t, tc.expected, MaxRecvMsgSize(), "max receive message size")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// bigStatusServer answers Status with a response larger than gRPC's 4 MB default
|
||||
// receive limit, which is what a detailed status on a large network looks like.
|
||||
type bigStatusServer struct {
|
||||
proto.UnimplementedDaemonServiceServer
|
||||
payload string
|
||||
}
|
||||
|
||||
func (s *bigStatusServer) Status(context.Context, *proto.StatusRequest) (*proto.StatusResponse, error) {
|
||||
return &proto.StatusResponse{Status: s.payload}, nil
|
||||
}
|
||||
|
||||
func startBigStatusServer(t *testing.T, payload string) string {
|
||||
t.Helper()
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err, "listen on loopback")
|
||||
|
||||
srv := grpc.NewServer()
|
||||
proto.RegisterDaemonServiceServer(srv, &bigStatusServer{payload: payload})
|
||||
go func() {
|
||||
_ = srv.Serve(listener)
|
||||
}()
|
||||
t.Cleanup(srv.Stop)
|
||||
|
||||
return "tcp://" + listener.Addr().String()
|
||||
}
|
||||
|
||||
func TestDialTargetAcceptsAStatusOverTheGrpcDefault(t *testing.T) {
|
||||
payload := strings.Repeat("x", 5*1024*1024)
|
||||
addr := startBigStatusServer(t, payload)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
target, opts := DialTarget(addr)
|
||||
conn, err := grpc.NewClient(target, opts...)
|
||||
require.NoError(t, err, "dial the daemon")
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
|
||||
resp, err := proto.NewDaemonServiceClient(conn).Status(ctx, &proto.StatusRequest{})
|
||||
require.NoError(t, err, "a detailed status must not be rejected for its size")
|
||||
assert.Len(t, resp.GetStatus(), len(payload), "the whole response must arrive")
|
||||
}
|
||||
|
||||
// TestDialTargetRaisesTheDefaultLimit is the negative control: the same response
|
||||
// over a connection carrying gRPC's own defaults is refused, which is the failure
|
||||
// reported by `netbird status -d` on a large deployment.
|
||||
func TestDialTargetRaisesTheDefaultLimit(t *testing.T) {
|
||||
payload := strings.Repeat("x", 5*1024*1024)
|
||||
addr := startBigStatusServer(t, payload)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
conn, err := grpc.NewClient(
|
||||
strings.TrimPrefix(addr, "tcp://"),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
require.NoError(t, err, "dial with the library defaults")
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
|
||||
_, err = proto.NewDaemonServiceClient(conn).Status(ctx, &proto.StatusRequest{})
|
||||
require.Error(t, err, "the library default must reject this response")
|
||||
assert.Equal(t, codes.ResourceExhausted, status.Code(err), "gRPC rejects an oversized message")
|
||||
}
|
||||
@@ -36,7 +36,10 @@ const (
|
||||
// address. The npipe scheme needs a context dialer because gRPC has no
|
||||
// named-pipe resolver; unix and tcp are handled by gRPC itself.
|
||||
func DialTarget(addr string) (string, []grpc.DialOption) {
|
||||
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxRecvMsgSize())),
|
||||
}
|
||||
|
||||
if name, ok := strings.CutPrefix(addr, pipeScheme); ok {
|
||||
paths := PipePaths(name)
|
||||
|
||||
Reference in New Issue
Block a user