Composite remote cursor into the framebuffer when the dashboard toggles it on

client/vnc/server/capture_darwin.go

@@ -38,9 +38,18 @@ var (
 	cfRelease                      func(uintptr)
 	cgPreflightScreenCaptureAccess func() bool
 	cgRequestScreenCaptureAccess   func() bool
+	cgEventCreate                  func(uintptr) uintptr
+	cgEventGetLocation             func(uintptr) cgPoint
 	darwinCaptureReady             bool
 )
 
+// cgPoint mirrors CoreGraphics CGPoint: two doubles, 16 bytes, returned
+// in registers on Darwin amd64/arm64. Used to receive cursor coordinates
+// from CGEventGetLocation via purego.
+type cgPoint struct {
+	X, Y float64
+}
+
 func initDarwinCapture() {
 	darwinCaptureOnce.Do(func() {
 		cg, err := purego.Dlopen("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics", purego.RTLD_NOW|purego.RTLD_GLOBAL)
@@ -76,6 +85,15 @@ func initDarwinCapture() {
 		if sym, err := purego.Dlsym(cg, "CGRequestScreenCaptureAccess"); err == nil {
 			purego.RegisterFunc(&cgRequestScreenCaptureAccess, sym)
 		}
+		// CGEventCreate / CGEventGetLocation feed the cursor position used
+		// by remote-cursor compositing. Optional; absence reports as a
+		// position-source error and disables that feature on this host.
+		if sym, err := purego.Dlsym(cg, "CGEventCreate"); err == nil {
+			purego.RegisterFunc(&cgEventCreate, sym)
+		}
+		if sym, err := purego.Dlsym(cg, "CGEventGetLocation"); err == nil {
+			purego.RegisterFunc(&cgEventGetLocation, sym)
+		}
 
 		darwinCaptureReady = true
 	})

client/vnc/server/capture_x11.go

@@ -425,6 +425,16 @@ func (p *X11Poller) Cursor() (*image.RGBA, int, int, uint64, error) {
 	return p.capturer.Cursor()
 }
 
+// CursorPos satisfies cursorPositionSource by forwarding to the X11Capturer.
+func (p *X11Poller) CursorPos() (int, int, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if err := p.ensureCapturerLocked(); err != nil {
+		return 0, 0, err
+	}
+	return p.capturer.CursorPos()
+}
+
 // Capture returns a fresh frame, serving from the short-lived cache if a
 // previous caller captured within freshWindow.
 func (p *X11Poller) Capture() (*image.RGBA, error) {

client/vnc/server/cursor_darwin.go

@@ -156,6 +156,21 @@ func (c *CGCapturer) Cursor() (*image.RGBA, int, int, uint64, error) {
 	return c.cursor.Cursor()
 }
 
+// CursorPos returns the current global mouse location via CGEventCreate /
+// CGEventGetLocation. Coordinates are screen pixels in the main display.
+func (c *CGCapturer) CursorPos() (int, int, error) {
+	if cgEventCreate == nil || cgEventGetLocation == nil {
+		return 0, 0, fmt.Errorf("CGEvent location APIs unavailable")
+	}
+	ev := cgEventCreate(0)
+	if ev == 0 {
+		return 0, 0, fmt.Errorf("CGEventCreate returned nil")
+	}
+	defer cfRelease(ev)
+	pt := cgEventGetLocation(ev)
+	return int(pt.X), int(pt.Y), nil
+}
+
 // Cursor on MacPoller forwards to the lazy CGCapturer. ensureCapturerLocked
 // returns an error when Screen Recording permission has not been granted;
 // in that case there is no usable cursor source either.
@@ -167,3 +182,13 @@ func (p *MacPoller) Cursor() (*image.RGBA, int, int, uint64, error) {
 	}
 	return p.capturer.Cursor()
 }
+
+// CursorPos forwards to the lazy CGCapturer.
+func (p *MacPoller) CursorPos() (int, int, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if err := p.ensureCapturerLocked(); err != nil {
+		return 0, 0, err
+	}
+	return p.capturer.CursorPos()
+}

client/vnc/server/cursor_windows.go

@@ -86,6 +86,9 @@ type cursorSnapshot struct {
 	img    *image.RGBA
 	hotX   int
 	hotY   int
+	posX   int
+	posY   int
+	hasPos bool
 	serial uint64
 	err    error
 }
@@ -118,14 +121,26 @@ func (s *cursorSampler) sample() (*cursorSnapshot, error) {
 		// treating this as a hard failure that would latch us off for
 		// the session.
 		if s.lastHandle == hiddenHandle {
+			s.snapshot.posX = int(ci.PtPos.X)
+			s.snapshot.posY = int(ci.PtPos.Y)
+			s.snapshot.hasPos = true
 			return s.snapshot, nil
 		}
 		s.lastHandle = hiddenHandle
 		s.serial++
-		s.snapshot = &cursorSnapshot{img: transparentCursorImage(), serial: s.serial}
+		s.snapshot = &cursorSnapshot{
+			img:    transparentCursorImage(),
+			posX:   int(ci.PtPos.X),
+			posY:   int(ci.PtPos.Y),
+			hasPos: true,
+			serial: s.serial,
+		}
 		return s.snapshot, nil
 	}
 	if ci.Cursor == s.lastHandle && s.snapshot != nil {
+		s.snapshot.posX = int(ci.PtPos.X)
+		s.snapshot.posY = int(ci.PtPos.Y)
+		s.snapshot.hasPos = true
 		return s.snapshot, nil
 	}
 	img, hotX, hotY, err := decodeCursor(ci.Cursor)
@@ -134,7 +149,15 @@ func (s *cursorSampler) sample() (*cursorSnapshot, error) {
 	}
 	s.lastHandle = ci.Cursor
 	s.serial++
-	s.snapshot = &cursorSnapshot{img: img, hotX: hotX, hotY: hotY, serial: s.serial}
+	s.snapshot = &cursorSnapshot{
+		img:    img,
+		hotX:   hotX,
+		hotY:   hotY,
+		posX:   int(ci.PtPos.X),
+		posY:   int(ci.PtPos.Y),
+		hasPos: true,
+		serial: s.serial,
+	}
 	return s.snapshot, nil
 }
 
@@ -348,3 +371,20 @@ func (c *DesktopCapturer) Cursor() (*image.RGBA, int, int, uint64, error) {
 	}
 	return snap.img, snap.hotX, snap.hotY, snap.serial, nil
 }
+
+// CursorPos returns the cursor screen position observed by the worker on
+// its last sample. Errors out if the worker hasn't yet captured a frame
+// or the most recent sample failed.
+func (c *DesktopCapturer) CursorPos() (int, int, error) {
+	snap := c.cursorState.load()
+	if snap == nil {
+		return 0, 0, fmt.Errorf("cursor position not sampled yet")
+	}
+	if snap.err != nil {
+		return 0, 0, snap.err
+	}
+	if !snap.hasPos {
+		return 0, 0, fmt.Errorf("cursor position unavailable")
+	}
+	return snap.posX, snap.posY, nil
+}

client/vnc/server/cursor_x11.go

@@ -21,6 +21,11 @@ type xfixesCursor struct {
 	// calls return quickly without another X round-trip. Some virtual
 	// displays advertise XFixes but reject GetCursorImage (Xvfb).
 	runtimeErr error
+	// lastPosX/lastPosY hold the cursor screen position observed on the
+	// most recent successful GetCursorImage. cursorPositionSource readers
+	// share this value so we do not pay a second X round-trip per frame.
+	lastPosX, lastPosY int
+	hasPos             bool
 }
 
 // newXFixesCursor initialises the XFixes extension on conn. Returns an
@@ -49,6 +54,7 @@ func (c *xfixesCursor) Cursor() (*image.RGBA, int, int, uint64, error) {
 		c.runtimeErr = fmt.Errorf("xfixes GetCursorImage: %w", err)
 		return nil, 0, 0, 0, c.runtimeErr
 	}
+	c.lastPosX, c.lastPosY, c.hasPos = int(reply.X), int(reply.Y), true
 	w, h := int(reply.Width), int(reply.Height)
 	if w <= 0 || h <= 0 {
 		return nil, 0, 0, 0, fmt.Errorf("cursor has zero extent")
@@ -85,3 +91,21 @@ func (x *X11Capturer) Cursor() (*image.RGBA, int, int, uint64, error) {
 	}
 	return cur.Cursor()
 }
+
+// CursorPos on X11Capturer returns the screen position from the most
+// recent successful Cursor() call. Sessions call Cursor() once per encode
+// cycle, so this stays current without a second X round-trip.
+func (x *X11Capturer) CursorPos() (int, int, error) {
+	x.mu.Lock()
+	cur := x.cursor
+	x.mu.Unlock()
+	if cur == nil {
+		return 0, 0, fmt.Errorf("cursor source not initialised")
+	}
+	cur.mu.Lock()
+	defer cur.mu.Unlock()
+	if !cur.hasPos {
+		return 0, 0, fmt.Errorf("cursor position not sampled yet")
+	}
+	return cur.lastPosX, cur.lastPosY, nil
+}

client/vnc/server/rfb.go

@@ -49,6 +49,14 @@ const (
 	// The opcode is in the vendor-specific range (>=128).
 	clientNetbirdTypeText = 250
 
+	// clientNetbirdShowRemoteCursor toggles "show remote cursor" mode.
+	// When enabled the encoder composites the server cursor sprite into
+	// the captured framebuffer and suppresses the Cursor pseudo-encoding
+	// so the dashboard sees a single pointer at the remote position.
+	// Wire format: 1-byte msgType + 1-byte enable flag + 6 padding bytes
+	// reserved for future arguments (so the message is fixed-size).
+	clientNetbirdShowRemoteCursor = 251
+
 	// Server message types.
 	serverFramebufferUpdate = 0
 	serverCutText           = 3

client/vnc/server/server.go

@@ -79,6 +79,14 @@ type cursorSource interface {
 	Cursor() (img *image.RGBA, hotX, hotY int, serial uint64, err error)
 }
 
+// cursorPositionSource adds the cursor's current screen-space position to
+// cursorSource so the encoder can alpha-blend the sprite into the captured
+// framebuffer for "show remote cursor" mode. Implementations should be
+// cheap; most platforms already get the position alongside the sprite.
+type cursorPositionSource interface {
+	CursorPos() (x, y int, err error)
+}
+
 // errFrameUnchanged is returned by capturers that hash the raw source
 // bytes (currently macOS) when the new frame is byte-identical to the
 // last one, so the encoder can short-circuit to an empty update.

client/vnc/server/session.go

@@ -92,6 +92,15 @@ type session struct {
 	// what the client advertises. Set for virtual sessions where no
 	// usable cursor source exists. Constant for the session lifetime.
 	disableCursor bool
+	// showRemoteCursor switches the encoder to compositing the server
+	// cursor sprite into the captured framebuffer at the remote position
+	// instead of emitting the Cursor pseudo-encoding. Toggled by the
+	// dashboard via clientNetbirdShowRemoteCursor.
+	showRemoteCursor bool
+	// cursorWarnOnce throttles the diagnostic emitted when remote-cursor
+	// compositing falls back to a no-op (capturer cannot supply a sprite
+	// or position). One line per session is enough to point at the cause.
+	cursorWarnOnce sync.Once
 	// clientJPEGQuality and clientZlibLevel hold the 0..9 levels the client
 	// advertised via the QualityLevel / CompressLevel pseudo-encodings, or
 	// -1 when the client has not expressed a preference. Applied to the
@@ -274,6 +283,8 @@ func (s *session) messageLoop() error {
 			err = s.handleQEMUMessage()
 		case clientNetbirdTypeText:
 			err = s.handleTypeText()
+		case clientNetbirdShowRemoteCursor:
+			err = s.handleShowRemoteCursor()
 		default:
 			return fmt.Errorf("unknown client message type: %d", msgType[0])
 		}

client/vnc/server/session_cursor.go

@@ -15,9 +15,10 @@ func (s *session) pendingCursorRect() []byte {
 	s.encMu.RLock()
 	supported := s.clientSupportsCursor
 	failed := s.cursorSourceFailed
+	composite := s.showRemoteCursor
 	lastSerial := s.lastCursorSerial
 	s.encMu.RUnlock()
-	if !supported || failed {
+	if !supported || failed || composite {
 		return nil
 	}
 	src, ok := s.capturer.(cursorSource)

client/vnc/server/session_encode.go

@@ -67,6 +67,8 @@ func (s *session) processFBRequest(req fbRequest) error {
 	}
 	s.captureRecovered()
 
+	s.maybeCompositeCursor(img)
+
 	if req.incremental && s.prevFrame != nil {
 		return s.processIncremental(img)
 	}

client/vnc/server/session_remote_cursor.go

@@ -0,0 +1,119 @@
+//go:build !js && !ios && !android
+
+package server
+
+import (
+	"fmt"
+	"image"
+	"io"
+)
+
+// handleShowRemoteCursor handles the NetBird-specific RFB message used by
+// the dashboard to toggle "show remote cursor" mode. Wire format: 1-byte
+// enable flag (0/1) plus 6 padding bytes reserved for future arguments.
+func (s *session) handleShowRemoteCursor() error {
+	var data [7]byte
+	if _, err := io.ReadFull(s.conn, data[:]); err != nil {
+		return fmt.Errorf("read showRemoteCursor: %w", err)
+	}
+	enable := data[0] != 0
+	s.encMu.Lock()
+	s.showRemoteCursor = enable
+	s.encMu.Unlock()
+	s.log.Debugf("show remote cursor: %v", enable)
+	return nil
+}
+
+// maybeCompositeCursor blends the current server cursor into img when the
+// dashboard has enabled "show remote cursor" mode. Returns silently in
+// every error path: a failed compositing must not stop the regular encode
+// flow.
+func (s *session) maybeCompositeCursor(img *image.RGBA) {
+	s.encMu.RLock()
+	enabled := s.showRemoteCursor
+	s.encMu.RUnlock()
+	if !enabled || img == nil {
+		return
+	}
+	src, ok := s.capturer.(cursorSource)
+	if !ok {
+		return
+	}
+	pos, ok := s.capturer.(cursorPositionSource)
+	if !ok {
+		return
+	}
+	cursorImg, hotX, hotY, _, err := src.Cursor()
+	if err != nil || cursorImg == nil {
+		s.cursorWarnOnce.Do(func() {
+			s.log.Warnf("remote cursor unavailable: %v", err)
+		})
+		return
+	}
+	posX, posY, err := pos.CursorPos()
+	if err != nil {
+		s.cursorWarnOnce.Do(func() {
+			s.log.Warnf("remote cursor position unavailable: %v", err)
+		})
+		return
+	}
+	compositeCursor(img, cursorImg, posX-hotX, posY-hotY)
+}
+
+// compositeCursor alpha-blends sprite onto frame at (dstX, dstY) using
+// straight (non-premultiplied) alpha. Out-of-bounds destinations are
+// clipped. Frames captured by our X11/Windows/macOS paths all advertise
+// RGBA with a 255-only alpha channel, so the result keeps the framebuffer
+// invariant ("opaque pixels everywhere") that the encoder depends on.
+func compositeCursor(frame, sprite *image.RGBA, dstX, dstY int) {
+	fw, fh := frame.Rect.Dx(), frame.Rect.Dy()
+	sw, sh := sprite.Rect.Dx(), sprite.Rect.Dy()
+	if sw == 0 || sh == 0 {
+		return
+	}
+
+	x0, y0 := dstX, dstY
+	x1, y1 := dstX+sw, dstY+sh
+	if x0 < 0 {
+		x0 = 0
+	}
+	if y0 < 0 {
+		y0 = 0
+	}
+	if x1 > fw {
+		x1 = fw
+	}
+	if y1 > fh {
+		y1 = fh
+	}
+	if x0 >= x1 || y0 >= y1 {
+		return
+	}
+
+	fStride := frame.Stride
+	sStride := sprite.Stride
+	for y := y0; y < y1; y++ {
+		sy := y - dstY
+		fbRow := y * fStride
+		sRow := sy * sStride
+		for x := x0; x < x1; x++ {
+			sx := x - dstX
+			fbOff := fbRow + x*4
+			sOff := sRow + sx*4
+			a := uint32(sprite.Pix[sOff+3])
+			if a == 0 {
+				continue
+			}
+			if a == 255 {
+				frame.Pix[fbOff+0] = sprite.Pix[sOff+0]
+				frame.Pix[fbOff+1] = sprite.Pix[sOff+1]
+				frame.Pix[fbOff+2] = sprite.Pix[sOff+2]
+				continue
+			}
+			inv := 255 - a
+			frame.Pix[fbOff+0] = byte((uint32(sprite.Pix[sOff+0])*a + uint32(frame.Pix[fbOff+0])*inv) / 255)
+			frame.Pix[fbOff+1] = byte((uint32(sprite.Pix[sOff+1])*a + uint32(frame.Pix[fbOff+1])*inv) / 255)
+			frame.Pix[fbOff+2] = byte((uint32(sprite.Pix[sOff+2])*a + uint32(frame.Pix[fbOff+2])*inv) / 255)
+		}
+	}
+}