diff --git a/client/vnc/server/capture_darwin.go b/client/vnc/server/capture_darwin.go index 2144fd96c..8f345ddc1 100644 --- a/client/vnc/server/capture_darwin.go +++ b/client/vnc/server/capture_darwin.go @@ -38,9 +38,18 @@ var ( cfRelease func(uintptr) cgPreflightScreenCaptureAccess func() bool cgRequestScreenCaptureAccess func() bool + cgEventCreate func(uintptr) uintptr + cgEventGetLocation func(uintptr) cgPoint darwinCaptureReady bool ) +// cgPoint mirrors CoreGraphics CGPoint: two doubles, 16 bytes, returned +// in registers on Darwin amd64/arm64. Used to receive cursor coordinates +// from CGEventGetLocation via purego. +type cgPoint struct { + X, Y float64 +} + func initDarwinCapture() { darwinCaptureOnce.Do(func() { cg, err := purego.Dlopen("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics", purego.RTLD_NOW|purego.RTLD_GLOBAL) @@ -76,6 +85,15 @@ func initDarwinCapture() { if sym, err := purego.Dlsym(cg, "CGRequestScreenCaptureAccess"); err == nil { purego.RegisterFunc(&cgRequestScreenCaptureAccess, sym) } + // CGEventCreate / CGEventGetLocation feed the cursor position used + // by remote-cursor compositing. Optional; absence reports as a + // position-source error and disables that feature on this host. + if sym, err := purego.Dlsym(cg, "CGEventCreate"); err == nil { + purego.RegisterFunc(&cgEventCreate, sym) + } + if sym, err := purego.Dlsym(cg, "CGEventGetLocation"); err == nil { + purego.RegisterFunc(&cgEventGetLocation, sym) + } darwinCaptureReady = true }) diff --git a/client/vnc/server/capture_x11.go b/client/vnc/server/capture_x11.go index 804c1f02c..93ee84ae1 100644 --- a/client/vnc/server/capture_x11.go +++ b/client/vnc/server/capture_x11.go @@ -425,6 +425,16 @@ func (p *X11Poller) Cursor() (*image.RGBA, int, int, uint64, error) { return p.capturer.Cursor() } +// CursorPos satisfies cursorPositionSource by forwarding to the X11Capturer. +func (p *X11Poller) CursorPos() (int, int, error) { + p.mu.Lock() + defer p.mu.Unlock() + if err := p.ensureCapturerLocked(); err != nil { + return 0, 0, err + } + return p.capturer.CursorPos() +} + // Capture returns a fresh frame, serving from the short-lived cache if a // previous caller captured within freshWindow. func (p *X11Poller) Capture() (*image.RGBA, error) { diff --git a/client/vnc/server/cursor_darwin.go b/client/vnc/server/cursor_darwin.go index 29b856630..324c38b79 100644 --- a/client/vnc/server/cursor_darwin.go +++ b/client/vnc/server/cursor_darwin.go @@ -156,6 +156,21 @@ func (c *CGCapturer) Cursor() (*image.RGBA, int, int, uint64, error) { return c.cursor.Cursor() } +// CursorPos returns the current global mouse location via CGEventCreate / +// CGEventGetLocation. Coordinates are screen pixels in the main display. +func (c *CGCapturer) CursorPos() (int, int, error) { + if cgEventCreate == nil || cgEventGetLocation == nil { + return 0, 0, fmt.Errorf("CGEvent location APIs unavailable") + } + ev := cgEventCreate(0) + if ev == 0 { + return 0, 0, fmt.Errorf("CGEventCreate returned nil") + } + defer cfRelease(ev) + pt := cgEventGetLocation(ev) + return int(pt.X), int(pt.Y), nil +} + // Cursor on MacPoller forwards to the lazy CGCapturer. ensureCapturerLocked // returns an error when Screen Recording permission has not been granted; // in that case there is no usable cursor source either. @@ -167,3 +182,13 @@ func (p *MacPoller) Cursor() (*image.RGBA, int, int, uint64, error) { } return p.capturer.Cursor() } + +// CursorPos forwards to the lazy CGCapturer. +func (p *MacPoller) CursorPos() (int, int, error) { + p.mu.Lock() + defer p.mu.Unlock() + if err := p.ensureCapturerLocked(); err != nil { + return 0, 0, err + } + return p.capturer.CursorPos() +} diff --git a/client/vnc/server/cursor_windows.go b/client/vnc/server/cursor_windows.go index 6b76638b4..85e34d833 100644 --- a/client/vnc/server/cursor_windows.go +++ b/client/vnc/server/cursor_windows.go @@ -86,6 +86,9 @@ type cursorSnapshot struct { img *image.RGBA hotX int hotY int + posX int + posY int + hasPos bool serial uint64 err error } @@ -118,14 +121,26 @@ func (s *cursorSampler) sample() (*cursorSnapshot, error) { // treating this as a hard failure that would latch us off for // the session. if s.lastHandle == hiddenHandle { + s.snapshot.posX = int(ci.PtPos.X) + s.snapshot.posY = int(ci.PtPos.Y) + s.snapshot.hasPos = true return s.snapshot, nil } s.lastHandle = hiddenHandle s.serial++ - s.snapshot = &cursorSnapshot{img: transparentCursorImage(), serial: s.serial} + s.snapshot = &cursorSnapshot{ + img: transparentCursorImage(), + posX: int(ci.PtPos.X), + posY: int(ci.PtPos.Y), + hasPos: true, + serial: s.serial, + } return s.snapshot, nil } if ci.Cursor == s.lastHandle && s.snapshot != nil { + s.snapshot.posX = int(ci.PtPos.X) + s.snapshot.posY = int(ci.PtPos.Y) + s.snapshot.hasPos = true return s.snapshot, nil } img, hotX, hotY, err := decodeCursor(ci.Cursor) @@ -134,7 +149,15 @@ func (s *cursorSampler) sample() (*cursorSnapshot, error) { } s.lastHandle = ci.Cursor s.serial++ - s.snapshot = &cursorSnapshot{img: img, hotX: hotX, hotY: hotY, serial: s.serial} + s.snapshot = &cursorSnapshot{ + img: img, + hotX: hotX, + hotY: hotY, + posX: int(ci.PtPos.X), + posY: int(ci.PtPos.Y), + hasPos: true, + serial: s.serial, + } return s.snapshot, nil } @@ -348,3 +371,20 @@ func (c *DesktopCapturer) Cursor() (*image.RGBA, int, int, uint64, error) { } return snap.img, snap.hotX, snap.hotY, snap.serial, nil } + +// CursorPos returns the cursor screen position observed by the worker on +// its last sample. Errors out if the worker hasn't yet captured a frame +// or the most recent sample failed. +func (c *DesktopCapturer) CursorPos() (int, int, error) { + snap := c.cursorState.load() + if snap == nil { + return 0, 0, fmt.Errorf("cursor position not sampled yet") + } + if snap.err != nil { + return 0, 0, snap.err + } + if !snap.hasPos { + return 0, 0, fmt.Errorf("cursor position unavailable") + } + return snap.posX, snap.posY, nil +} diff --git a/client/vnc/server/cursor_x11.go b/client/vnc/server/cursor_x11.go index 5dd06f5c5..2ac8d2798 100644 --- a/client/vnc/server/cursor_x11.go +++ b/client/vnc/server/cursor_x11.go @@ -21,6 +21,11 @@ type xfixesCursor struct { // calls return quickly without another X round-trip. Some virtual // displays advertise XFixes but reject GetCursorImage (Xvfb). runtimeErr error + // lastPosX/lastPosY hold the cursor screen position observed on the + // most recent successful GetCursorImage. cursorPositionSource readers + // share this value so we do not pay a second X round-trip per frame. + lastPosX, lastPosY int + hasPos bool } // newXFixesCursor initialises the XFixes extension on conn. Returns an @@ -49,6 +54,7 @@ func (c *xfixesCursor) Cursor() (*image.RGBA, int, int, uint64, error) { c.runtimeErr = fmt.Errorf("xfixes GetCursorImage: %w", err) return nil, 0, 0, 0, c.runtimeErr } + c.lastPosX, c.lastPosY, c.hasPos = int(reply.X), int(reply.Y), true w, h := int(reply.Width), int(reply.Height) if w <= 0 || h <= 0 { return nil, 0, 0, 0, fmt.Errorf("cursor has zero extent") @@ -85,3 +91,21 @@ func (x *X11Capturer) Cursor() (*image.RGBA, int, int, uint64, error) { } return cur.Cursor() } + +// CursorPos on X11Capturer returns the screen position from the most +// recent successful Cursor() call. Sessions call Cursor() once per encode +// cycle, so this stays current without a second X round-trip. +func (x *X11Capturer) CursorPos() (int, int, error) { + x.mu.Lock() + cur := x.cursor + x.mu.Unlock() + if cur == nil { + return 0, 0, fmt.Errorf("cursor source not initialised") + } + cur.mu.Lock() + defer cur.mu.Unlock() + if !cur.hasPos { + return 0, 0, fmt.Errorf("cursor position not sampled yet") + } + return cur.lastPosX, cur.lastPosY, nil +} diff --git a/client/vnc/server/rfb.go b/client/vnc/server/rfb.go index e8e6e11c2..291d3529a 100644 --- a/client/vnc/server/rfb.go +++ b/client/vnc/server/rfb.go @@ -49,6 +49,14 @@ const ( // The opcode is in the vendor-specific range (>=128). clientNetbirdTypeText = 250 + // clientNetbirdShowRemoteCursor toggles "show remote cursor" mode. + // When enabled the encoder composites the server cursor sprite into + // the captured framebuffer and suppresses the Cursor pseudo-encoding + // so the dashboard sees a single pointer at the remote position. + // Wire format: 1-byte msgType + 1-byte enable flag + 6 padding bytes + // reserved for future arguments (so the message is fixed-size). + clientNetbirdShowRemoteCursor = 251 + // Server message types. serverFramebufferUpdate = 0 serverCutText = 3 diff --git a/client/vnc/server/server.go b/client/vnc/server/server.go index 1e6a21bac..076825074 100644 --- a/client/vnc/server/server.go +++ b/client/vnc/server/server.go @@ -79,6 +79,14 @@ type cursorSource interface { Cursor() (img *image.RGBA, hotX, hotY int, serial uint64, err error) } +// cursorPositionSource adds the cursor's current screen-space position to +// cursorSource so the encoder can alpha-blend the sprite into the captured +// framebuffer for "show remote cursor" mode. Implementations should be +// cheap; most platforms already get the position alongside the sprite. +type cursorPositionSource interface { + CursorPos() (x, y int, err error) +} + // errFrameUnchanged is returned by capturers that hash the raw source // bytes (currently macOS) when the new frame is byte-identical to the // last one, so the encoder can short-circuit to an empty update. diff --git a/client/vnc/server/session.go b/client/vnc/server/session.go index ae138f135..18eb4461d 100644 --- a/client/vnc/server/session.go +++ b/client/vnc/server/session.go @@ -92,6 +92,15 @@ type session struct { // what the client advertises. Set for virtual sessions where no // usable cursor source exists. Constant for the session lifetime. disableCursor bool + // showRemoteCursor switches the encoder to compositing the server + // cursor sprite into the captured framebuffer at the remote position + // instead of emitting the Cursor pseudo-encoding. Toggled by the + // dashboard via clientNetbirdShowRemoteCursor. + showRemoteCursor bool + // cursorWarnOnce throttles the diagnostic emitted when remote-cursor + // compositing falls back to a no-op (capturer cannot supply a sprite + // or position). One line per session is enough to point at the cause. + cursorWarnOnce sync.Once // clientJPEGQuality and clientZlibLevel hold the 0..9 levels the client // advertised via the QualityLevel / CompressLevel pseudo-encodings, or // -1 when the client has not expressed a preference. Applied to the @@ -274,6 +283,8 @@ func (s *session) messageLoop() error { err = s.handleQEMUMessage() case clientNetbirdTypeText: err = s.handleTypeText() + case clientNetbirdShowRemoteCursor: + err = s.handleShowRemoteCursor() default: return fmt.Errorf("unknown client message type: %d", msgType[0]) } diff --git a/client/vnc/server/session_cursor.go b/client/vnc/server/session_cursor.go index bf3765adc..2ab62ea27 100644 --- a/client/vnc/server/session_cursor.go +++ b/client/vnc/server/session_cursor.go @@ -15,9 +15,10 @@ func (s *session) pendingCursorRect() []byte { s.encMu.RLock() supported := s.clientSupportsCursor failed := s.cursorSourceFailed + composite := s.showRemoteCursor lastSerial := s.lastCursorSerial s.encMu.RUnlock() - if !supported || failed { + if !supported || failed || composite { return nil } src, ok := s.capturer.(cursorSource) diff --git a/client/vnc/server/session_encode.go b/client/vnc/server/session_encode.go index 2197bf8bc..8068bb1d7 100644 --- a/client/vnc/server/session_encode.go +++ b/client/vnc/server/session_encode.go @@ -67,6 +67,8 @@ func (s *session) processFBRequest(req fbRequest) error { } s.captureRecovered() + s.maybeCompositeCursor(img) + if req.incremental && s.prevFrame != nil { return s.processIncremental(img) } diff --git a/client/vnc/server/session_remote_cursor.go b/client/vnc/server/session_remote_cursor.go new file mode 100644 index 000000000..2ed77320c --- /dev/null +++ b/client/vnc/server/session_remote_cursor.go @@ -0,0 +1,119 @@ +//go:build !js && !ios && !android + +package server + +import ( + "fmt" + "image" + "io" +) + +// handleShowRemoteCursor handles the NetBird-specific RFB message used by +// the dashboard to toggle "show remote cursor" mode. Wire format: 1-byte +// enable flag (0/1) plus 6 padding bytes reserved for future arguments. +func (s *session) handleShowRemoteCursor() error { + var data [7]byte + if _, err := io.ReadFull(s.conn, data[:]); err != nil { + return fmt.Errorf("read showRemoteCursor: %w", err) + } + enable := data[0] != 0 + s.encMu.Lock() + s.showRemoteCursor = enable + s.encMu.Unlock() + s.log.Debugf("show remote cursor: %v", enable) + return nil +} + +// maybeCompositeCursor blends the current server cursor into img when the +// dashboard has enabled "show remote cursor" mode. Returns silently in +// every error path: a failed compositing must not stop the regular encode +// flow. +func (s *session) maybeCompositeCursor(img *image.RGBA) { + s.encMu.RLock() + enabled := s.showRemoteCursor + s.encMu.RUnlock() + if !enabled || img == nil { + return + } + src, ok := s.capturer.(cursorSource) + if !ok { + return + } + pos, ok := s.capturer.(cursorPositionSource) + if !ok { + return + } + cursorImg, hotX, hotY, _, err := src.Cursor() + if err != nil || cursorImg == nil { + s.cursorWarnOnce.Do(func() { + s.log.Warnf("remote cursor unavailable: %v", err) + }) + return + } + posX, posY, err := pos.CursorPos() + if err != nil { + s.cursorWarnOnce.Do(func() { + s.log.Warnf("remote cursor position unavailable: %v", err) + }) + return + } + compositeCursor(img, cursorImg, posX-hotX, posY-hotY) +} + +// compositeCursor alpha-blends sprite onto frame at (dstX, dstY) using +// straight (non-premultiplied) alpha. Out-of-bounds destinations are +// clipped. Frames captured by our X11/Windows/macOS paths all advertise +// RGBA with a 255-only alpha channel, so the result keeps the framebuffer +// invariant ("opaque pixels everywhere") that the encoder depends on. +func compositeCursor(frame, sprite *image.RGBA, dstX, dstY int) { + fw, fh := frame.Rect.Dx(), frame.Rect.Dy() + sw, sh := sprite.Rect.Dx(), sprite.Rect.Dy() + if sw == 0 || sh == 0 { + return + } + + x0, y0 := dstX, dstY + x1, y1 := dstX+sw, dstY+sh + if x0 < 0 { + x0 = 0 + } + if y0 < 0 { + y0 = 0 + } + if x1 > fw { + x1 = fw + } + if y1 > fh { + y1 = fh + } + if x0 >= x1 || y0 >= y1 { + return + } + + fStride := frame.Stride + sStride := sprite.Stride + for y := y0; y < y1; y++ { + sy := y - dstY + fbRow := y * fStride + sRow := sy * sStride + for x := x0; x < x1; x++ { + sx := x - dstX + fbOff := fbRow + x*4 + sOff := sRow + sx*4 + a := uint32(sprite.Pix[sOff+3]) + if a == 0 { + continue + } + if a == 255 { + frame.Pix[fbOff+0] = sprite.Pix[sOff+0] + frame.Pix[fbOff+1] = sprite.Pix[sOff+1] + frame.Pix[fbOff+2] = sprite.Pix[sOff+2] + continue + } + inv := 255 - a + frame.Pix[fbOff+0] = byte((uint32(sprite.Pix[sOff+0])*a + uint32(frame.Pix[fbOff+0])*inv) / 255) + frame.Pix[fbOff+1] = byte((uint32(sprite.Pix[sOff+1])*a + uint32(frame.Pix[fbOff+1])*inv) / 255) + frame.Pix[fbOff+2] = byte((uint32(sprite.Pix[sOff+2])*a + uint32(frame.Pix[fbOff+2])*inv) / 255) + } + } +}