diff --git a/client/vnc/server/capture_darwin.go b/client/vnc/server/capture_darwin.go index 2efe02f08..2144fd96c 100644 --- a/client/vnc/server/capture_darwin.go +++ b/client/vnc/server/capture_darwin.go @@ -91,6 +91,11 @@ type CGCapturer struct { hashSeed maphash.Seed lastHash uint64 hasHash bool + // cursor lazily binds the private CGSCreateCurrentCursorImage symbol + // so we can emit the Cursor pseudo-encoding without a per-frame cost + // on builds that never query it. + cursorOnce sync.Once + cursor *cgCursor } // PrimeScreenCapturePermission triggers the macOS Screen Recording diff --git a/client/vnc/server/capture_windows.go b/client/vnc/server/capture_windows.go index 60c3afb66..8afbad110 100644 --- a/client/vnc/server/capture_windows.go +++ b/client/vnc/server/capture_windows.go @@ -266,6 +266,11 @@ type DesktopCapturer struct { wake chan struct{} // done is closed when Close is called, terminating the worker. done chan struct{} + + // cursorState holds the latest cursor sprite sampled by the worker. + // The worker calls GetCursorInfo every capture and decodes a new + // sprite only when the HCURSOR changes. + cursorState cursorState } // captureReq is a single capture request awaiting a reply. Reply channel is @@ -439,6 +444,7 @@ type captureWorker struct { desktopFails int lastDesktop string nextInitRetry time.Time + cursor cursorSampler } // handleNextRequest waits for either shutdown or a capture request and runs @@ -468,6 +474,11 @@ func (w *captureWorker) serveRequest(req captureReq) { req.reply <- captureReply{err: err} return } + if snap, err := w.cursor.sample(); err != nil { + w.c.cursorState.store(&cursorSnapshot{err: err}) + } else { + w.c.cursorState.store(snap) + } req.reply <- captureReply{img: img} } diff --git a/client/vnc/server/capture_x11.go b/client/vnc/server/capture_x11.go index fd3eb5859..804c1f02c 100644 --- a/client/vnc/server/capture_x11.go +++ b/client/vnc/server/capture_x11.go @@ -34,6 +34,11 @@ type X11Capturer struct { // happens on first use and on geometry change. bufs [2]*image.RGBA cur int + // cursor is the XFixes binding used to report the current sprite. + // Allocated lazily on the first Cursor call. cursorInitErr latches + // a permanent init failure so we stop retrying every frame. + cursor *xfixesCursor + cursorInitErr error } // detectX11Display finds the active X11 display and sets DISPLAY/XAUTHORITY @@ -408,6 +413,18 @@ func (p *X11Poller) Height() int { return p.h } +// Cursor satisfies cursorSource by forwarding to the lazily-initialised +// X11Capturer. Asking for the cursor on an idle poller triggers the same +// lazy X11 connection setup as a capture would. +func (p *X11Poller) Cursor() (*image.RGBA, int, int, uint64, error) { + p.mu.Lock() + defer p.mu.Unlock() + if err := p.ensureCapturerLocked(); err != nil { + return nil, 0, 0, 0, err + } + return p.capturer.Cursor() +} + // Capture returns a fresh frame, serving from the short-lived cache if a // previous caller captured within freshWindow. func (p *X11Poller) Capture() (*image.RGBA, error) { diff --git a/client/vnc/server/cursor_darwin.go b/client/vnc/server/cursor_darwin.go new file mode 100644 index 000000000..29b856630 --- /dev/null +++ b/client/vnc/server/cursor_darwin.go @@ -0,0 +1,169 @@ +//go:build darwin && !ios + +package server + +import ( + "fmt" + "hash/maphash" + "image" + "sync" + "unsafe" + + "github.com/ebitengine/purego" + log "github.com/sirupsen/logrus" +) + +var ( + darwinCursorOnce sync.Once + cgsCreateCursor func() uintptr + darwinCursorErr error +) + +// initDarwinCursor binds a private symbol that returns the current +// system cursor image. The classic CGSCreateCurrentCursorImage moved +// from CoreGraphics to SkyLight around macOS 13 and is gone entirely +// in Sequoia; we probe both frameworks for any of the historical +// names so this keeps working on whichever release the binding still +// exists. Without a hit the remote-cursor compositing path becomes a +// no-op and we log the candidates we tried. +func initDarwinCursor() { + darwinCursorOnce.Do(func() { + libs := []string{ + "/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight", + "/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics", + } + names := []string{ + "CGSCreateCurrentCursorImage", + "CGSCopyCurrentCursorImage", + "CGSCurrentCursorImage", + "CGSHardwareCursorActiveImage", + } + var tried []string + for _, path := range libs { + h, err := purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + tried = append(tried, fmt.Sprintf("dlopen %s: %v", path, err)) + continue + } + for _, name := range names { + sym, err := purego.Dlsym(h, name) + if err != nil { + tried = append(tried, fmt.Sprintf("%s!%s missing", path, name)) + continue + } + purego.RegisterFunc(&cgsCreateCursor, sym) + log.Infof("macOS cursor: bound %s from %s", name, path) + return + } + } + darwinCursorErr = fmt.Errorf("no cursor image symbol available; tried: %v", tried) + }) +} + +// cgCursor holds the cached macOS cursor sprite and bumps a serial when +// the bytes change. Hotspot is left at (0, 0): the public Cocoa hot-spot +// query lives on NSCursor which is process-local and not reachable from +// our purego-based bindings; the visual cost is a small misalignment for +// non-arrow cursors (I-beam, crosshair, etc.). +type cgCursor struct { + mu sync.Mutex + hashSeed maphash.Seed + lastSum uint64 + cached *image.RGBA + serial uint64 +} + +func newCGCursor() *cgCursor { + initDarwinCursor() + return &cgCursor{hashSeed: maphash.MakeSeed()} +} + +// Cursor returns the current cursor sprite as RGBA. Errors that come from +// missing private symbols are sticky; transient empty-image responses are +// reported as such so the encoder skips this cycle. +func (c *cgCursor) Cursor() (*image.RGBA, int, int, uint64, error) { + c.mu.Lock() + defer c.mu.Unlock() + if darwinCursorErr != nil { + return nil, 0, 0, 0, darwinCursorErr + } + if cgsCreateCursor == nil { + return nil, 0, 0, 0, fmt.Errorf("CGSCreateCurrentCursorImage unavailable") + } + cgImage := cgsCreateCursor() + if cgImage == 0 { + return nil, 0, 0, 0, fmt.Errorf("no cursor image available") + } + defer cgImageRelease(cgImage) + + w := int(cgImageGetWidth(cgImage)) + h := int(cgImageGetHeight(cgImage)) + if w <= 0 || h <= 0 { + return nil, 0, 0, 0, fmt.Errorf("cursor has zero extent") + } + bytesPerRow := int(cgImageGetBytesPerRow(cgImage)) + bpp := int(cgImageGetBitsPerPixel(cgImage)) + if bpp != 32 { + return nil, 0, 0, 0, fmt.Errorf("unsupported cursor bpp: %d", bpp) + } + provider := cgImageGetDataProvider(cgImage) + if provider == 0 { + return nil, 0, 0, 0, fmt.Errorf("cursor data provider missing") + } + cfData := cgDataProviderCopyData(provider) + if cfData == 0 { + return nil, 0, 0, 0, fmt.Errorf("cursor data copy failed") + } + defer cfRelease(cfData) + dataLen := int(cfDataGetLength(cfData)) + dataPtr := cfDataGetBytePtr(cfData) + if dataPtr == 0 || dataLen == 0 { + return nil, 0, 0, 0, fmt.Errorf("cursor data empty") + } + src := unsafe.Slice((*byte)(unsafe.Pointer(dataPtr)), dataLen) + + sum := maphash.Bytes(c.hashSeed, src) + if c.cached != nil && sum == c.lastSum { + return c.cached, 0, 0, c.serial, nil + } + + img := image.NewRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + srcOff := y * bytesPerRow + dstOff := y * w * 4 + for x := 0; x < w; x++ { + si := srcOff + x*4 + di := dstOff + x*4 + img.Pix[di+0] = src[si+2] + img.Pix[di+1] = src[si+1] + img.Pix[di+2] = src[si+0] + img.Pix[di+3] = src[si+3] + } + } + + c.lastSum = sum + c.cached = img + c.serial++ + return img, 0, 0, c.serial, nil +} + +// Cursor on CGCapturer satisfies cursorSource. The cgCursor wrapper is +// allocated lazily so a build that never asks for the cursor pays no cost. +func (c *CGCapturer) Cursor() (*image.RGBA, int, int, uint64, error) { + c.cursorOnce.Do(func() { + c.cursor = newCGCursor() + }) + return c.cursor.Cursor() +} + +// Cursor on MacPoller forwards to the lazy CGCapturer. ensureCapturerLocked +// returns an error when Screen Recording permission has not been granted; +// in that case there is no usable cursor source either. +func (p *MacPoller) Cursor() (*image.RGBA, int, int, uint64, error) { + p.mu.Lock() + defer p.mu.Unlock() + if err := p.ensureCapturerLocked(); err != nil { + return nil, 0, 0, 0, err + } + return p.capturer.Cursor() +} diff --git a/client/vnc/server/cursor_windows.go b/client/vnc/server/cursor_windows.go new file mode 100644 index 000000000..6b76638b4 --- /dev/null +++ b/client/vnc/server/cursor_windows.go @@ -0,0 +1,350 @@ +//go:build windows + +package server + +import ( + "fmt" + "image" + "sync" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + procGetCursorInfo = user32.NewProc("GetCursorInfo") + procGetIconInfo = user32.NewProc("GetIconInfo") + procGetObjectW = gdi32.NewProc("GetObjectW") + procGetDIBits = gdi32.NewProc("GetDIBits") +) + +const ( + cursorShowing = 0x00000001 + diRgbColors = 0 + biRgb = 0 + dibSectionBytes = 40 // sizeof(BITMAPINFOHEADER) +) + +// hiddenHandle is a sentinel stored in cursorSampler.lastHandle while +// Windows reports the cursor as hidden. It is not a valid HCURSOR value; +// real handles never collide with this constant. +const hiddenHandle = windows.Handle(^uintptr(0)) + +// transparentCursorImage returns a 1x1 fully transparent sprite. The +// client renders this as "no cursor"; emitting it explicitly lets us +// recover when an app un-hides the cursor a moment later. +func transparentCursorImage() *image.RGBA { + return image.NewRGBA(image.Rect(0, 0, 1, 1)) +} + +type winPoint struct { + X, Y int32 +} + +type winCursorInfo struct { + Size uint32 + Flags uint32 + Cursor windows.Handle + PtPos winPoint +} + +type winIconInfo struct { + FIcon int32 + XHotspot uint32 + YHotspot uint32 + HbmMask windows.Handle + HbmColor windows.Handle +} + +type winBitmap struct { + BmType int32 + BmWidth int32 + BmHeight int32 + BmWidthBytes int32 + BmPlanes uint16 + BmBitsPixel uint16 + BmBits uintptr +} + +type winBitmapInfoHeader struct { + BiSize uint32 + BiWidth int32 + BiHeight int32 + BiPlanes uint16 + BiBitCount uint16 + BiCompression uint32 + BiSizeImage uint32 + BiXPelsPerMeter int32 + BiYPelsPerMeter int32 + BiClrUsed uint32 + BiClrImportant uint32 +} + +// cursorSnapshot is the captured cursor state shared between the worker +// (which polls the OS) and the session encoder (which reads it). +type cursorSnapshot struct { + img *image.RGBA + hotX int + hotY int + serial uint64 + err error +} + +// cursorSampler captures the foreground process's cursor sprite via Win32 +// APIs. It must be called from a goroutine attached to the same window +// station and desktop as the user session (the capture worker does this +// via switchToInputDesktop). lastHandle dedupes per-shape work so we only +// touch GDI when Windows hands us a new cursor. +type cursorSampler struct { + lastHandle windows.Handle + serial uint64 + snapshot *cursorSnapshot +} + +// sample queries the current cursor and decodes a new sprite when Windows +// reports a different HCURSOR than last time. Returns the current snapshot +// regardless of whether anything changed; callers diff by serial. +func (s *cursorSampler) sample() (*cursorSnapshot, error) { + var ci winCursorInfo + ci.Size = uint32(unsafe.Sizeof(ci)) + r, _, err := procGetCursorInfo.Call(uintptr(unsafe.Pointer(&ci))) + if r == 0 { + return nil, fmt.Errorf("GetCursorInfo: %w", err) + } + if ci.Flags&cursorShowing == 0 || ci.Cursor == 0 { + // Cursor temporarily hidden by an app (text fields toggle it on + // focus). Emit a 1x1 transparent sprite so the client renders no + // cursor and stay armed for the next handle change rather than + // treating this as a hard failure that would latch us off for + // the session. + if s.lastHandle == hiddenHandle { + return s.snapshot, nil + } + s.lastHandle = hiddenHandle + s.serial++ + s.snapshot = &cursorSnapshot{img: transparentCursorImage(), serial: s.serial} + return s.snapshot, nil + } + if ci.Cursor == s.lastHandle && s.snapshot != nil { + return s.snapshot, nil + } + img, hotX, hotY, err := decodeCursor(ci.Cursor) + if err != nil { + return nil, err + } + s.lastHandle = ci.Cursor + s.serial++ + s.snapshot = &cursorSnapshot{img: img, hotX: hotX, hotY: hotY, serial: s.serial} + return s.snapshot, nil +} + +// decodeCursor extracts the sprite at hCur as RGBA along with the hotspot. +// Color cursors are read from the colour bitmap with the AND mask combined +// in for alpha. Monochrome cursors collapse the two halves of the mask +// bitmap into a single visible sprite where the AND bit drives alpha. +func decodeCursor(hCur windows.Handle) (*image.RGBA, int, int, error) { + var info winIconInfo + r, _, err := procGetIconInfo.Call(uintptr(hCur), uintptr(unsafe.Pointer(&info))) + if r == 0 { + return nil, 0, 0, fmt.Errorf("GetIconInfo: %w", err) + } + defer func() { + if info.HbmMask != 0 { + procDeleteObject.Call(uintptr(info.HbmMask)) + } + if info.HbmColor != 0 { + procDeleteObject.Call(uintptr(info.HbmColor)) + } + }() + hotX, hotY := int(info.XHotspot), int(info.YHotspot) + if info.HbmColor != 0 { + img, err := decodeColorCursor(info.HbmColor, info.HbmMask) + if err != nil { + return nil, 0, 0, err + } + return img, hotX, hotY, nil + } + img, err := decodeMonoCursor(info.HbmMask) + if err != nil { + return nil, 0, 0, err + } + return img, hotX, hotY, nil +} + +// readBitmap returns the BITMAP descriptor for hbm. +func readBitmap(hbm windows.Handle) (winBitmap, error) { + var bm winBitmap + r, _, err := procGetObjectW.Call(uintptr(hbm), unsafe.Sizeof(bm), uintptr(unsafe.Pointer(&bm))) + if r == 0 { + return winBitmap{}, fmt.Errorf("GetObject: %w", err) + } + return bm, nil +} + +// dibCopy reads hbm as 32bpp top-down BGRA into a freshly allocated slice +// matching w*h*4 bytes. The bitmap may be selected into the screen DC so +// we use a memory DC to keep the call cheap. +func dibCopy(hbm windows.Handle, w, h int32) ([]byte, error) { + hdcScreen, _, _ := procGetDC.Call(0) + if hdcScreen == 0 { + return nil, fmt.Errorf("GetDC: failed") + } + defer procReleaseDC.Call(0, hdcScreen) + hdcMem, _, _ := procCreateCompatDC.Call(hdcScreen) + if hdcMem == 0 { + return nil, fmt.Errorf("CreateCompatibleDC: failed") + } + defer procDeleteDC.Call(hdcMem) + + var bih winBitmapInfoHeader + bih.BiSize = dibSectionBytes + bih.BiWidth = w + bih.BiHeight = -h // top-down + bih.BiPlanes = 1 + bih.BiBitCount = 32 + bih.BiCompression = biRgb + + buf := make([]byte, int(w)*int(h)*4) + r, _, err := procGetDIBits.Call( + hdcMem, + uintptr(hbm), + 0, + uintptr(h), + uintptr(unsafe.Pointer(&buf[0])), + uintptr(unsafe.Pointer(&bih)), + diRgbColors, + ) + if r == 0 { + return nil, fmt.Errorf("GetDIBits: %w", err) + } + return buf, nil +} + +// decodeColorCursor reads a 32bpp colour cursor and folds the AND mask into +// the alpha channel when the colour bitmap leaves it zero. +func decodeColorCursor(hbmColor, hbmMask windows.Handle) (*image.RGBA, error) { + bm, err := readBitmap(hbmColor) + if err != nil { + return nil, err + } + w, h := bm.BmWidth, bm.BmHeight + color, err := dibCopy(hbmColor, w, h) + if err != nil { + return nil, err + } + var mask []byte + if hbmMask != 0 { + mask, _ = dibCopy(hbmMask, w, h) + } + img := image.NewRGBA(image.Rect(0, 0, int(w), int(h))) + hasAlpha := false + for i := 0; i < len(color); i += 4 { + if color[i+3] != 0 { + hasAlpha = true + break + } + } + for y := int32(0); y < h; y++ { + for x := int32(0); x < w; x++ { + si := (y*w + x) * 4 + di := (y*w + x) * 4 + b := color[si] + g := color[si+1] + r := color[si+2] + a := color[si+3] + if !hasAlpha { + a = 255 + if mask != nil { + // AND mask: 1 = transparent, 0 = opaque. The DIB + // representation we requested is 32bpp so each "bit" + // is a 4-byte entry; we use the first byte as the + // effective AND value. + if mask[si] != 0 { + a = 0 + } + } + } + img.Pix[di+0] = r + img.Pix[di+1] = g + img.Pix[di+2] = b + img.Pix[di+3] = a + } + } + return img, nil +} + +// decodeMonoCursor handles legacy 1bpp cursors where hbmMask is twice as +// tall as the visible sprite: rows [0..h) are the AND mask and rows [h..2h) +// are the XOR mask. We render the visible half into RGBA, treating +// AND-mask=1 as transparent and the XOR bit as a black/white pixel. +func decodeMonoCursor(hbmMask windows.Handle) (*image.RGBA, error) { + bm, err := readBitmap(hbmMask) + if err != nil { + return nil, err + } + w, fullH := bm.BmWidth, bm.BmHeight + if fullH%2 != 0 { + return nil, fmt.Errorf("unexpected mono cursor shape: %dx%d", w, fullH) + } + h := fullH / 2 + data, err := dibCopy(hbmMask, w, fullH) + if err != nil { + return nil, err + } + img := image.NewRGBA(image.Rect(0, 0, int(w), int(h))) + for y := int32(0); y < h; y++ { + for x := int32(0); x < w; x++ { + and := data[(y*w+x)*4] + xor := data[((y+h)*w+x)*4] + di := (y*w + x) * 4 + if and != 0 { + img.Pix[di+3] = 0 + continue + } + c := byte(0) + if xor != 0 { + c = 255 + } + img.Pix[di+0] = c + img.Pix[di+1] = c + img.Pix[di+2] = c + img.Pix[di+3] = 255 + } + } + return img, nil +} + +// cursorState is the latest snapshot shared between the worker and +// session readers. +type cursorState struct { + mu sync.Mutex + snapshot *cursorSnapshot +} + +func (s *cursorState) store(snap *cursorSnapshot) { + s.mu.Lock() + s.snapshot = snap + s.mu.Unlock() +} + +func (s *cursorState) load() *cursorSnapshot { + s.mu.Lock() + snap := s.snapshot + s.mu.Unlock() + return snap +} + +// Cursor satisfies cursorSource by returning the latest snapshot the +// capture worker decoded. The "no sample yet" and "cursor hidden" cases +// return img=nil with no error so callers skip emission this cycle +// without latching the source off for the rest of the session. +func (c *DesktopCapturer) Cursor() (*image.RGBA, int, int, uint64, error) { + snap := c.cursorState.load() + if snap == nil { + return nil, 0, 0, 0, nil + } + if snap.err != nil { + return nil, 0, 0, 0, snap.err + } + return snap.img, snap.hotX, snap.hotY, snap.serial, nil +} diff --git a/client/vnc/server/cursor_x11.go b/client/vnc/server/cursor_x11.go new file mode 100644 index 000000000..5dd06f5c5 --- /dev/null +++ b/client/vnc/server/cursor_x11.go @@ -0,0 +1,87 @@ +//go:build unix && !darwin && !ios && !android + +package server + +import ( + "fmt" + "image" + "sync" + + "github.com/jezek/xgb" + "github.com/jezek/xgb/xfixes" +) + +// xfixesCursor reports the current X cursor sprite via the XFixes extension. +// CursorSerial changes whenever the server picks a different cursor, so +// callers can cache by serial without comparing pixels. +type xfixesCursor struct { + mu sync.Mutex + conn *xgb.Conn + // runtimeErr latches the first GetCursorImage failure so subsequent + // calls return quickly without another X round-trip. Some virtual + // displays advertise XFixes but reject GetCursorImage (Xvfb). + runtimeErr error +} + +// newXFixesCursor initialises the XFixes extension on conn. Returns an +// error if the extension is unavailable; callers can fall back to no +// cursor emission instead of asking on every frame. +func newXFixesCursor(conn *xgb.Conn) (*xfixesCursor, error) { + if err := xfixes.Init(conn); err != nil { + return nil, fmt.Errorf("xfixes init: %w", err) + } + if _, err := xfixes.QueryVersion(conn, 4, 0).Reply(); err != nil { + return nil, fmt.Errorf("xfixes query version: %w", err) + } + return &xfixesCursor{conn: conn}, nil +} + +// Cursor returns the current cursor sprite as RGBA along with its hotspot +// and serial. Callers should treat an unchanged serial as "no update". +func (c *xfixesCursor) Cursor() (*image.RGBA, int, int, uint64, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.runtimeErr != nil { + return nil, 0, 0, 0, c.runtimeErr + } + reply, err := xfixes.GetCursorImage(c.conn).Reply() + if err != nil { + c.runtimeErr = fmt.Errorf("xfixes GetCursorImage: %w", err) + return nil, 0, 0, 0, c.runtimeErr + } + w, h := int(reply.Width), int(reply.Height) + if w <= 0 || h <= 0 { + return nil, 0, 0, 0, fmt.Errorf("cursor has zero extent") + } + if len(reply.CursorImage) < w*h { + return nil, 0, 0, 0, fmt.Errorf("cursor pixel buffer truncated: %d < %d", len(reply.CursorImage), w*h) + } + img := image.NewRGBA(image.Rect(0, 0, w, h)) + // XFixes packs each pixel as a uint32 in ARGB order with premultiplied + // alpha. Unpack into the standard RGBA byte layout. + for i, p := range reply.CursorImage[:w*h] { + o := i * 4 + img.Pix[o+0] = byte(p >> 16) + img.Pix[o+1] = byte(p >> 8) + img.Pix[o+2] = byte(p) + img.Pix[o+3] = byte(p >> 24) + } + return img, int(reply.Xhot), int(reply.Yhot), uint64(reply.CursorSerial), nil +} + +// Cursor on X11Capturer satisfies cursorSource. The XFixes binding is +// created lazily on the same X connection used for screen capture; the +// first init failure is latched so we stop asking on every frame. +func (x *X11Capturer) Cursor() (*image.RGBA, int, int, uint64, error) { + x.mu.Lock() + if x.cursor == nil && x.cursorInitErr == nil { + x.cursor, x.cursorInitErr = newXFixesCursor(x.conn) + } + cur := x.cursor + initErr := x.cursorInitErr + x.mu.Unlock() + if initErr != nil { + return nil, 0, 0, 0, initErr + } + return cur.Cursor() +} diff --git a/client/vnc/server/rfb.go b/client/vnc/server/rfb.go index 11a288eb3..e8e6e11c2 100644 --- a/client/vnc/server/rfb.go +++ b/client/vnc/server/rfb.go @@ -63,6 +63,7 @@ const ( // Pseudo-encodings carried over wire as rects with a negative // encoding value. The client advertises supported optional protocol // extensions by listing these in SetEncodings. + pseudoEncCursor = -239 pseudoEncDesktopSize = -223 pseudoEncLastRect = -224 pseudoEncQEMUExtendedKeyEvent = -258 diff --git a/client/vnc/server/server.go b/client/vnc/server/server.go index 0cbd95182..1e6a21bac 100644 --- a/client/vnc/server/server.go +++ b/client/vnc/server/server.go @@ -72,6 +72,13 @@ type captureIntoer interface { CaptureInto(dst *image.RGBA) error } +// cursorSource is implemented by capturers that can report the platform +// cursor sprite so the session can emit it via the Cursor pseudo-encoding +// (RFB 7.7.4). serial bumps on shape changes; callers cache by serial. +type cursorSource interface { + Cursor() (img *image.RGBA, hotX, hotY int, serial uint64, err error) +} + // errFrameUnchanged is returned by capturers that hash the raw source // bytes (currently macOS) when the new frame is byte-identical to the // last one, so the encoder can short-circuit to an empty update. @@ -556,6 +563,10 @@ func (s *Server) handleConnection(conn net.Conn) { serverW: capturer.Width(), serverH: capturer.Height(), log: connLog, + // Virtual sessions run on Xvfb which has no usable cursor source, + // so we skip the Cursor pseudo-encoding and let the dashboard's + // local fallback show instead. + disableCursor: header.mode == ModeSession, } sess.serve() } diff --git a/client/vnc/server/session.go b/client/vnc/server/session.go index 0228701c6..ae138f135 100644 --- a/client/vnc/server/session.go +++ b/client/vnc/server/session.go @@ -78,7 +78,20 @@ type session struct { clientSupportsLastRect bool clientSupportsQEMUKey bool clientSupportsExtClipboard bool + clientSupportsCursor bool extClipCapsSent bool + // lastCursorSerial is the serial of the cursor sprite last emitted. + // The encoder re-queries the source each cycle and only emits when + // the serial changes. + lastCursorSerial uint64 + // cursorSourceFailed latches a permanent failure from the cursor + // source so the encoder stops polling for the rest of the session. + // Reset on SetEncodings so a reconnect can retry. + cursorSourceFailed bool + // disableCursor suppresses the Cursor pseudo-encoding regardless of + // what the client advertises. Set for virtual sessions where no + // usable cursor source exists. Constant for the session lifetime. + disableCursor bool // clientJPEGQuality and clientZlibLevel hold the 0..9 levels the client // advertised via the QualityLevel / CompressLevel pseudo-encodings, or // -1 when the client has not expressed a preference. Applied to the @@ -362,6 +375,8 @@ func (s *session) resetEncodingCaps() { s.clientSupportsLastRect = false s.clientSupportsQEMUKey = false s.clientSupportsExtClipboard = false + s.clientSupportsCursor = false + s.cursorSourceFailed = false s.clientJPEGQuality = -1 s.clientZlibLevel = -1 } @@ -395,6 +410,12 @@ func (s *session) applyEncoding(enc int32) string { case pseudoEncExtendedClipboard: s.clientSupportsExtClipboard = true return "ext-clipboard" + case pseudoEncCursor: + if s.disableCursor { + return "" + } + s.clientSupportsCursor = true + return "cursor" case encTight: s.useTight = true return "tight" diff --git a/client/vnc/server/session_cursor.go b/client/vnc/server/session_cursor.go new file mode 100644 index 000000000..bf3765adc --- /dev/null +++ b/client/vnc/server/session_cursor.go @@ -0,0 +1,87 @@ +//go:build !js && !ios && !android + +package server + +import ( + "encoding/binary" + "image" +) + +// pendingCursorRect returns the Cursor pseudo-rect for the current sprite +// when the client negotiated the encoding and the platform exposes a +// cursor source whose serial has changed since the last emission. A nil +// return means "do not include a cursor rect in this FramebufferUpdate". +func (s *session) pendingCursorRect() []byte { + s.encMu.RLock() + supported := s.clientSupportsCursor + failed := s.cursorSourceFailed + lastSerial := s.lastCursorSerial + s.encMu.RUnlock() + if !supported || failed { + return nil + } + src, ok := s.capturer.(cursorSource) + if !ok { + return nil + } + img, hotX, hotY, serial, err := src.Cursor() + if err != nil { + s.encMu.Lock() + s.cursorSourceFailed = true + s.encMu.Unlock() + s.log.Debugf("cursor source unavailable: %v", err) + return nil + } + if img == nil || serial == lastSerial { + return nil + } + buf := encodeCursorPseudoRect(img, hotX, hotY) + s.encMu.Lock() + s.lastCursorSerial = serial + s.encMu.Unlock() + return buf +} + +// encodeCursorPseudoRect packs the cursor sprite into a Cursor pseudo +// rectangle (RFB 7.7.4, pseudo-encoding -239). Layout: 12-byte rect header +// followed by w*h*4 BGRX pixel bytes and a 1-bit mask of (w+7)/8 bytes per +// row, MSB-first, with each row independently padded. +func encodeCursorPseudoRect(img *image.RGBA, hotX, hotY int) []byte { + w, h := img.Rect.Dx(), img.Rect.Dy() + pixelBytes := w * h * 4 + maskStride := (w + 7) / 8 + maskBytes := maskStride * h + buf := make([]byte, 12+pixelBytes+maskBytes) + + binary.BigEndian.PutUint16(buf[0:2], uint16(hotX)) + binary.BigEndian.PutUint16(buf[2:4], uint16(hotY)) + binary.BigEndian.PutUint16(buf[4:6], uint16(w)) + binary.BigEndian.PutUint16(buf[6:8], uint16(h)) + enc := int32(pseudoEncCursor) + binary.BigEndian.PutUint32(buf[8:12], uint32(enc)) + + pix := buf[12 : 12+pixelBytes] + mask := buf[12+pixelBytes:] + src := img.Pix + stride := img.Stride + for y := 0; y < h; y++ { + row := y * stride + dstRow := y * w * 4 + maskRow := y * maskStride + for x := 0; x < w; x++ { + r := src[row+x*4+0] + g := src[row+x*4+1] + b := src[row+x*4+2] + a := src[row+x*4+3] + off := dstRow + x*4 + pix[off+0] = b + pix[off+1] = g + pix[off+2] = r + pix[off+3] = 0 + if a >= 0x80 { + mask[maskRow+x/8] |= 0x80 >> (x % 8) + } + } + } + return buf +} diff --git a/client/vnc/server/session_encode.go b/client/vnc/server/session_encode.go index 4d6501591..2197bf8bc 100644 --- a/client/vnc/server/session_encode.go +++ b/client/vnc/server/session_encode.go @@ -377,14 +377,22 @@ func (s *session) swapPrevCur() { s.prevFrame, s.curFrame = s.curFrame, s.prevFrame } -// sendEmptyUpdate sends a FramebufferUpdate with zero rectangles. +// sendEmptyUpdate sends a FramebufferUpdate with zero pixel rectangles. +// When the cursor source reports a fresh sprite we still slip the Cursor +// pseudo-rect into the same message so a shape change (e.g. hovering onto +// a resize handle) reaches the client without waiting for a dirty frame. func (s *session) sendEmptyUpdate() error { - var buf [4]byte + cursorRect := s.pendingCursorRect() + if cursorRect == nil { + var buf [4]byte + buf[0] = serverFramebufferUpdate + return s.writeFramed(buf[:]) + } + buf := make([]byte, 4+len(cursorRect)) buf[0] = serverFramebufferUpdate - s.writeMu.Lock() - _, err := s.conn.Write(buf[:]) - s.writeMu.Unlock() - return err + binary.BigEndian.PutUint16(buf[2:4], 1) + copy(buf[4:], cursorRect) + return s.writeFramed(buf) } func (s *session) sendFullUpdate(img *image.RGBA) error { @@ -398,27 +406,40 @@ func (s *session) sendFullUpdate(img *image.RGBA) error { zlib := s.zlib s.encMu.RUnlock() - if useTight && tight != nil && pfIsTightCompatible(pf) { - rectBuf := encodeTightRect(img, pf, 0, 0, w, h, tight) - buf := make([]byte, 4+len(rectBuf)) - buf[0] = serverFramebufferUpdate - binary.BigEndian.PutUint16(buf[2:4], 1) - copy(buf[4:], rectBuf) - s.writeMu.Lock() - _, err := s.conn.Write(buf) - s.writeMu.Unlock() - return err + cursorRect := s.pendingCursorRect() + rectCount := uint16(1) + if cursorRect != nil { + rectCount++ } - if useZlib && zlib != nil { - buf := encodeZlibRect(img, pf, 0, 0, w, h, zlib) - s.writeMu.Lock() - _, err := s.conn.Write(buf) - s.writeMu.Unlock() - return err + var rectBuf []byte + switch { + case useTight && tight != nil && pfIsTightCompatible(pf): + rectBuf = encodeTightRect(img, pf, 0, 0, w, h, tight) + case useZlib && zlib != nil: + // encodeZlibRect bakes in its own FBU header; reuse it for the + // single-rect path when there is no cursor to prepend. + if cursorRect == nil { + return s.writeFramed(encodeZlibRect(img, pf, 0, 0, w, h, zlib)) + } + rectBuf = encodeZlibRect(img, pf, 0, 0, w, h, zlib)[4:] + default: + if cursorRect == nil { + return s.writeFramed(encodeRawRect(img, pf, 0, 0, w, h)) + } + rectBuf = encodeRawRect(img, pf, 0, 0, w, h)[4:] } - buf := encodeRawRect(img, pf, 0, 0, w, h) + buf := make([]byte, 4+len(cursorRect)+len(rectBuf)) + buf[0] = serverFramebufferUpdate + binary.BigEndian.PutUint16(buf[2:4], rectCount) + off := 4 + off += copy(buf[off:], cursorRect) + copy(buf[off:], rectBuf) + return s.writeFramed(buf) +} + +func (s *session) writeFramed(buf []byte) error { s.writeMu.Lock() _, err := s.conn.Write(buf) s.writeMu.Unlock() @@ -430,11 +451,15 @@ func (s *session) sendFullUpdate(img *image.RGBA) error { // their source tiles are read from the client's pre-update framebuffer state, // before any subsequent rect overwrites them. func (s *session) sendDirtyAndMoves(img *image.RGBA, moves []copyRectMove, rects [][4]int) error { - if len(moves) == 0 && len(rects) == 0 { + cursorRect := s.pendingCursorRect() + if len(moves) == 0 && len(rects) == 0 && cursorRect == nil { return nil } total := len(moves) + len(rects) + if cursorRect != nil { + total++ + } header := make([]byte, 4) header[0] = serverFramebufferUpdate binary.BigEndian.PutUint16(header[2:4], uint16(total)) @@ -446,6 +471,12 @@ func (s *session) sendDirtyAndMoves(img *image.RGBA, moves []copyRectMove, rects return err } + if cursorRect != nil { + if _, err := s.conn.Write(cursorRect); err != nil { + return err + } + } + ts := tileSize for _, m := range moves { body := encodeCopyRectBody(m.srcX, m.srcY, m.dstX, m.dstY, ts, ts)