Restore Hextile SolidFill and Zlib encoding paths

client/vnc/server/rfb.go

@@ -77,6 +77,10 @@ const (
 	pseudoEncCompressLevelMin = -256
 	pseudoEncCompressLevelMax = -247
 
+	// Hextile sub-encoding bits used by the SolidFill fast path.
+	hextileBackgroundSpecified = 0x02
+	hextileSubSize             = 16
+
 	// Tight compression-control byte top nibble. Stream-reset bits 0-3
 	// (one per zlib stream) are unused while we run a single stream.
 	tightFillSubenc  = 0x80
@@ -248,6 +252,73 @@ func encodeRawRect(img *image.RGBA, pf clientPixelFormat, x, y, w, h int) []byte
 	return buf
 }
 
+// encodeZlibRect encodes a framebuffer region using the standalone Zlib
+// encoding. The zlib stream is continuous for the entire VNC session: the
+// client keeps a single inflate context and reuses it across rects. The
+// returned buffer includes the 4-byte FramebufferUpdate header.
+func encodeZlibRect(img *image.RGBA, pf clientPixelFormat, x, y, w, h int, z *zlibState) []byte {
+	zw, zbuf := z.w, z.buf
+	zbuf.Reset()
+
+	rowBytes := w * 4
+	total := rowBytes * h
+	if cap(z.scratch) < total {
+		z.scratch = make([]byte, total)
+	}
+	scratch := z.scratch[:total]
+	writePixels(scratch, img, pf, rect{x, y, w, h})
+	for row := 0; row < h; row++ {
+		if _, err := zw.Write(scratch[row*rowBytes : (row+1)*rowBytes]); err != nil {
+			log.Debugf("zlib write row %d: %v", row, err)
+			return nil
+		}
+	}
+	if err := zw.Flush(); err != nil {
+		log.Debugf("zlib flush: %v", err)
+		return nil
+	}
+	compressed := zbuf.Bytes()
+
+	buf := make([]byte, 4+12+4+len(compressed))
+	buf[0] = serverFramebufferUpdate
+	binary.BigEndian.PutUint16(buf[2:4], 1)
+	binary.BigEndian.PutUint16(buf[4:6], uint16(x))
+	binary.BigEndian.PutUint16(buf[6:8], uint16(y))
+	binary.BigEndian.PutUint16(buf[8:10], uint16(w))
+	binary.BigEndian.PutUint16(buf[10:12], uint16(h))
+	binary.BigEndian.PutUint32(buf[12:16], uint32(encZlib))
+	binary.BigEndian.PutUint32(buf[16:20], uint32(len(compressed)))
+	copy(buf[20:], compressed)
+	return buf
+}
+
+// encodeHextileSolidRect emits a Hextile-encoded rectangle whose every
+// pixel is the same colour. The first sub-tile carries the background
+// pixel; remaining sub-tiles inherit it via a zero sub-encoding byte,
+// collapsing a uniform 64×64 tile down to ~20 bytes. The returned buffer
+// starts with the 12-byte rect header; callers prepend a FramebufferUpdate
+// header.
+func encodeHextileSolidRect(r, g, b byte, pf clientPixelFormat, rc rect) []byte {
+	cols := (rc.w + hextileSubSize - 1) / hextileSubSize
+	rows := (rc.h + hextileSubSize - 1) / hextileSubSize
+	subs := cols * rows
+	// One sub-encoding byte plus a 32bpp pixel for the first sub-tile, then
+	// one zero byte per remaining sub-tile to inherit the background.
+	bodySize := 1 + 4 + (subs - 1)
+	buf := make([]byte, 12+bodySize)
+
+	binary.BigEndian.PutUint16(buf[0:2], uint16(rc.x))
+	binary.BigEndian.PutUint16(buf[2:4], uint16(rc.y))
+	binary.BigEndian.PutUint16(buf[4:6], uint16(rc.w))
+	binary.BigEndian.PutUint16(buf[6:8], uint16(rc.h))
+	binary.BigEndian.PutUint32(buf[8:12], uint32(encHextile))
+
+	buf[12] = hextileBackgroundSpecified
+	pixel := (uint32(r) << pf.rShift) | (uint32(g) << pf.gShift) | (uint32(b) << pf.bShift)
+	binary.LittleEndian.PutUint32(buf[13:17], pixel)
+	return buf
+}
+
 // writePixels writes a rectangle of img into dst as 32bpp little-endian
 // pixels at the negotiated RGB shifts. The pixel format is constrained at
 // SetPixelFormat time so we can assume 4 bytes per pixel, 8-bit channels,
@@ -719,6 +790,10 @@ func sampledColorCountInto(seen map[uint32]struct{}, img *image.RGBA, x, y, w, h
 type zlibState struct {
 	buf *bytes.Buffer
 	w   *zlib.Writer
+	// scratch stages the packed pixel stream for a rect before it is fed
+	// to the deflater. Grown to the largest rect seen in the session and
+	// reused to keep the steady-state encode allocation-free.
+	scratch []byte
 }
 
 func newZlibStateLevel(level int) *zlibState {

client/vnc/server/session.go

@@ -50,7 +50,10 @@ type session struct {
 	pf          clientPixelFormat
 	useTight    bool
 	useCopyRect bool
+	useZlib     bool
+	useHextile  bool
 	tight       *tightState
+	zlib        *zlibState
 	copyRectDet *copyRectDetector
 	// Pseudo-encodings the client advertised support for. Updated under
 	// encMu by handleSetEncodings and read by the encoder goroutine.
@@ -336,6 +339,8 @@ func (s *session) handleSetEncodings() error {
 func (s *session) resetEncodingCaps() {
 	s.useTight = false
 	s.useCopyRect = false
+	s.useZlib = false
+	s.useHextile = false
 	s.clientSupportsDesktopSize = false
 	s.clientSupportsExtendedDesktopSize = false
 	s.clientSupportsDesktopName = false
@@ -378,6 +383,15 @@ func (s *session) applyEncoding(enc int32) string {
 	case encTight:
 		s.useTight = true
 		return "tight"
+	case encZlib:
+		s.useZlib = true
+		if s.zlib == nil {
+			s.zlib = newZlibStateLevel(zlibLevelFor(-1))
+		}
+		return "zlib"
+	case encHextile:
+		s.useHextile = true
+		return "hextile"
 	}
 	if enc >= pseudoEncQualityLevelMin && enc <= pseudoEncQualityLevelMax {
 		s.clientJPEGQuality = int(enc - pseudoEncQualityLevelMin)

client/vnc/server/session_encode.go

@@ -291,12 +291,11 @@ func (s *session) sendFullUpdate(img *image.RGBA) error {
 	pf := s.pf
 	useTight := s.useTight
 	tight := s.tight
+	useZlib := s.useZlib
+	zlib := s.zlib
 	s.encMu.RUnlock()
 
 	if useTight && tight != nil && pfIsTightCompatible(pf) {
-		// Tight encodes arbitrary sizes natively (Fill for uniform, JPEG
-		// for photo-like, Basic+zlib otherwise). Wrap the rect bytes with
-		// the 4-byte FramebufferUpdate header.
 		rectBuf := encodeTightRect(img, pf, 0, 0, w, h, tight)
 		buf := make([]byte, 4+len(rectBuf))
 		buf[0] = serverFramebufferUpdate
@@ -308,6 +307,14 @@ func (s *session) sendFullUpdate(img *image.RGBA) error {
 		return err
 	}
 
+	if useZlib && zlib != nil {
+		buf := encodeZlibRect(img, pf, 0, 0, w, h, zlib)
+		s.writeMu.Lock()
+		_, err := s.conn.Write(buf)
+		s.writeMu.Unlock()
+		return err
+	}
+
 	buf := encodeRawRect(img, pf, 0, 0, w, h)
 	s.writeMu.Lock()
 	_, err := s.conn.Write(buf)
@@ -366,13 +373,27 @@ func (s *session) sendDirtyAndMoves(img *image.RGBA, moves []copyRectMove, rects
 func (s *session) encodeTile(img *image.RGBA, x, y, w, h int) []byte {
 	s.encMu.RLock()
 	pf := s.pf
+	useHextile := s.useHextile
 	useTight := s.useTight
 	tight := s.tight
+	useZlib := s.useZlib
+	zlib := s.zlib
 	s.encMu.RUnlock()
 
+	if useHextile {
+		if pixel, uniform := tileIsUniform(img, x, y, w, h); uniform {
+			r := byte(pixel)
+			g := byte(pixel >> 8)
+			b := byte(pixel >> 16)
+			return encodeHextileSolidRect(r, g, b, pf, rect{x, y, w, h})
+		}
+	}
 	if useTight && tight != nil && pfIsTightCompatible(pf) {
 		return encodeTightRect(img, pf, x, y, w, h, tight)
 	}
+	if useZlib && zlib != nil {
+		return encodeZlibRect(img, pf, x, y, w, h, zlib)[4:]
+	}
 	return encodeRawRect(img, pf, x, y, w, h)[4:]
 }