Signal Zlib encode failure and fall back to Raw

client/vnc/server/rfb.go

@@ -266,7 +266,7 @@ func encodeRawRect(img *image.RGBA, pf clientPixelFormat, x, y, w, h int) []byte
 // encoding. The zlib stream is continuous for the entire VNC session: the
 // client keeps a single inflate context and reuses it across rects. The
 // returned buffer includes the 4-byte FramebufferUpdate header.
-func encodeZlibRect(img *image.RGBA, pf clientPixelFormat, x, y, w, h int, z *zlibState) []byte {
+func encodeZlibRect(img *image.RGBA, pf clientPixelFormat, x, y, w, h int, z *zlibState) ([]byte, bool) {
 	zw, zbuf := z.w, z.buf
 	zbuf.Reset()
 
@@ -280,12 +280,12 @@ func encodeZlibRect(img *image.RGBA, pf clientPixelFormat, x, y, w, h int, z *zl
 	for row := 0; row < h; row++ {
 		if _, err := zw.Write(scratch[row*rowBytes : (row+1)*rowBytes]); err != nil {
 			log.Debugf("zlib write row %d: %v", row, err)
-			return nil
+			return nil, false
 		}
 	}
 	if err := zw.Flush(); err != nil {
 		log.Debugf("zlib flush: %v", err)
-		return nil
+		return nil, false
 	}
 	compressed := zbuf.Bytes()
 
@@ -299,7 +299,7 @@ func encodeZlibRect(img *image.RGBA, pf clientPixelFormat, x, y, w, h int, z *zl
 	binary.BigEndian.PutUint32(buf[12:16], uint32(encZlib))
 	binary.BigEndian.PutUint32(buf[16:20], uint32(len(compressed)))
 	copy(buf[20:], compressed)
-	return buf
+	return buf, true
 }
 
 // encodeHextileSolidRect emits a Hextile-encoded rectangle whose every

client/vnc/server/session_encode.go

@@ -450,11 +450,18 @@ func (s *session) sendFullUpdate(img *image.RGBA) error {
 		rectBuf = encodeTightRect(img, pf, 0, 0, w, h, tight)
 	case useZlib && zlib != nil:
 		// encodeZlibRect bakes in its own FBU header; reuse it for the
-		// single-rect path when there is no cursor to prepend.
+		// single-rect path when there is no cursor to prepend. Fall back
-		if cursorRect == nil {
+		// to Raw if the compressor errors out.
-			return s.writeFramed(encodeZlibRect(img, pf, 0, 0, w, h, zlib))
+		if zb, ok := encodeZlibRect(img, pf, 0, 0, w, h, zlib); ok {
+			if cursorRect == nil {
+				return s.writeFramed(zb)
+			}
+			rectBuf = zb[4:]
+		} else if cursorRect == nil {
+			return s.writeFramed(encodeRawRect(img, pf, 0, 0, w, h))
+		} else {
+			rectBuf = encodeRawRect(img, pf, 0, 0, w, h)[4:]
 		}
-		rectBuf = encodeZlibRect(img, pf, 0, 0, w, h, zlib)[4:]
 	default:
 		if cursorRect == nil {
 			return s.writeFramed(encodeRawRect(img, pf, 0, 0, w, h))
@@ -558,7 +565,9 @@ func (s *session) encodeTile(img *image.RGBA, x, y, w, h int) []byte {
 		return encodeTightRect(img, pf, x, y, w, h, tight)
 	}
 	if useZlib && zlib != nil {
-		return encodeZlibRect(img, pf, x, y, w, h, zlib)[4:]
+		if zb, ok := encodeZlibRect(img, pf, x, y, w, h, zlib); ok {
+			return zb[4:]
+		}
 	}
 	return encodeRawRect(img, pf, x, y, w, h)[4:]
 }