From 5efadf29ae4e895f48e156e1b0d3d9beedb8d20b Mon Sep 17 00:00:00 2001 From: Marco Fortina Date: Sun, 17 May 2026 06:42:51 +0000 Subject: [PATCH] libvncclient: add framebuffer viewport mode --- CMakeLists.txt | 9 ++ include/rfb/rfbclient.h | 50 +++++++++++ src/libvncclient/rfbclient.c | 60 ++++++++++++- src/libvncclient/vncviewer.c | 163 ++++++++++++++++++++++++++++------- test/client_viewporttest.c | 105 ++++++++++++++++++++++ 5 files changed, 357 insertions(+), 30 deletions(-) create mode 100644 test/client_viewporttest.c diff --git a/CMakeLists.txt b/CMakeLists.txt index d9709c809..4333cc908 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -704,6 +704,12 @@ if(WITH_LIBVNCSERVER) ) endif(WITH_LIBVNCSERVER) +if(WITH_LIBVNCCLIENT) + list(APPEND SIMPLETESTS + client_viewporttest + ) +endif(WITH_LIBVNCCLIENT) + if(WITH_THREADS AND (CMAKE_USE_PTHREADS_INIT OR CMAKE_USE_WIN32_THREADS_INIT) AND WITH_LIBVNCSERVER AND WITH_LIBVNCCLIENT) set(SIMPLETESTS @@ -759,6 +765,9 @@ endif(LIBVNCSERVER_WITH_WEBSOCKETS AND WITH_LIBVNCSERVER) if(WITH_LIBVNCSERVER) add_test(NAME cargs COMMAND test_cargstest) endif(WITH_LIBVNCSERVER) +if(WITH_LIBVNCCLIENT) + add_test(NAME client_viewport COMMAND test_client_viewporttest) +endif(WITH_LIBVNCCLIENT) if(UNIX) if(WITH_LIBVNCSERVER) add_test(NAME includetest_server COMMAND ${TESTS_DIR}/includetest.sh ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR} ${CMAKE_MAKE_PROGRAM} "rfb/rfb.h") diff --git a/include/rfb/rfbclient.h b/include/rfb/rfbclient.h index f72d84a49..6e5d6bc82 100644 --- a/include/rfb/rfbclient.h +++ b/include/rfb/rfbclient.h @@ -506,6 +506,22 @@ typedef struct _rfbClient { rfbBool isUpdateRectManagedByLib; GetX509CertFingerprintMismatchDecisionProc GetX509CertFingerprintMismatchDecision; + + /** + * Optional local framebuffer viewport. + * + * When enabled before rfbClientInitialise(), update requests still use + * framebuffer coordinates from the remote server, but the default framebuffer + * allocation and default drawing callbacks store only this sub-rectangle + * locally. Encodings that require pixels outside the requested viewport are + * not requested by the default encoding negotiation while this mode is + * enabled. + */ + rfbBool useFrameBufferViewport; + int frameBufferViewportX; + int frameBufferViewportY; + int frameBufferViewportW; + int frameBufferViewportH; } rfbClient; /* cursor.c */ @@ -528,6 +544,40 @@ extern rfbBool rfbEnableClientLogging; typedef void (*rfbClientLogProc)(const char *format, ...); extern rfbClientLogProc rfbClientLog,rfbClientErr; extern rfbBool ConnectToRFBServer(rfbClient* client,const char *hostname, int port); +/** + * Enables the default LibVNCClient framebuffer viewport mode. + * + * The supplied rectangle is expressed in remote framebuffer coordinates. This + * must be called before rfbClientInitialise() or before any application-provided + * framebuffer allocation. Once client->frameBuffer is allocated, changing or + * clearing the viewport is rejected/ignored because the framebuffer stride and + * size have already been chosen. + * + * When active, the default framebuffer allocator will allocate storage only for + * this viewport, and the default drawing callbacks will clip incoming + * rectangles to it before writing into client->frameBuffer. The update + * rectangle is set to the same viewport. + * + * This mode is intentionally conservative: the default SetEncodings request is + * limited to encodings whose default callbacks can safely draw into the reduced + * local framebuffer. If an application overrides drawing callbacks such as + * GotBitmap, GotFillRect or GotCopyRect, that application is responsible for + * applying the same remote-to-local viewport mapping. + * + * Returns FALSE if the client is NULL, the rectangle is invalid, the viewport is + * outside the known remote framebuffer size, or client->frameBuffer has already + * been allocated. + */ +extern rfbBool rfbClientSetFrameBufferViewport(rfbClient *client, int x, int y, int w, int h); +/** + * Disables the default LibVNCClient framebuffer viewport mode. + * + * This must also be called before rfbClientInitialise() or before any + * framebuffer allocation. If client->frameBuffer is already allocated, the + * function leaves the existing viewport enabled to avoid making the allocated + * framebuffer inconsistent with the requested update rectangle. + */ +extern void rfbClientClearFrameBufferViewport(rfbClient *client); extern rfbBool ConnectToRFBRepeater(rfbClient* client,const char *repeaterHost, int repeaterPort, const char *destHost, int destPort); extern void SetClientAuthSchemes(rfbClient* client,const uint32_t *authSchemes, int size); extern rfbBool InitialiseRFBConnection(rfbClient* client); diff --git a/src/libvncclient/rfbclient.c b/src/libvncclient/rfbclient.c index f6baf641c..1e6964ab3 100644 --- a/src/libvncclient/rfbclient.c +++ b/src/libvncclient/rfbclient.c @@ -1281,7 +1281,14 @@ SetFormatAndEncodings(rfbClient* client) se->pad = 0; se->nEncodings = 0; - if (client->appData.encodingsString) { + if (client->useFrameBufferViewport) { + rfbClientLog("Framebuffer viewport active: requesting viewport-safe encodings only\n"); + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingHextile); + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingCoRRE); + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRRE); + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRaw); + } + else if (client->appData.encodingsString) { const char *encStr = client->appData.encodingsString; int encStrLen; do { @@ -2021,6 +2028,57 @@ rfbClientProcessExtServerCutText(rfbClient* client, char *data, int len) } #endif +rfbBool rfbClientSetFrameBufferViewport(rfbClient *client, int x, int y, int w, int h) +{ + rfbRectangle rect; + + if (!client || x < 0 || y < 0 || w <= 0 || h <= 0) + return FALSE; + + if (client->frameBuffer) { + rfbClientErr("Cannot change framebuffer viewport after framebuffer allocation\n"); + return FALSE; + } + + if ((client->width > 0 && (uint64_t)x + (uint64_t)w > (uint64_t)client->width) || + (client->height > 0 && (uint64_t)y + (uint64_t)h > (uint64_t)client->height)) { + rfbClientErr("Framebuffer viewport %dx%d at (%d, %d) exceeds remote framebuffer %dx%d\n", + w, h, x, y, client->width, client->height); + return FALSE; + } + + client->useFrameBufferViewport = TRUE; + client->frameBufferViewportX = x; + client->frameBufferViewportY = y; + client->frameBufferViewportW = w; + client->frameBufferViewportH = h; + + rect.x = x; + rect.y = y; + rect.w = w; + rect.h = h; + rfbClientSetUpdateRect(client, &rect); + return TRUE; +} + +void rfbClientClearFrameBufferViewport(rfbClient *client) +{ + if (!client) + return; + + if (client->frameBuffer) { + rfbClientErr("Cannot clear framebuffer viewport after framebuffer allocation\n"); + return; + } + + client->useFrameBufferViewport = FALSE; + client->frameBufferViewportX = 0; + client->frameBufferViewportY = 0; + client->frameBufferViewportW = 0; + client->frameBufferViewportH = 0; + rfbClientSetUpdateRect(client, NULL); +} + void rfbClientSetUpdateRect(rfbClient *client, rfbRectangle *rect) { if (rect) { client->updateRect.x = rect->x; diff --git a/src/libvncclient/vncviewer.c b/src/libvncclient/vncviewer.c index 2af3785e0..20f9c1c8a 100644 --- a/src/libvncclient/vncviewer.c +++ b/src/libvncclient/vncviewer.c @@ -83,19 +83,83 @@ static char* ReadPassword(rfbClient* client) { #endif return p; } +static int FrameBufferStridePixels(rfbClient* client) +{ + if (client->useFrameBufferViewport) + return client->frameBufferViewportW; + return client->width; +} + +static rfbBool ClipToFrameBuffer(rfbClient* client, int x, int y, int w, int h, + int* localX, int* localY, + int* srcX, int* srcY, + int* clippedW, int* clippedH) +{ + int fbX = 0; + int fbY = 0; + int fbW = client->width; + int fbH = client->height; + int x0, y0, x1, y1; + + if (w <= 0 || h <= 0) + return FALSE; + + if (client->useFrameBufferViewport) { + fbX = client->frameBufferViewportX; + fbY = client->frameBufferViewportY; + fbW = client->frameBufferViewportW; + fbH = client->frameBufferViewportH; + } + + x0 = x > fbX ? x : fbX; + y0 = y > fbY ? y : fbY; + x1 = (x + w) < (fbX + fbW) ? (x + w) : (fbX + fbW); + y1 = (y + h) < (fbY + fbH) ? (y + h) : (fbY + fbH); + + if (x0 >= x1 || y0 >= y1) + return FALSE; + + *localX = x0 - fbX; + *localY = y0 - fbY; + *srcX = x0 - x; + *srcY = y0 - y; + *clippedW = x1 - x0; + *clippedH = y1 - y0; + return TRUE; +} + +static rfbBool RectFullyInsideFrameBuffer(rfbClient* client, int x, int y, int w, int h, + int* localX, int* localY) +{ + int srcX, srcY, clippedW, clippedH; + + if (!ClipToFrameBuffer(client, x, y, w, h, + localX, localY, &srcX, &srcY, &clippedW, &clippedH)) + return FALSE; + + return srcX == 0 && srcY == 0 && clippedW == w && clippedH == h; +} + static rfbBool MallocFrameBuffer(rfbClient* client) { uint64_t allocSize; + int width = client->width; + int height = client->height; if(client->frameBuffer) { free(client->frameBuffer); client->frameBuffer = NULL; } + if (client->useFrameBufferViewport) { + width = client->frameBufferViewportW; + height = client->frameBufferViewportH; + } + /* SECURITY: promote 'width' into uint64_t so that the multiplication does not overflow 'width' and 'height' are 16-bit integers per RFB protocol design SIZE_MAX is the maximum value that can fit into size_t */ - allocSize = (uint64_t)client->width * client->height * client->format.bitsPerPixel/8; + allocSize = (uint64_t)width * height * client->format.bitsPerPixel/8; if (allocSize >= SIZE_MAX) { rfbClientErr("CRITICAL: cannot allocate frameBuffer, requested size is too large\n"); @@ -112,25 +176,25 @@ static rfbBool MallocFrameBuffer(rfbClient* client) { /* messages */ -static rfbBool CheckRect(rfbClient* client, int x, int y, int w, int h) { - return x + w <= client->width && y + h <= client->height; -} - static void FillRectangle(rfbClient* client, int x, int y, int w, int h, uint32_t colour) { int i,j; + int localX, localY, srcX, srcY, clippedW, clippedH; + int stride; if (client->frameBuffer == NULL) { return; } - if (!CheckRect(client, x, y, w, h)) { - rfbClientLog("Rect out of bounds: %dx%d at (%d, %d)\n", x, y, w, h); + if (!ClipToFrameBuffer(client, x, y, w, h, + &localX, &localY, &srcX, &srcY, &clippedW, &clippedH)) { return; } + stride = FrameBufferStridePixels(client); + #define FILL_RECT(BPP) \ - for(j=y*client->width;j<(y+h)*client->width;j+=client->width) \ - for(i=x;iframeBuffer)[j+i]=colour; switch(client->format.bitsPerPixel) { @@ -144,22 +208,31 @@ static void FillRectangle(rfbClient* client, int x, int y, int w, int h, uint32_ static void CopyRectangle(rfbClient* client, const uint8_t* buffer, int x, int y, int w, int h) { int j; + int localX, localY, srcX, srcY, clippedW, clippedH; + int stride; if (client->frameBuffer == NULL) { return; } - if (!CheckRect(client, x, y, w, h)) { - rfbClientLog("Rect out of bounds: %dx%d at (%d, %d)\n", x, y, w, h); + if (!ClipToFrameBuffer(client, x, y, w, h, + &localX, &localY, &srcX, &srcY, &clippedW, &clippedH)) { return; } + stride = FrameBufferStridePixels(client); + #define COPY_RECT(BPP) \ { \ - int rs = w * BPP / 8, rs2 = client->width * BPP / 8; \ - for (j = ((x * (BPP / 8)) + (y * rs2)); j < (y + h) * rs2; j += rs2) { \ - memcpy(client->frameBuffer + j, buffer, rs); \ - buffer += rs; \ + int srcStride = w * BPP / 8; \ + int dstStride = stride * BPP / 8; \ + int rowBytes = clippedW * BPP / 8; \ + const uint8_t* src = buffer + (srcY * srcStride) + (srcX * BPP / 8); \ + uint8_t* dst = client->frameBuffer + (localY * dstStride) + (localX * BPP / 8); \ + for (j = 0; j < clippedH; j++) { \ + memcpy(dst, src, rowBytes); \ + src += srcStride; \ + dst += dstStride; \ } \ } @@ -175,44 +248,59 @@ static void CopyRectangle(rfbClient* client, const uint8_t* buffer, int x, int y /* TODO: test */ static void CopyRectangleFromRectangle(rfbClient* client, int src_x, int src_y, int w, int h, int dest_x, int dest_y) { int i,j; + int localSrcX, localSrcY, localDestX, localDestY; + int destClipLocalX, destClipLocalY, destClipSrcX, destClipSrcY, clippedW, clippedH; + int clippedSrcX, clippedSrcY; + int stride; if (client->frameBuffer == NULL) { return; } - if (!CheckRect(client, src_x, src_y, w, h)) { - rfbClientLog("Source rect out of bounds: %dx%d at (%d, %d)\n", src_x, src_y, w, h); + if (!ClipToFrameBuffer(client, dest_x, dest_y, w, h, + &destClipLocalX, &destClipLocalY, + &destClipSrcX, &destClipSrcY, + &clippedW, &clippedH)) { return; } - if (!CheckRect(client, dest_x, dest_y, w, h)) { - rfbClientLog("Dest rect out of bounds: %dx%d at (%d, %d)\n", dest_x, dest_y, w, h); + clippedSrcX = src_x + destClipSrcX; + clippedSrcY = src_y + destClipSrcY; + + if (!RectFullyInsideFrameBuffer(client, clippedSrcX, clippedSrcY, clippedW, clippedH, + &localSrcX, &localSrcY) || + !RectFullyInsideFrameBuffer(client, dest_x + destClipSrcX, dest_y + destClipSrcY, + clippedW, clippedH, &localDestX, &localDestY)) { + rfbClientLog("CopyRect outside local framebuffer viewport: src=%dx%d at (%d, %d), dest=%dx%d at (%d, %d)\n", + w, h, src_x, src_y, w, h, dest_x, dest_y); return; } + stride = FrameBufferStridePixels(client); + #define COPY_RECT_FROM_RECT(BPP) \ { \ - uint##BPP##_t* _buffer=((uint##BPP##_t*)client->frameBuffer)+(src_y-dest_y)*client->width+src_x-dest_x; \ - if (dest_y < src_y) { \ - for(j = dest_y*client->width; j < (dest_y+h)*client->width; j += client->width) { \ - if (dest_x < src_x) { \ - for(i = dest_x; i < dest_x+w; i++) { \ + uint##BPP##_t* _buffer=((uint##BPP##_t*)client->frameBuffer)+(localSrcY-localDestY)*stride+localSrcX-localDestX; \ + if (localDestY < localSrcY) { \ + for(j = localDestY*stride; j < (localDestY+clippedH)*stride; j += stride) { \ + if (localDestX < localSrcX) { \ + for(i = localDestX; i < localDestX+clippedW; i++) { \ ((uint##BPP##_t*)client->frameBuffer)[j+i]=_buffer[j+i]; \ } \ } else { \ - for(i = dest_x+w-1; i >= dest_x; i--) { \ + for(i = localDestX+clippedW-1; i >= localDestX; i--) { \ ((uint##BPP##_t*)client->frameBuffer)[j+i]=_buffer[j+i]; \ } \ } \ } \ } else { \ - for(j = (dest_y+h-1)*client->width; j >= dest_y*client->width; j-=client->width) { \ - if (dest_x < src_x) { \ - for(i = dest_x; i < dest_x+w; i++) { \ + for(j = (localDestY+clippedH-1)*stride; j >= localDestY*stride; j-=stride) { \ + if (localDestX < localSrcX) { \ + for(i = localDestX; i < localDestX+clippedW; i++) { \ ((uint##BPP##_t*)client->frameBuffer)[j+i]=_buffer[j+i]; \ } \ } else { \ - for(i = dest_x+w-1; i >= dest_x; i--) { \ + for(i = localDestX+clippedW-1; i >= localDestX; i--) { \ ((uint##BPP##_t*)client->frameBuffer)[j+i]=_buffer[j+i]; \ } \ } \ @@ -319,6 +407,11 @@ rfbClient* rfbGetClient(int bitsPerSample,int samplesPerPixel, /* default: use complete frame buffer */ client->updateRect.x = -1; + client->useFrameBufferViewport = FALSE; + client->frameBufferViewportX = 0; + client->frameBufferViewportY = 0; + client->frameBufferViewportW = 0; + client->frameBufferViewportH = 0; client->frameBuffer = NULL; client->outputWindow = 0; @@ -437,6 +530,18 @@ rfbBool rfbClientInitialise(rfbClient* client) { client->width=client->si.framebufferWidth; client->height=client->si.framebufferHeight; + + if (client->useFrameBufferViewport) { + if (client->frameBufferViewportX + client->frameBufferViewportW > client->width || + client->frameBufferViewportY + client->frameBufferViewportH > client->height) { + rfbClientErr("Framebuffer viewport %dx%d at (%d, %d) exceeds remote framebuffer %dx%d\n", + client->frameBufferViewportW, client->frameBufferViewportH, + client->frameBufferViewportX, client->frameBufferViewportY, + client->width, client->height); + return FALSE; + } + } + if (!client->MallocFrameBuffer(client)) return FALSE; diff --git a/test/client_viewporttest.c b/test/client_viewporttest.c new file mode 100644 index 000000000..8729f1115 --- /dev/null +++ b/test/client_viewporttest.c @@ -0,0 +1,105 @@ +#include + +#include +#include +#include + +static void fail(const char *message) +{ + fprintf(stderr, "%s\n", message); + exit(1); +} + +static void assert_u32(uint32_t actual, uint32_t expected, const char *message) +{ + if (actual != expected) { + fprintf(stderr, "%s: got 0x%08x expected 0x%08x\n", message, actual, expected); + exit(1); + } +} + +static uint32_t pixel_value(int x, int y) +{ + return (uint32_t)(0x01000000u | ((uint32_t)y << 8) | (uint32_t)x); +} + +int main(void) +{ + rfbClient *client = rfbGetClient(8, 3, 4); + uint32_t raw[6 * 5]; + uint32_t *fb; + int x, y; + + if (!client) + fail("rfbGetClient failed"); + + client->width = 100; + client->height = 80; + + if (rfbClientSetFrameBufferViewport(client, -1, 20, 4, 3)) + fail("negative viewport x was accepted"); + if (rfbClientSetFrameBufferViewport(client, 10, 20, 0, 3)) + fail("zero viewport width was accepted"); + if (rfbClientSetFrameBufferViewport(client, 98, 20, 4, 3)) + fail("out-of-bounds viewport was accepted"); + + if (!rfbClientSetFrameBufferViewport(client, 10, 20, 4, 3)) + fail("rfbClientSetFrameBufferViewport failed"); + + if (!client->useFrameBufferViewport) + fail("viewport flag was not enabled"); + if (client->updateRect.x != 10 || client->updateRect.y != 20 || + client->updateRect.w != 4 || client->updateRect.h != 3) + fail("viewport did not update updateRect"); + + rfbClientClearFrameBufferViewport(client); + if (client->useFrameBufferViewport) + fail("pre-allocation clear did not disable viewport"); + if (client->updateRect.x != 0 || client->updateRect.y != 0 || + client->updateRect.w != client->width || client->updateRect.h != client->height) + fail("pre-allocation clear did not restore full updateRect"); + + if (!rfbClientSetFrameBufferViewport(client, 10, 20, 4, 3)) + fail("rfbClientSetFrameBufferViewport after clear failed"); + + if (!client->MallocFrameBuffer(client)) + fail("viewport framebuffer allocation failed"); + + if (rfbClientSetFrameBufferViewport(client, 0, 0, 2, 2)) + fail("post-allocation viewport change was accepted"); + + rfbClientClearFrameBufferViewport(client); + if (!client->useFrameBufferViewport) + fail("post-allocation clear disabled viewport unsafely"); + if (client->updateRect.x != 10 || client->updateRect.y != 20 || + client->updateRect.w != 4 || client->updateRect.h != 3) + fail("post-allocation clear changed updateRect unsafely"); + + fb = (uint32_t *)client->frameBuffer; + for (y = 0; y < 3; y++) + for (x = 0; x < 4; x++) + fb[y * 4 + x] = 0; + + for (y = 0; y < 5; y++) + for (x = 0; x < 6; x++) + raw[y * 6 + x] = pixel_value(8 + x, 19 + y); + + client->GotBitmap(client, (const uint8_t *)raw, 8, 19, 6, 5); + + assert_u32(fb[0], pixel_value(10, 20), "top-left clipped raw pixel"); + assert_u32(fb[3], pixel_value(13, 20), "top-right clipped raw pixel"); + assert_u32(fb[8], pixel_value(10, 22), "bottom-left clipped raw pixel"); + assert_u32(fb[11], pixel_value(13, 22), "bottom-right clipped raw pixel"); + + client->GotFillRect(client, 12, 21, 10, 10, 0xdeadbeefu); + + assert_u32(fb[1 * 4 + 1], pixel_value(11, 21), "pixel before clipped fill"); + assert_u32(fb[1 * 4 + 2], 0xdeadbeefu, "first clipped fill pixel"); + assert_u32(fb[1 * 4 + 3], 0xdeadbeefu, "second clipped fill pixel"); + assert_u32(fb[2 * 4 + 2], 0xdeadbeefu, "bottom clipped fill pixel"); + + free(client->frameBuffer); + client->frameBuffer = NULL; + rfbClientCleanup(client); + return 0; +}