From 984b71449ebc6c09e909b7078d36769e46f87ca0 Mon Sep 17 00:00:00 2001
From: matt335672 <30179339+matt335672@users.noreply.github.com>
Date: Mon, 9 Sep 2024 15:07:31 +0100
Subject: [PATCH] Regression: Support fixed-size VNC sessions

This is a regression introduced in v0.10.x

This version introduced a state machine to handle resizes requested
by the client and the server. Most configurations support resizeable
sessions, but one that doesn't is xrdp connecting to x11vnc on (e.g.) a
Raspberry PI.

If the session size requested by a client is differnt from the x11vnc
size, an error is logged and the state machine fails to complete,
resulting in a black screen.

This PR handles the problem by queueing a resize to the supported
server size and then continuing with the state machine. It's not an
optimal solution, but involves the least change to v0.10.x code.
---
 vnc/vnc.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/vnc/vnc.c b/vnc/vnc.c
index 13d71bb43..d433ee282 100644
--- a/vnc/vnc.c
+++ b/vnc/vnc.c
@@ -1248,10 +1248,6 @@ lib_framebuffer_waiting_for_resize_confirm(struct vnc *v)
                 LOG(LOG_LEVEL_DEBUG, "VNC server successfully resized");
                 log_screen_layout(LOG_LEVEL_INFO, "NewLayout", &layout);
                 v->server_layout = layout;
-                // If this resize was requested by the client mid-session
-                // (dynamic resize), we need to tell xrdp_mm that
-                // it's OK to continue with the resize state machine.
-                error = v->server_monitor_resize_done(v);
             }
             else
             {
@@ -1259,10 +1255,23 @@ lib_framebuffer_waiting_for_resize_confirm(struct vnc *v)
                     "VNC server resize failed - error code %d [%s]",
                     response_code,
                     rfb_get_eds_status_msg(response_code));
-                /* Force client to same size as server */
+                // This is awkward. The client has asked for a specific size
+                // which we can't support.
+                //
+                // Currently we handle this by queueing a resize to our
+                // supported size, and continuing with the resize state
+                // machine in xrdp_mm.c
                 LOG(LOG_LEVEL_WARNING, "Resizing client to server");
                 error = resize_client_to_server(v, 0);
             }
+
+            if (error == 0)
+            {
+                // If this resize was requested by the client mid-session
+                // (dynamic resize), we need to tell xrdp_mm that
+                // it's OK to continue with the resize state machine.
+                error = v->server_monitor_resize_done(v);
+            }
             v->resize_status = VRS_DONE;
         }
     }