From 21d8048b1a18ac4bec563b3fadce5e8c4e1b4ea3 Mon Sep 17 00:00:00 2001
From: matt335672 <30179339+matt335672@users.noreply.github.com>
Date: Sat, 30 Jul 2022 03:15:01 +0100
Subject: [PATCH] Ignore screen size changes which don't change anything (#203)

* Only reallocate shared memory if the size changes
Co-authored-by: Nexarian <cmp@pitstick.net>
---
 module/rdpClientCon.c | 61 ++++++++++++++++++++++++-------------------
 module/rdpClientCon.h |  1 +
 2 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/module/rdpClientCon.c b/module/rdpClientCon.c
index 9aea3d01..e607f4a1 100644
--- a/module/rdpClientCon.c
+++ b/module/rdpClientCon.c
@@ -675,6 +675,37 @@ rdpClientConProcessMsgVersion(rdpPtr dev, rdpClientCon *clientCon,
     return 0;
 }
 
+/**************************************************************************//**
+ * Allocate shared memory
+ *
+ * This memory is shared with the xup driver in xrdp which avoids a lot
+ * of unnecessary copying
+ *
+ * @param clientCon Client connection
+ * @param bytes Size of area to attach
+ */
+static void
+rdpClientConAllocateSharedMemory(rdpClientCon *clientCon, int bytes)
+{
+    if (clientCon->shmemptr != NULL && clientCon->shmem_bytes == bytes)
+    {
+        LLOGLN(0, ("rdpClientConAllocateSharedMemory: reusing shmemid %d",
+               clientCon->shmemid));
+        return;
+    }
+    
+    if (clientCon->shmemptr != 0)
+    {
+        shmdt(clientCon->shmemptr);
+    }
+    clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
+    clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
+    clientCon->shmem_bytes = bytes;
+    shmctl(clientCon->shmemid, IPC_RMID, NULL);
+    LLOGLN(0, ("rdpClientConAllocateSharedMemory: shmemid %d shmemptr %p bytes %d",
+           clientCon->shmemid, clientCon->shmemptr,
+           clientCon->shmem_bytes));
+}
 /******************************************************************************/
 /*
     this from miScreenInit
@@ -727,17 +758,9 @@ rdpClientConProcessScreenSizeMsg(rdpPtr dev, rdpClientCon *clientCon,
 
     clientCon->cap_stride_bytes = clientCon->rdp_width * clientCon->rdp_Bpp;
 
-    if (clientCon->shmemptr != 0)
-    {
-        shmdt(clientCon->shmemptr);
-    }
     bytes = clientCon->rdp_width * clientCon->rdp_height *
             clientCon->rdp_Bpp;
-    clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
-    clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
-    shmctl(clientCon->shmemid, IPC_RMID, NULL);
-    LLOGLN(0, ("rdpClientConProcessScreenSizeMsg: shmemid %d shmemptr %p",
-           clientCon->shmemid, clientCon->shmemptr));
+    rdpClientConAllocateSharedMemory(clientCon, bytes);
     clientCon->shmem_lineBytes = clientCon->rdp_Bpp * clientCon->rdp_width;
 
     if (clientCon->shmRegion != 0)
@@ -868,17 +891,9 @@ rdpClientConProcessMsgClientInfo(rdpPtr dev, rdpClientCon *clientCon)
         clientCon->cap_height = RDPALIGN(clientCon->rdp_height, 64);
         LLOGLN(0, ("  cap_width %d cap_height %d",
                clientCon->cap_width, clientCon->cap_height));
-        if (clientCon->shmemptr != 0)
-        {
-            shmdt(clientCon->shmemptr);
-        }
         bytes = clientCon->cap_width * clientCon->cap_height *
                 clientCon->rdp_Bpp;
-        clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
-        clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
-        shmctl(clientCon->shmemid, IPC_RMID, NULL);
-        LLOGLN(0, ("rdpClientConProcessMsgClientInfo: shmemid %d shmemptr %p "
-               "bytes %d", clientCon->shmemid, clientCon->shmemptr, bytes));
+        rdpClientConAllocateSharedMemory(clientCon, bytes);
         clientCon->shmem_lineBytes = clientCon->rdp_Bpp * clientCon->cap_width;
         clientCon->cap_stride_bytes = clientCon->cap_width * 4;
         shmemstatus = SHM_RFX_ACTIVE;
@@ -890,16 +905,8 @@ rdpClientConProcessMsgClientInfo(rdpPtr dev, rdpClientCon *clientCon)
         clientCon->cap_height = clientCon->rdp_height;
         LLOGLN(0, ("  cap_width %d cap_height %d",
                clientCon->cap_width, clientCon->cap_height));
-        if (clientCon->shmemptr != 0)
-        {
-            shmdt(clientCon->shmemptr);
-        }
         bytes = clientCon->cap_width * clientCon->cap_height * 2;
-        clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
-        clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
-        shmctl(clientCon->shmemid, IPC_RMID, NULL);
-        LLOGLN(0, ("rdpClientConProcessMsgClientInfo: shmemid %d shmemptr %p "
-               "bytes %d", clientCon->shmemid, clientCon->shmemptr, bytes));
+        rdpClientConAllocateSharedMemory(clientCon, bytes);
         clientCon->shmem_lineBytes = clientCon->rdp_Bpp * clientCon->cap_width;
         clientCon->cap_stride_bytes = clientCon->cap_width * 4;
         shmemstatus = SHM_H264_ACTIVE;
diff --git a/module/rdpClientCon.h b/module/rdpClientCon.h
index 42b39c64..da845a7b 100644
--- a/module/rdpClientCon.h
+++ b/module/rdpClientCon.h
@@ -107,6 +107,7 @@ struct _rdpClientCon
 
     uint8_t *shmemptr;
     int shmemid;
+    int shmem_bytes;
     int shmem_lineBytes;
     RegionPtr shmRegion;
     int rect_id;