move to posix shm

neutrinolabs · Jun 2, 2023 · a243f30 · a243f30
1 parent bf49ee1
commit a243f30
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 21 deletions.
diff --git a/module/rdp.h b/module/rdp.h
@@ -124,7 +124,8 @@ struct image_data
     int lineBytes;
     uint8_t *pixels;
     uint8_t *shmem_pixels;
-    int shmem_id;
+    int shmem_fd;
+    int shmem_bytes;
     int shmem_offset;
     int shmem_lineBytes;
 };

diff --git a/module/rdpClientCon.c b/module/rdpClientCon.c
@@ -687,25 +687,38 @@ rdpClientConProcessMsgVersion(rdpPtr dev, rdpClientCon *clientCon,
 static void
 rdpClientConAllocateSharedMemory(rdpClientCon *clientCon, int bytes)
 {
+    void *shmemptr;
+    int shmemfd;
+
     if (clientCon->shmemptr != NULL && clientCon->shmem_bytes == bytes)
     {
-        LLOGLN(0, ("rdpClientConAllocateSharedMemory: reusing shmemid %d",
-               clientCon->shmemid));
+        LLOGLN(0, ("rdpClientConAllocateSharedMemory: reusing shmemfd %d",
+               clientCon->shmemfd));
         return;
     }
-
-    if (clientCon->shmemptr != 0)
+    if (clientCon->shmemfd != -1)
     {
-        shmdt(clientCon->shmemptr);
+        g_free_unmap_fd(clientCon->shmemptr,
+                        clientCon->shmemfd,
+                        clientCon->shmem_bytes);
+    }
+    if (g_alloc_shm_map_fd(&shmemptr, &shmemfd, bytes) == 0)
+    {
+        clientCon->shmemptr = shmemptr;
+        clientCon->shmemfd = shmemfd;
+        clientCon->shmem_bytes = bytes;
+        LLOGLN(0, ("rdpClientConAllocateSharedMemory: shmemfd %d shmemptr %p "
+               "bytes %d",
+               clientCon->shmemfd, clientCon->shmemptr,
+               clientCon->shmem_bytes));
+    }
+    else
+    {
+        LLOGLN(0, ("rdpClientConAllocateSharedMemory: g_alloc_shm_map_fd "
+               "failed"));
     }
-    clientCon->shmemid = shmget(IPC_PRIVATE, bytes, IPC_CREAT | 0777);
-    clientCon->shmemptr = shmat(clientCon->shmemid, 0, 0);
-    clientCon->shmem_bytes = bytes;
-    shmctl(clientCon->shmemid, IPC_RMID, NULL);
-    LLOGLN(0, ("rdpClientConAllocateSharedMemory: shmemid %d shmemptr %p bytes %d",
-           clientCon->shmemid, clientCon->shmemptr,
-           clientCon->shmem_bytes));
 }
+
 /******************************************************************************/
 /*
     this from miScreenInit
@@ -2380,7 +2393,7 @@ rdpClientConCheckDirtyScreen(rdpPtr dev, rdpClientCon *clientCon)
 
 /******************************************************************************/
 static int
-rdpClientConSendPaintRectShmEx(rdpPtr dev, rdpClientCon *clientCon,
+rdpClientConSendPaintRectShmFd(rdpPtr dev, rdpClientCon *clientCon,
                                struct image_data *id,
                                RegionPtr dirtyReg,
                                BoxPtr copyRects, int numCopyRects)
@@ -2402,15 +2415,15 @@ rdpClientConSendPaintRectShmEx(rdpPtr dev, rdpClientCon *clientCon,
     num_rects_c = numCopyRects;
     if ((num_rects_c < 1) || (num_rects_d < 1))
     {
-        LLOGLN(10, ("rdpClientConSendPaintRectShmEx: nothing to send"));
+        LLOGLN(10, ("rdpClientConSendPaintRectShmFd: nothing to send"));
         return 0;
     }
     size = 2 + 2 + 2 + num_rects_d * 8 + 2 + num_rects_c * 8;
-    size += 4 + 4 + 4 + 4 + 2 + 2;
+    size += 4 + 4 + 4 + 4 + 2 + 2 + 2 + 2;
     rdpClientConPreCheck(dev, clientCon, size);
 
     s = clientCon->out_s;
-    out_uint16_le(s, 61);
+    out_uint16_le(s, 64);
     out_uint16_le(s, size);
     clientCon->count++;
 
@@ -2445,11 +2458,16 @@ rdpClientConSendPaintRectShmEx(rdpPtr dev, rdpClientCon *clientCon,
     out_uint32_le(s, 0);
     clientCon->rect_id++;
     out_uint32_le(s, clientCon->rect_id);
-    out_uint32_le(s, id->shmem_id);
+    out_uint32_le(s, id->shmem_bytes);
     out_uint32_le(s, id->shmem_offset);
+    out_uint16_le(s, clientCon->cap_left);
+    out_uint16_le(s, clientCon->cap_top);
     out_uint16_le(s, clientCon->cap_width);
     out_uint16_le(s, clientCon->cap_height);
 
+    rdpClientConSendPending(clientCon->dev, clientCon);
+    g_sck_send_fd_set(clientCon->sck, "int", 4, &(id->shmem_fd), 1);
+
     rdpClientConEndUpdate(dev, clientCon);
 
     return 0;
@@ -2492,7 +2510,7 @@ rdpCapRect(rdpClientCon *clientCon, BoxPtr cap_rect, struct image_data *id)
         if (rdpCapture(clientCon, cap_dirty, &rects, &num_rects, id))
         {
             LLOGLN(10, ("rdpCapRect: num_rects %d", num_rects));
-            rdpClientConSendPaintRectShmEx(clientCon->dev, clientCon, id,
+            rdpClientConSendPaintRectShmFd(clientCon->dev, clientCon, id,
                                            cap_dirty, rects, num_rects);
             free(rects);
         }
@@ -2739,7 +2757,8 @@ rdpClientConGetScreenImageRect(rdpPtr dev, rdpClientCon *clientCon,
     id->lineBytes = dev->paddedWidthInBytes;
     id->pixels = dev->pfbMemory;
     id->shmem_pixels = clientCon->shmemptr;
-    id->shmem_id = clientCon->shmemid;
+    id->shmem_fd = clientCon->shmemfd;
+    id->shmem_bytes = clientCon->shmem_bytes;
     id->shmem_offset = 0;
     id->shmem_lineBytes = clientCon->shmem_lineBytes;
 }

diff --git a/module/rdpClientCon.h b/module/rdpClientCon.h
@@ -104,7 +104,7 @@ struct _rdpClientCon
     struct xrdp_client_info client_info;
 
     uint8_t *shmemptr;
-    int shmemid;
+    int shmemfd;
     int shmem_bytes;
     int shmem_lineBytes;
     RegionPtr shmRegion;