add support for nvenc and accel_assist

configure.ac

@@ -120,6 +120,11 @@ AC_ARG_ENABLE(glamor, AS_HELP_STRING([--enable-glamor],
               [], [enable_glamor=no])
 AM_CONDITIONAL(WITH_GLAMOR, [test x$enable_glamor = xyes])
 
+AC_ARG_ENABLE(lrandr, AS_HELP_STRING([--enable-lrandr],
+              [Use local randr (default: no)]),
+              [], [enable_lrandr=no])
+AM_CONDITIONAL(WITH_LRANDR, [test x$enable_lrandr = xyes])
+
 AM_CONDITIONAL(WITH_SIMD_AMD64, [test x$simd_arch = xx86_64])
 AM_CONDITIONAL(WITH_SIMD_X86, [test x$simd_arch = xi386])
 

module/Makefile.am

@@ -27,7 +27,14 @@ EGLLIB += -L$(XORG_SERVER_MODULES) -lglamoregl -lepoxy
 LIB_SEARCH_PATH += -R$(XORG_SERVER_MODULES)
 endif
 
+if WITH_LRANDR
+EXTRA_FLAGS += -DXORGXRDP_LRANDR
+EXTRA_SOURCES += rdpLRandR.c
+EXTRA_HEADERS += rdpLRandR.h
+endif
+
 AM_CFLAGS = \
+  -DXRDP_LIBEXEC_PATH=\"${libexecdir}/xrdp\" \
   $(XORG_SERVER_CFLAGS) \
   $(XRDP_CFLAGS) \
   -DPACKAGE_VERSION_MAJOR=@package_version_major@ \

module/rdp.h

@@ -316,6 +316,7 @@ struct _rdpRec
     int monitorCount;
     /* glamor */
     Bool glamor;
+    Bool nvidia;
     PixmapPtr screenSwPixmap;
     void *xvPutImage;
     /* dri */
@@ -326,7 +327,7 @@ struct _rdpRec
 };
 typedef struct _rdpRec rdpRec;
 typedef struct _rdpRec * rdpPtr;
-#define XRDPPTR(_p) ((rdpPtr)((_p)->driverPrivate))
+#define XRDPPTR(_p) ((rdpPtr)((_p)->reservedPtr[0]))
 
 struct _rdpGCRec
 {

module/rdpCapture.c

@@ -725,6 +725,64 @@ rdpCopyBox_a8r8g8b8_to_nv12_709fr(rdpClientCon *clientCon,
     return 0;
 }
 
+/******************************************************************************/
+static Bool
+rdpCopyBoxList(rdpClientCon *clientCon, PixmapPtr dstPixmap,
+               BoxPtr out_rects, int num_out_rects,
+               int srcx, int srcy,
+               int dstx, int dsty)
+{
+    PixmapPtr hwPixmap;
+    BoxPtr pbox;
+    ScreenPtr pScreen;
+    GCPtr copyGC;
+    ChangeGCVal tmpval[1];
+    int count;
+    int index;
+    int left;
+    int top;
+    int width;
+    int height;
+    char pix1[16];
+    rdpPtr dev;
+
+    LLOGLN(10, ("rdpCopyBoxList:"));
+
+    dev = clientCon->dev;
+    pScreen = dev->pScreen;
+    hwPixmap = pScreen->GetScreenPixmap(pScreen);
+    copyGC = GetScratchGC(dev->depth, pScreen);
+    if (copyGC == NULL)
+    {
+        return FALSE;
+    }
+    tmpval[0].val = GXcopy;
+    ChangeGC(NullClient, copyGC, GCFunction, tmpval);
+    ValidateGC(&(hwPixmap->drawable), copyGC);
+    count = num_out_rects;
+    pbox = out_rects;
+    for (index = 0; index < count; index++)
+    {
+        left = pbox[index].x1;
+        top = pbox[index].y1;
+        width = pbox[index].x2 - pbox[index].x1;
+        height = pbox[index].y2 - pbox[index].y1;
+        if ((width > 0) && (height > 0))
+        {
+            copyGC->ops->CopyArea(&(hwPixmap->drawable),
+                                    &(dstPixmap->drawable), copyGC,
+                                    left - srcx, top - srcy,
+                                    width, height,
+                                    left - dstx, top - dsty);
+        }
+    }
+    FreeScratchGC(copyGC);
+    pScreen->GetImage(&(dstPixmap->drawable), 0, 0, 1, 1, ZPixmap,
+                          0xffffffff, pix1);
+
+    return TRUE;
+}
+
 /******************************************************************************/
 static Bool
 isShmStatusActive(enum shared_memory_status status) {
@@ -800,6 +858,17 @@ rdpCaptureSimple(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
         (*out_rects)[i] = rect;
     }
 
+    if (clientCon->dev->glamor || clientCon->dev->nvidia)
+    {
+        /* copy vmem to smem */
+        if (!rdpCopyBoxList(clientCon, clientCon->dev->screenSwPixmap,
+                            *out_rects, *num_out_rects,
+                            0, 0, 0, 0))
+        {
+            return FALSE;
+        }
+    }
+
     src = id->pixels;
     dst = id->shmem_pixels;
     dst_format = clientCon->rdp_format;
@@ -947,6 +1016,17 @@ rdpCaptureSufA16(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
         index++;
     }
 
+    if (clientCon->dev->glamor || clientCon->dev->nvidia)
+    {
+        /* copy vmem to smem */
+        if (!rdpCopyBoxList(clientCon, clientCon->dev->screenSwPixmap,
+                            *out_rects, *num_out_rects,
+                            0, 0, 0, 0))
+        {
+            return FALSE;
+        }
+    }
+
     src = id->pixels;
     dst = id->shmem_pixels;
     dst_format = clientCon->rdp_format;
@@ -1031,6 +1111,17 @@ rdpCaptureGfxPro(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
         return FALSE;
     }
 
+    if (clientCon->dev->glamor || clientCon->dev->nvidia)
+    {
+        /* copy vmem to smem */
+        if (!rdpCopyBoxList(clientCon, clientCon->dev->screenSwPixmap,
+                            REGION_RECTS(in_reg), REGION_NUM_RECTS(in_reg),
+                            0, 0, 0, 0))
+        {
+            return FALSE;
+        }
+    }
+
     *out_rects = g_new(BoxRec, RDP_MAX_TILES);
     if (*out_rects == NULL)
     {
@@ -1164,6 +1255,7 @@ rdpCaptureSufA2(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
     int src_stride;
     int dst_stride;
     int dst_format;
+    int monitor_index;
 
     LLOGLN(10, ("rdpCaptureSufA2:"));
 
@@ -1184,6 +1276,26 @@ rdpCaptureSufA2(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
         return FALSE;
     }
 
+    monitor_index = (id->flags >> 28) & 0xF;
+    if (clientCon->accelAssistPixmaps[monitor_index] != NULL)
+    {
+        /* copy vmem to vmem */
+        rv = rdpCopyBoxList(clientCon,
+                            clientCon->accelAssistPixmaps[monitor_index],
+                            *out_rects, *num_out_rects,
+                            0, 0, id->left, id->top);
+        id->flags |= 1;
+        return rv;
+        /* accel assist will do the rest */
+    }
+    else if (clientCon->dev->glamor || clientCon->dev->nvidia)
+    {
+        /* copy vmem to smem */
+        rv = rdpCopyBoxList(clientCon, clientCon->dev->screenSwPixmap,
+                            *out_rects, *num_out_rects,
+                            0, 0, id->left, id->top);
+    }
+
     *num_out_rects = num_rects;
 
     *out_rects = g_new(BoxRec, num_rects * 4);
@@ -1252,6 +1364,7 @@ rdpCaptureGfxA2(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
     int src_stride;
     int dst_stride;
     int dst_format;
+    int monitor_index;
 
     LLOGLN(10, ("rdpCaptureGfxA2:"));
 
@@ -1262,8 +1375,6 @@ rdpCaptureGfxA2(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
         return FALSE;
     }
 
-    rv = TRUE;
-
     rdpRegionTranslate(in_reg, -id->left, -id->top);
 
     num_rects = REGION_NUM_RECTS(in_reg);
@@ -1300,21 +1411,48 @@ rdpCaptureGfxA2(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
         (*out_rects)[index] = rect;
         index++;
     }
+    rv = TRUE;
+    monitor_index = (id->flags >> 28) & 0xF;
+    if (clientCon->accelAssistPixmaps[monitor_index] != NULL)
+    {
+        LLOGLN(10, ("rdpCaptureGfxA2: a monitor_index %d left %d top %d",
+               monitor_index, id->left, id->top));
+        /* copy vmem to vmem */
+        rv = rdpCopyBoxList(clientCon,
+                            clientCon->accelAssistPixmaps[monitor_index],
+                            *out_rects, num_rects,
+                            -id->left, -id->top, 0, 0);
+        id->flags |= 1;
+        return rv;
+        /* accel assist will do the rest */
+    }
+    else if (clientCon->dev->glamor || clientCon->dev->nvidia)
+    {
+        LLOGLN(10, ("rdpCaptureGfxA2: b monitor_index %d left %d top %d",
+               monitor_index, id->left, id->top));
+        /* copy vmem to smem */
+        if (!rdpCopyBoxList(clientCon,
+                            clientCon->dev->screenSwPixmap,
+                            *out_rects, num_rects,
+                            -id->left, -id->top, -id->left, -id->top))
+        {
+            return FALSE;
+        }
+    }
 
     src = id->pixels;
     dst = id->shmem_pixels;
     dst_format = clientCon->rdp_format;
     src_stride = id->lineBytes;
     dst_stride = id->width;
 
-    src = src + src_stride * id->top + id->left * 4;
-
     if (dst_format == XRDP_nv12_709fr)
     {
         dst_uv = dst;
         dst_uv += id->width * id->height;
         rdpCopyBox_a8r8g8b8_to_nv12_709fr(clientCon,
-                                          src, src_stride, 0, 0,
+                                          src, src_stride,
+                                          -id->left, -id->top,
                                           dst, dst_stride,
                                           dst_uv, dst_stride,
                                           0, 0,
@@ -1328,65 +1466,6 @@ rdpCaptureGfxA2(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
     return rv;
 }
 
-#if defined(XORGXRDP_GLAMOR)
-/******************************************************************************/
-static int
-copy_vmem(rdpPtr dev, RegionPtr in_reg)
-{
-    PixmapPtr hwPixmap;
-    PixmapPtr swPixmap;
-    BoxPtr pbox;
-    ScreenPtr pScreen;
-    GCPtr copyGC;
-    ChangeGCVal tmpval[1];
-    int count;
-    int index;
-    int left;
-    int top;
-    int width;
-    int height;
-
-    /* copy the dirty area from the screen hw pixmap to a sw pixmap
-       this should do a dma */
-    pScreen = dev->pScreen;
-    hwPixmap = pScreen->GetScreenPixmap(pScreen);
-    swPixmap = dev->screenSwPixmap;
-    copyGC = GetScratchGC(dev->depth, pScreen);
-    if (copyGC != NULL)
-    {
-        tmpval[0].val = GXcopy;
-        ChangeGC(NullClient, copyGC, GCFunction, tmpval);
-        ValidateGC(&(hwPixmap->drawable), copyGC);
-        count = REGION_NUM_RECTS(in_reg);
-        pbox = REGION_RECTS(in_reg);
-        for (index = 0; index < count; index++)
-        {
-            left = pbox[index].x1;
-            top = pbox[index].y1;
-            width = pbox[index].x2 - pbox[index].x1;
-            height = pbox[index].y2 - pbox[index].y1;
-            if ((width > 0) && (height > 0))
-            {
-                LLOGLN(10, ("copy_vmem: hwPixmap tex 0x%8.8x "
-                       "swPixmap tex 0x%8.8x",
-                       glamor_get_pixmap_texture(hwPixmap),
-                       glamor_get_pixmap_texture(swPixmap)));
-                 copyGC->ops->CopyArea(&(hwPixmap->drawable),
-                                       &(swPixmap->drawable),
-                                       copyGC, left, top,
-                                       width, height, left, top);
-            }
-        }
-        FreeScratchGC(copyGC);
-    }
-    else
-    {
-        return 1;
-    }
-    return 0;
-}
-#endif
-
 /**
  * Copy an array of rectangles from one memory area to another
  *****************************************************************************/
@@ -1398,17 +1477,6 @@ rdpCapture(rdpClientCon *clientCon, RegionPtr in_reg, BoxPtr *out_rects,
 
     LLOGLN(10, ("rdpCapture:"));
     mode = clientCon->client_info.capture_code;
-    if (clientCon->dev->glamor)
-    {
-#if defined(XORGXRDP_GLAMOR)
-        if ((mode == 2) || (mode == 4))
-        {
-            return rdpEglCaptureRfx(clientCon, in_reg, out_rects,
-                                    num_out_rects, id);
-        }
-        copy_vmem(clientCon->dev, in_reg);
-#endif
-    }
     switch (mode)
     {
         case CC_SIMPLE: