Implement VideoToolbox-powered H.264 encoder for macOS

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "trans.h": "c"
+    }
+}

Makefile.am

@@ -59,7 +59,8 @@ SUBDIRS = \
   pkgconfig \
   $(XRDPVRDIR) \
   tests \
-  tools
+  tools \
+  xorgxrdp_helper
 
 distclean-local:
 	-rm -f xrdp_configure_options.h

common/ms-rdpbcgr.h

@@ -94,10 +94,10 @@
 /* the maximum height of the virtual desktop resulting from the union of the monitors  */
 /* contained in the monitorDefArray field MUST NOT exceed 32,766 pixels. */
 /* The minimum permitted size of the virtual desktop is 200 x 200 pixels. */
-#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_WIDTH  0xC8
-#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_HEIGHT 0xC8
-#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_WIDTH  0x7FFE
-#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_HEIGHT 0x7FFE
+#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_WIDTH  0xC8   // 200
+#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_HEIGHT 0xC8   // 200
+#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_WIDTH  0x7FFE // 32766
+#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_HEIGHT 0x7FFE // 32766
 
 /* 2.2.1.3.6.1 Monitor Definition (TS_MONITOR_DEF) */
 #define TS_MONITOR_PRIMARY 0x00000001

common/ms-rdpedisp.h

@@ -38,4 +38,7 @@
 #define ORIENTATION_LANDSCAPE_FLIPPED                      180
 #define ORIENTATION_PORTRAIT_FLIPPED                       270
 
+/* Display Control Monitor Layout (2.2.2.2.1) */
+#define DISPLAYCONTROL_MONITOR_PRIMARY         0x00000001
+
 #endif /* MS_RDPEDISP_H */

common/pixman-region.h

@@ -68,9 +68,11 @@ pixman_bool_t           pixman_region_subtract           (pixman_region16_t *reg
         /**/                                              pixman_region16_t *reg_m,
         /**/                                              pixman_region16_t *reg_s);
 pixman_bool_t           pixman_region_intersect          (pixman_region16_t *new_reg,
-        /**/                                              pixman_region16_t *reg1,
-        /**/                                              pixman_region16_t *reg2);
+        pixman_region16_t *reg1,
+        pixman_region16_t *reg2);
 pixman_box16_t         *pixman_region_rectangles         (pixman_region16_t *region,
-        /**/                                              int               *n_rects);
+        int *n_rects);
+pixman_bool_t           pixman_region_not_empty          (pixman_region16_t *region);
+pixman_box16_t         *pixman_region_extents            (pixman_region16_t *region);
 
 #endif

common/trans.c

@@ -1065,3 +1065,23 @@ trans_shutdown_tls_mode(struct trans *self)
 
     return 0;
 }
+
+/******************************************************************************/
+int
+trans_use_helper()
+{
+    const char *xrdp_use_helper = getenv("XRDP_USE_HELPER");
+    if (xrdp_use_helper == NULL)
+    {
+        return 0;
+    }
+    if (g_strcmp(xrdp_use_helper, "0") == 0)
+    {
+        return 0;
+    }
+    if (g_strcmp(xrdp_use_helper, "1") == 0)
+    {
+        return 1;
+    }
+    return 0;
+}

common/trans.h

@@ -57,6 +57,8 @@ enum xrdp_source
     XRDP_SOURCE_SESMAN,
     XRDP_SOURCE_CHANSRV,
     XRDP_SOURCE_MOD,
+    XORGXRDP_SOURCE_XORG,
+    XORGXRDP_SOURCE_XRDP,
 
     XRDP_SOURCE_MAX_COUNT
 };
@@ -179,5 +181,7 @@ int
 trans_shutdown_tls_mode(struct trans *self);
 int
 trans_tcp_force_read_s(struct trans *self, struct stream *in_s, int size);
+int
+trans_use_helper();
 
 #endif

common/xrdp_client_info.h

@@ -66,6 +66,58 @@ struct display_size_description
     unsigned int session_height;
 };
 
+enum display_resize_state
+{
+    WMRZ_QUEUED = 0,
+    WMRZ_ENCODER_DELETE,
+    WMRZ_EGFX_DELETE_SURFACE,
+    WMRZ_EGFX_CONN_CLOSE,
+    WMRZ_EGFX_CONN_CLOSING,
+    WMRZ_EGFX_CONN_CLOSED,
+    WRMZ_EGFX_DELETE,
+    WMRZ_SERVER_MONITOR_RESIZE,
+    WMRZ_SERVER_VERSION_MESSAGE_START,
+    WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING,
+    WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED,
+    WMRZ_XRDP_CORE_RESIZE,
+    WMRZ_EGFX_INITIALIZE,
+    WMRZ_EGFX_INITALIZING,
+    WMRZ_EGFX_INITIALIZED,
+    WMRZ_SERVER_INVALIDATE,
+    WMRZ_COMPLETE,
+    WMRZ_ERROR
+};
+
+struct dynamic_monitor_description
+{
+    struct display_size_description description;
+    enum display_resize_state state;
+    int last_state_update_timestamp;
+    int start_time;
+};
+
+#define XRDP_DISPLAY_RESIZE_STATE_TO_STR(status) \
+    ((status) == WMRZ_QUEUED ? "WMRZ_QUEUED" : \
+     (status) == WMRZ_ENCODER_DELETE ? "WMRZ_ENCODER_DELETE" : \
+     (status) == WMRZ_EGFX_DELETE_SURFACE ? "EGFX_DELETE_SURFACE" : \
+     (status) == WMRZ_EGFX_CONN_CLOSE ? "EGFX_CONN_CLOSE" : \
+     (status) == WMRZ_EGFX_CONN_CLOSING ? "EGFX_CONN_CLOSING" : \
+     (status) == WMRZ_EGFX_CONN_CLOSED ? "EGFX_CONN_CLOSED" : \
+     (status) == WRMZ_EGFX_DELETE ? "EGFX_DELETE" : \
+     (status) == WMRZ_SERVER_MONITOR_RESIZE ? "SERVER_MONITOR_RESIZE" : \
+     (status) == WMRZ_SERVER_VERSION_MESSAGE_START ? "SERVER_VERSION_MESSAGE_START" : \
+     (status) == WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING ? "SERVER_MONITOR_MESSAGE_PROCESSING" : \
+     (status) == WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED ? "SERVER_MONITOR_MESSAGE_PROCESSED" : \
+     (status) == WMRZ_XRDP_CORE_RESIZE ? "XRDP_CORE_RESIZE" : \
+     (status) == WMRZ_EGFX_INITIALIZE ? "EGFX_INITIALIZE" : \
+     (status) == WMRZ_EGFX_INITALIZING ? "EGFX_INITALIZING" : \
+     (status) == WMRZ_EGFX_INITIALIZED ? "EGFX_INITIALIZED" : \
+     (status) == WMRZ_SERVER_INVALIDATE ? "SERVER_INVALIDATE" : \
+     (status) == WMRZ_COMPLETE ? "COMPLETE" : \
+     (status) == WMRZ_ERROR ? "ERROR" : \
+     "unknown" \
+    )
+
 /**
  * Information about the xrdp client
  *
@@ -204,6 +256,7 @@ struct xrdp_client_info
 
     /* xrdp.override_* values */
     struct xrdp_keyboard_overrides xrdp_keyboard_overrides;
+    int gfx;
 };
 
 /* yyyymmdd of last incompatible change to xrdp_client_info */

configure.ac

@@ -161,6 +161,27 @@ AC_ARG_ENABLE(pixman, AS_HELP_STRING([--enable-pixman],
               [], [enable_pixman=no])
 AM_CONDITIONAL(XRDP_PIXMAN, [test x$enable_pixman = xyes])
 
+#example XRDP_NVENC_CFLAGS="-I/usr/local/cuda-11.3/include -I$HOME/Video_Codec_SDK_11.0.10/Interface" XRDP_NVENC_LIBS="-lcuda -lnvidia-encode" ./configure --enable-nvenc
+AC_ARG_ENABLE(nvenc, AS_HELP_STRING([--enable-nvenc],
+              [Use nvenc library (default: no), env vars XRDP_NVENC_CFLAGS and
+               XRDP_NVENC_LIBS should be set if used]),
+              [], [enable_nvenc=no])
+AM_CONDITIONAL(XRDP_NVENC, [test x$enable_nvenc = xyes])
+
+AC_ARG_ENABLE(nvenc, AS_HELP_STRING([--enable-videotoolbox],
+              [Use macOS VideoToolbox (default: no)]),
+              [], [enable_videotoolbox=no])
+AM_CONDITIONAL(XRDP_VIDEOTOOLBOX, [test x$enable_videotoolbox = xyes])
+
+AC_ARG_ENABLE(yami, AS_HELP_STRING([--enable-yami],
+              [Use yami library (default: no)]),
+              [], [enable_yami=no])
+AM_CONDITIONAL(XRDP_YAMI, [test x$enable_yami = xyes])
+AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264],
+              [Use x264 library (default: no)]),
+              [], [enable_x264=no])
+AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes])
+
 AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter],
               [Do not use included painter library (default: no)]),
               [], [enable_painter=yes])
@@ -393,6 +414,32 @@ fi
 
 AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >= 0.1.0)] )
 
+AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] )
+
+if test "x$enable_nvenc" = "xyes"
+then
+if test ! -z "$XRDP_NVENC_CFLAGS"
+then
+  AC_SUBST(XRDP_NVENC_CFLAGS, ["$XRDP_NVENC_CFLAGS"])
+fi
+if test ! -z "$XRDP_NVENC_LIBS"
+then
+  AC_SUBST(XRDP_NVENC_LIBS, ["$XRDP_NVENC_LIBS"])
+fi
+fi
+
+if test "x$enable_yami" = "xyes"
+then
+if test ! -z "$XRDP_YAMI_CFLAGS"
+then
+  AC_SUBST(XRDP_YAMI_CFLAGS, ["$XRDP_YAMI_CFLAGS"])
+fi
+if test ! -z "$XRDP_YAMI_LIBS"
+then
+  AC_SUBST(XRDP_YAMI_LIBS, ["$XRDP_YAMI_LIBS"])
+fi
+fi
+
 # checking for TurboJPEG
 if test "x$enable_tjpeg" = "xyes"
 then
@@ -515,6 +562,7 @@ AC_CONFIG_FILES([
   xrdp/Makefile
   xrdpvr/Makefile
   xup/Makefile
+  xorgxrdp_helper/Makefile
 ])
 
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
@@ -529,6 +577,9 @@ echo "  fdkaac                  $enable_fdkaac"
 echo "  jpeg                    $enable_jpeg"
 echo "  turbo jpeg              $enable_tjpeg"
 echo "  rfxcodec                $enable_rfxcodec"
+echo "  nvenc                   $enable_nvenc"
+echo "  yami                    $enable_yami"
+echo "  x264                    $enable_x264"
 echo "  painter                 $enable_painter"
 echo "  pixman                  $enable_pixman"
 echo "  fuse                    $enable_fuse"

libxrdp/libxrdp.c

@@ -1374,6 +1374,10 @@ libxrdp_send_to_channel(struct xrdp_session *session, int channel_id,
         free_stream(s);
         return 1;
     }
+    else
+    {
+        LOG(LOG_LEVEL_TRACE, "libxrdp_send_to_channel: xrdp_channel_init successful!");
+    }
 
     /* here we make a copy of the data */
     out_uint8a(s, data, data_len);
@@ -2136,3 +2140,15 @@ libxrdp_process_monitor_stream(struct stream *s,
     }
     return 0;
 }
+int EXPORT_CC
+libxrdp_planar_compress(char *in_data, int width, int height,
+                        struct stream *s, int bpp, int byte_limit,
+                        int start_line, struct stream *temp_s,
+                        int e, int flags)
+{
+    return xrdp_bitmap32_compress(in_data, width, height,
+                                  s, bpp, byte_limit,
+                                  start_line, temp_s,
+                                  e, flags);
+}
+

libxrdp/libxrdpinc.h

@@ -306,6 +306,11 @@ libxrdp_fastpath_send_frame_marker(struct xrdp_session *session,
 int EXPORT_CC
 libxrdp_send_session_info(struct xrdp_session *session, const char *data,
                           int data_bytes);
+int EXPORT_CC
+libxrdp_planar_compress(char *in_data, int width, int height,
+                        struct stream *s, int bpp, int byte_limit,
+                        int start_line, struct stream *temp_s,
+                        int e, int flags);
 
 /**
  * Processes a stream that is based on either

libxrdp/xrdp_sec.c

@@ -24,8 +24,11 @@
 
 #include "libxrdp.h"
 #include "ms-rdpbcgr.h"
+#include "ms-rdpedisp.h"
 #include "log.h"
 #include "string_calls.h"
+#include <limits.h>
+
 
 /* some compilers need unsigned char to avoid warnings */
 static tui8 g_pad_54[40] =
@@ -2073,6 +2076,15 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s)
     {
         self->rdp_layer->client_info.bpp = 32;
     }
+    if (earlyCapabilityFlags & 0x100) /* RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL */
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client supports gfx");
+        self->rdp_layer->client_info.gfx = 1;
+    }
+    else
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client DOES NOT support gfx");
+    }
 
     if (!s_check_rem(s, 64))
     {

scripts/install_xrdp_build_dependencies_with_apt.sh

@@ -58,7 +58,7 @@ PACKAGES=" \
     make \
     nasm \
     pkg-config \
-    check \
+    check
     "
 
 case "$ARCH"
@@ -69,7 +69,9 @@ in
             libssl-dev \
             libx11-dev \
             libxrandr-dev \
-            libxfixes-dev"
+            libxfixes-dev \
+            libepoxy-dev \
+            libepoxy0"
 
         case "$FEATURE_SET"
         in
@@ -111,7 +113,9 @@ in
             libxfixes-dev:i386 \
             libxrandr-dev:i386 \
             libxrender-dev:i386 \
-            libfuse-dev:i386"
+            libfuse-dev:i386 \
+            libepoxy-dev:i386 \
+            libepoxy0:i386"
 
         dpkg --add-architecture i386
         dpkg --print-architecture

sesman/sesman.ini.in

@@ -113,7 +113,7 @@ EnableSyslog=true
 param=Xorg
 ; Leave the rest paramaters as-is unless you understand what will happen.
 param=-config
-param=xrdp/xorg.conf
+param=xrdp/xorg_nvidia.conf
 param=-noreset
 param=-nolisten
 param=tcp
@@ -166,3 +166,4 @@ EnableSyslog=true
 
 [SessionVariables]
 PULSE_SCRIPT=@sesmansysconfdir@/pulse/default.pa
+XRDP_USE_HELPER=1

xorgxrdp_helper/Makefile.am

@@ -0,0 +1,35 @@
+AM_CPPFLAGS = \
+  -I$(top_srcdir)/common
+
+XRDP_EXTRA_LIBS =
+XRDP_EXTRA_SOURCES =
+
+if XRDP_NVENC
+AM_CPPFLAGS += -DXRDP_NVENC
+AM_CPPFLAGS += $(XRDP_NVENC_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_NVENC_LIBS)
+XRDP_EXTRA_SOURCES += xorgxrdp_helper_nvenc.c xorgxrdp_helper_nvenc.h
+endif
+
+if XRDP_YAMI
+AM_CPPFLAGS += -DXRDP_YAMI
+AM_CPPFLAGS += $(XRDP_YAMI_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_YAMI_LIBS)
+XRDP_EXTRA_SOURCES += xorgxrdp_helper_yami.c xorgxrdp_helper_yami.h
+endif
+
+bin_PROGRAMS = \
+  xorgxrdp_helper
+
+xorgxrdp_helper_SOURCES = \
+  xorgxrdp_helper.c \
+  xorgxrdp_helper.h \
+  xorgxrdp_helper_x11.c \
+  xorgxrdp_helper_x11.h \
+  $(XRDP_EXTRA_SOURCES)
+
+xorgxrdp_helper_LDADD = \
+  $(top_builddir)/common/libcommon.la \
+  $(XRDP_EXTRA_LIBS) \
+  -lX11 -lepoxy
+