Merge branch 'release/2024.1.0'

CHANGES.md

@@ -11,6 +11,20 @@
 
 ## develop
 
+## 2024.1.0 (2024-01-16)
+
+- [CHANGE] JetPack 5.1.2 に対応
+  - JetPack 5.1.1, 5.1.2 で動作を確認
+  - JetPack 5.1 では、互換性の問題で JetsonJpegDecoder がエラーになることを確認
+  - @enm10k
+- [ADD] ForwardingFilter の項目に version と metadata を追加
+  - version と metadata はオプション項目として追加し、値がない場合は項目を設定しない
+  - @torikizi
+- [FIX] DataChannel シグナリングが有効な場合、一部のシグナリングメッセージのコールバックが上がらないことがあるのを修正
+  - @melpon
+- [FIX] ForwardingFilter の action 項目は本来オプション項目だったが、 action は `std::optional` 型になっていなかったため修正
+  - @torikizi
+
 ## 2023.17.0 (2023-12-25)
 
 - [UPDATE] WebRTC を `m120.6099.1.2` に上げる

CMakeLists.txt

@@ -577,13 +577,18 @@ elseif (SORA_TARGET_OS STREQUAL "jetson")
       PUBLIC
         nvv4l2
         nvv4lconvert
-        nvbuf_utils
         #nvbuf_fdmap
         #nvddk_vic
         #nvddk_2d_v2
         nvjpeg
         nvbufsurface
         nvbufsurftransform
+
+        # nvbuf_utils を NvUtils に移行した際、ドキュメントには libnvbuf_utils.so を参照するように記載があったが、
+        # そのような so ファイルは存在しないためリンクが失敗した
+        # nvbuf_fdmap を追加したらリンクが通った
+        # https://developer.nvidia.com/sites/default/files/akamai/embedded/nvbuf_utils_to_nvutils_migration_guide.pdf
+        nvbuf_fdmap
         #nvos
     )
   endif(USE_JETSON_ENCODER)

README.md

@@ -27,7 +27,7 @@ Please read https://github.com/shiguredo/oss/blob/master/README.en.md before use
     - Apple macOS / iOS Video Toolbox
         - H.264 / H.265
     - Google Android HWA
-        - VP8 / VP9 / H.264
+        - VP8 / VP9 / H.264 / H.265
     - [Intel oneVPL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onevpl.html) (Intel Media SDK の後継)
         - VP9 / AV1 / H.264
 - 各プラットフォームへの [google/lyra: A Very Low\-Bitrate Codec for Speech Compression](https://github.com/google/lyra) 対応
@@ -48,7 +48,7 @@ https://github.com/shiguredo/sora-cpp-sdk/releases
 
 - Windows 10.1809 x86_64 以降
 - macOS 13.5 arm64 以降
-- Ubuntu 20.04 ARMv8 Jetson (Jetpack 5.0 以降)
+- Ubuntu 20.04 ARMv8 Jetson (JetPack 5.1.1 以降)
     - Jetson AGX Orin
     - Jetson AGX Xavier
     - Jetson Xavier NX
@@ -77,8 +77,9 @@ https://github.com/shiguredo/sora-cpp-sdk-samples
 
 **詳細は Discord やメールなどでお気軽にお問い合わせください**
 
-- Windows arm64 対応
+- NVIDIA Jetson JetPack 6 対応
 - AMD 系 HWA 対応
+- Windows arm64 対応
 
 ## サポートについて
 

VERSION

@@ -1,4 +1,4 @@
-SORA_CPP_SDK_VERSION=2023.17.0
+SORA_CPP_SDK_VERSION=2024.1.0
 WEBRTC_BUILD_VERSION=m120.6099.1.2
 BOOST_VERSION=1.83.0
 CMAKE_VERSION=3.27.7

include/sora/sora_signaling.h

@@ -108,13 +108,15 @@ struct SoraSignalingConfig {
   std::vector<DataChannel> data_channels;
 
   struct ForwardingFilter {
-    std::string action;
+    boost::optional<std::string> action;
     struct Rule {
       std::string field;
       std::string op;
       std::vector<std::string> values;
     };
     std::vector<std::vector<Rule>> rules;
+    boost::optional<std::string> version;
+    boost::optional<boost::json::value> metadata;
   };
   boost::optional<ForwardingFilter> forwarding_filter;
 

multistrap/ubuntu-20.04_armv8_jetson.conf

@@ -13,11 +13,11 @@ components=main universe
 [Jetson]
 packages=
 source=https://repo.download.nvidia.com/jetson/common
-suite=r35.2
+suite=r35.4
 components=main
 
 [T194]
 packages=nvidia-l4t-camera nvidia-l4t-jetson-multimedia-api
 source=https://repo.download.nvidia.com/jetson/t194
-suite=r35.2
+suite=r35.4
 components=main

src/hwenc_jetson/jetson_buffer.cpp

@@ -9,7 +9,8 @@
 #include <third_party/libyuv/include/libyuv.h>
 
 // Jetson Linux Multimedia API
-#include <nvbuf_utils.h>
+#include <nvbufsurface.h>
+#include <nvbufsurftransform.h>
 
 namespace sora {
 
@@ -62,62 +63,65 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> JetsonBuffer::ToI420() {
     int32_t buffer_width = ((scaled_width_ + 15) / 16) * 16;
     int32_t buffer_height = ((scaled_height_ + 15) / 16) * 16;
 
-    NvBufferCreateParams input_params = {0};
-    input_params.payloadType = NvBufferPayload_SurfArray;
-    input_params.width = buffer_width;
-    input_params.height = buffer_height;
-    input_params.layout = NvBufferLayout_Pitch;
-    input_params.colorFormat = NvBufferColorFormat_YUV420;
-    input_params.nvbuf_tag = NvBufferTag_NONE;
-
-    int dmabuf_fd;
-    if (NvBufferCreateEx(&dmabuf_fd, &input_params) == -1) {
-      RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufferCreateEx";
+    NvBufSurfaceAllocateParams input_params = {0};
+    input_params.params.width = buffer_width;
+    input_params.params.height = buffer_height;
+    input_params.params.layout = NVBUF_LAYOUT_PITCH;
+    input_params.params.colorFormat = NVBUF_COLOR_FORMAT_YUV420;
+    input_params.params.memType = NVBUF_MEM_SURFACE_ARRAY;
+    input_params.memtag = NvBufSurfaceTag_NONE;
+
+    NvBufSurface* dst_surf = 0;
+
+    if (NvBufSurfaceAllocate(
+            &dst_surf,
+            1, /* NvUtils では複数のバッファーを同時に初期化できるため、バッファーの数を指定する */
+            &input_params) == -1) {
+      RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufSurfaceAllocate";
       return scaled_buffer;
     }
+    NvBufSurfaceParams params = dst_surf->surfaceList[0];
 
-    NvBufferParams params = {0};
-    if (NvBufferGetParams(fd_, &params) == -1) {
-      RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufferGetParams";
-      return scaled_buffer;
-    }
-
-    NvBufferRect src_rect, dest_rect;
+    NvBufSurfTransformRect src_rect, dest_rect;
     src_rect.top = 0;
     src_rect.left = 0;
-    src_rect.width = params.width[0];
-    src_rect.height = params.height[0];
+    src_rect.width = params.width;
+    src_rect.height = params.height;
     dest_rect.top = 0;
     dest_rect.left = 0;
     dest_rect.width = buffer_width;
     dest_rect.height = buffer_height;
 
-    NvBufferTransformParams trans_params;
+    NvBufSurfTransformParams trans_params;
     memset(&trans_params, 0, sizeof(trans_params));
-    trans_params.transform_flag = NVBUFFER_TRANSFORM_FILTER;
-    trans_params.transform_flip = NvBufferTransform_None;
-    trans_params.transform_filter = NvBufferTransform_Filter_Smart;
-    trans_params.src_rect = src_rect;
-    trans_params.dst_rect = dest_rect;
-
-    if (NvBufferTransform(fd_, dmabuf_fd, &trans_params) == -1) {
-      RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufferTransform";
+    trans_params.transform_flag = NVBUFSURF_TRANSFORM_FILTER;
+    trans_params.transform_flip = NvBufSurfTransform_None;
+    trans_params.transform_filter = NvBufSurfTransformInter_Algo3;
+    trans_params.src_rect = &src_rect;
+    trans_params.dst_rect = &dest_rect;
+
+    NvBufSurface* src_surf = 0;
+    if (NvBufSurfaceFromFd(fd_, (void**)(&src_surf)) == -1) {
+      RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufSurfaceFromFd";
       return scaled_buffer;
     }
 
-    NvBufferParams dmabuf_params = {0};
-    if (NvBufferGetParams(dmabuf_fd, &dmabuf_params) == -1) {
-      RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufferGetParams";
+    if (NvBufSurfTransform(src_surf, dst_surf, &trans_params) !=
+        NvBufSurfTransformError_Success) {
+      RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to NvBufSurfTransform";
       return scaled_buffer;
     }
 
     int ret;
     void* data_addr;
     uint8_t* dest_addr;
-    for (int plane = 0; plane < dmabuf_params.num_planes; plane++) {
-      ret = NvBufferMemMap(dmabuf_fd, plane, NvBufferMem_Read, &data_addr);
+    int num_planes = dst_surf->surfaceList->planeParams.num_planes;
+    int index = 0;
+    for (int plane = 0; plane < num_planes; plane++) {
+      ret = NvBufSurfaceMap(dst_surf, index, plane, NVBUF_MAP_READ);
       if (ret == 0) {
-        NvBufferMemSyncForCpu(dmabuf_fd, plane, &data_addr);
+        NvBufSurfaceSyncForCpu(dst_surf, index, plane);
+        data_addr = dst_surf->surfaceList->mappedAddr.addr[plane];
         int height, width;
         if (plane == 0) {
           dest_addr = scaled_buffer.get()->MutableDataY();
@@ -134,18 +138,20 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> JetsonBuffer::ToI420() {
         }
         for (int i = 0; i < height; i++) {
           memcpy(dest_addr + width * i,
-                 (uint8_t*)data_addr + dmabuf_params.pitch[plane] * i, width);
+                 (uint8_t*)data_addr +
+                     dst_surf->surfaceList->planeParams.pitch[plane] * i,
+                 width);
         }
       }
-      NvBufferMemUnMap(dmabuf_fd, plane, &data_addr);
+      NvBufSurfaceUnMap(dst_surf, index, plane);
       if (ret == -1) {
         RTC_LOG(LS_ERROR) << __FUNCTION__
-                          << " Failed to NvBufferMemMap plane=" << plane;
+                          << " Failed to NvBufSurfaceMap plane=" << plane;
         return scaled_buffer;
       }
     }
 
-    NvBufferDestroy(dmabuf_fd);
+    NvBufSurfaceDestroy(dst_surf);
 
     return scaled_buffer;
   } else {

src/hwenc_jetson/jetson_jpeg_decoder_pool.cpp

@@ -11,15 +11,19 @@ namespace sora {
 std::shared_ptr<JetsonJpegDecoder> JetsonJpegDecoderPool::Pop() {
   std::shared_ptr<NvJPEGDecoder> nv_decoder;
 
-  {
-    std::lock_guard<std::mutex> lock(mtx_);
-    if (decoder_queue_.size() == 0) {
-      nv_decoder.reset(NvJPEGDecoder::createJPEGDecoder("jpegdec"));
-    } else {
-      nv_decoder = std::move(decoder_queue_.front());
-      decoder_queue_.pop();
-    }
-  }
+  // JetPack 5.1.2 で同じフレームが送信され続ける問題が発生したため、キューを無効化した
+  // JetPack 5.1.1 では正常に動作していた
+  // momo で同様の問題に対応した際の PR: https://github.com/shiguredo/momo/pull/297/
+  // {
+  //   std::lock_guard<std::mutex> lock(mtx_);
+  //   if (decoder_queue_.size() == 0) {
+  //     nv_decoder.reset(NvJPEGDecoder::createJPEGDecoder("jpegdec"));
+  //   } else {
+  //     nv_decoder = std::move(decoder_queue_.front());
+  //     decoder_queue_.pop();
+  //   }
+  // }
+  nv_decoder.reset(NvJPEGDecoder::createJPEGDecoder("jpegdec"));
 
   std::shared_ptr<JetsonJpegDecoder> decoder(
       new JetsonJpegDecoder(shared_from_this(), std::move(nv_decoder)));
@@ -28,7 +32,7 @@ std::shared_ptr<JetsonJpegDecoder> JetsonJpegDecoderPool::Pop() {
 
 void JetsonJpegDecoderPool::Push(std::shared_ptr<NvJPEGDecoder> decoder) {
   std::lock_guard<std::mutex> lock(mtx_);
-  decoder_queue_.push(std::move(decoder));
+  // decoder_queue_.push(std::move(decoder));
 }
 
 }  // namespace sora