Reduced RGB resolution for faster ArUco detection (#11)

Went from 5 FPS to 15 FPS --------- Co-authored-by: Binghamton Rover <[email protected]>
BinghamtonRover · Feb 12, 2024 · 80b4317 · 80b4317
1 parent 1f3555d
commit 80b4317
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 26 deletions.
diff --git a/lib/src/generated/librealsense_ffi_bindings.dart b/lib/src/generated/librealsense_ffi_bindings.dart
@@ -163,10 +163,16 @@ abstract class BurtRsStatus {
 
 final class BurtRsConfig extends ffi.Struct {
   @ffi.Int()
-  external int width;
+  external int depth_width;
 
   @ffi.Int()
-  external int height;
+  external int depth_height;
+
+  @ffi.Int()
+  external int rgb_width;
+
+  @ffi.Int()
+  external int rgb_height;
 
   @ffi.Float()
   external double scale;

diff --git a/lib/src/isolates/realsense.dart b/lib/src/isolates/realsense.dart
@@ -68,7 +68,7 @@ class RealSenseIsolate extends CameraIsolate {
 
     // Compress colorized frame
     final Pointer<Uint8> rawColorized = frames.ref.colorized_data;
-    final Pointer<Mat> colorizedMatrix = getMatrix(camera.height, camera.width, rawColorized);
+    final Pointer<Mat> colorizedMatrix = getMatrix(camera.depthResolution.height, camera.depthResolution.width, rawColorized);
     final OpenCVImage? colorizedJpg = encodeJpg(colorizedMatrix, quality: details.quality);
     if (colorizedJpg == null) {
       sendLog(LogLevel.debug, "Could not encode colorized frame"); 
@@ -78,7 +78,7 @@ class RealSenseIsolate extends CameraIsolate {
 
     // Get RGB frame
     final Pointer<Uint8> rawRGB = frames.ref.rgb_data;
-    final Pointer<Mat> rgbMatrix = getMatrix(camera.height, camera.width, rawRGB);
+    final Pointer<Mat> rgbMatrix = getMatrix(camera.rgbResolution.height, camera.rgbResolution.width, rawRGB);
     detectAndAnnotateFrames(rgbMatrix);  // detect ArUco tags
 
     // Compress the RGB frame into a JPG

diff --git a/lib/src/realsense/interface.dart b/lib/src/realsense/interface.dart
@@ -6,6 +6,9 @@ import "package:video/video.dart";
 import "realsense_ffi.dart";
 import "realsense_stub.dart";
 
+/// The resolution of an image.
+typedef Resolution = ({int width, int height});
+
 /// An interface for reading the RealSense camera.
 abstract class RealSenseInterface {
   /// A const constructor.
@@ -23,10 +26,10 @@ abstract class RealSenseInterface {
   /// Stops the stream but keeps the device alive.
   void stopStream();
 
-  /// The width of the frames.
-  int get width;
-  /// The height of the frames.
-  int get height;
+  /// The resolution of the depth frames.
+  Resolution get depthResolution;
+  /// The resolution of the RGB frames.
+  Resolution get rgbResolution;
   /// The depth scale -- each pixel in the depth frame is an integer multiple of this, in meters.
   double get scale;
 

diff --git a/lib/src/realsense/realsense_ffi.dart b/lib/src/realsense/realsense_ffi.dart
@@ -8,8 +8,8 @@ class RealSenseFFI extends RealSenseInterface {
   /// The native FFI device. 
   final device = realsenseLib.RealSense_create();
   @override late double scale;
-  @override int height = 0;
-  @override int width = 0;
+  @override Resolution depthResolution = (height: 0, width: 0);
+  @override Resolution rgbResolution = (height: 0, width: 0);
 
   @override
   bool init() {
@@ -27,8 +27,8 @@ class RealSenseFFI extends RealSenseInterface {
       return false;
     }
     final config = realsenseLib.RealSense_getDeviceConfig(device);
-    height = config.height;
-    width = config.width;
+    depthResolution = (height: config.depth_height, width: config.depth_width);
+    rgbResolution = (height: config.rgb_height, width: config.rgb_width);
     scale = config.scale;
     return true;
   }

diff --git a/lib/src/realsense/realsense_stub.dart b/lib/src/realsense/realsense_stub.dart
@@ -12,8 +12,8 @@ class RealSenseStub extends RealSenseInterface {
   @override bool startStream() => true;
   @override void stopStream() { }
 
-  @override int get width => 0;
-  @override int get height => 0;
+  @override Resolution get depthResolution => (height: 0, width: 0);
+  @override Resolution get rgbResolution => (height: 0, width: 0);
   @override double get scale => 0;
 
 

diff --git a/src/realsense_ffi.h b/src/realsense_ffi.h
@@ -23,8 +23,10 @@ typedef enum {
 } BurtRsStatus;
 
 typedef struct {
-	int width;
-	int height;
+	int depth_width;
+	int depth_height;
+	int rgb_width;
+	int rgb_height;
 	float scale;
 } BurtRsConfig;
 

diff --git a/src/realsense_internal.cpp b/src/realsense_internal.cpp
@@ -4,7 +4,8 @@
 #include <iostream>
 #include <unistd.h>
 
-#define WIDTH 640
+#define DEPTH_WIDTH 640
+#define RGB_WIDTH 320
 #define HEIGHT 0
 
 using namespace std;
@@ -46,19 +47,24 @@ const char* burt_rs::RealSense::getDeviceName() {
 
 BurtRsStatus burt_rs::RealSense::startStream() {
   rs2::config rs_config;
-  rs_config.enable_stream(RS2_STREAM_DEPTH, WIDTH, HEIGHT);
-  rs_config.enable_stream(RS2_STREAM_COLOR, WIDTH, HEIGHT, RS2_FORMAT_BGR8);
+  rs_config.enable_stream(RS2_STREAM_DEPTH, DEPTH_WIDTH, HEIGHT);
+  rs_config.enable_stream(RS2_STREAM_COLOR, RGB_WIDTH, HEIGHT, RS2_FORMAT_BGR8);
   auto profile = pipeline.start(rs_config);
   auto frames = pipeline.wait_for_frames();
-  auto frame = frames.get_depth_frame();
-  auto width = frame.get_width();
-  auto height = frame.get_height();
-
-  if (width == 0 || height == 0) {
+  auto depth_frame = frames.get_depth_frame();
+  auto rgb_frame = frames.get_color_frame();
+  auto depth_width = depth_frame.get_width();
+  auto depth_height = depth_frame.get_height();
+  auto rgb_width = rgb_frame.get_width();
+  auto rgb_height = rgb_frame.get_height();
+
+  if (rgb_width == 0 || rgb_height == 0 || depth_width == 0 || depth_height == 0) {
     return BurtRsStatus::BurtRsStatus_resolution_unknown;
   } else {
-    config.width = width;
-    config.height = height;
+    config.depth_width = depth_width;
+    config.depth_height = depth_height;
+    config.rgb_width = rgb_width;
+    config.rgb_height = rgb_height;
     return BurtRsStatus::BurtRsStatus_ok;
   }
 }