From 202827d5fc26dbcdf4ff6f4b5b5ba079c8b9e696 Mon Sep 17 00:00:00 2001 From: Levi Lesches Date: Mon, 29 Jan 2024 23:54:15 -0500 Subject: [PATCH] Documented --- analysis_options.yaml | 2 +- lib/src/isolates/child.dart | 37 +++++++++++++++------------ lib/src/isolates/parent.dart | 2 +- lib/src/isolates/payload.dart | 18 ++++++++++--- lib/src/isolates/realsense.dart | 26 ++++++++++++++++++- lib/src/realsense/ffi.dart | 11 ++++---- lib/src/realsense/interface.dart | 14 +++++++++- lib/src/realsense/realsense_ffi.dart | 2 ++ lib/src/realsense/realsense_stub.dart | 1 + lib/src/server.dart | 2 ++ 10 files changed, 87 insertions(+), 28 deletions(-) diff --git a/analysis_options.yaml b/analysis_options.yaml index 70e6cfe..49c9dd0 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -48,4 +48,4 @@ linter: cascade_invocations: false # Cascades are less readable # Temporarily disabled until we are ready to document - public_member_api_docs: false + # public_member_api_docs: false diff --git a/lib/src/isolates/child.dart b/lib/src/isolates/child.dart index c30b854..daea2da 100644 --- a/lib/src/isolates/child.dart +++ b/lib/src/isolates/child.dart @@ -7,18 +7,14 @@ import "package:opencv_ffi/opencv_ffi.dart"; import "package:video/video.dart"; -const maxPacketLength = 60000; // max UDP packet size in bytes - -extension on CameraDetails { - bool get interferesWithAutonomy => hasResolutionHeight() - || hasResolutionWidth() - || hasFps() - || hasStatus(); -} +/// The maximum size of a UDP packet, in bytes (minus a few to be safe). +const maxPacketLength = 60000; +/// A child isolate that manages a single camera and streams frames from it. abstract class CameraIsolate extends IsolateChild { /// Holds the current details of the camera. final CameraDetails details; + /// A constructor with initial details. CameraIsolate({required this.details}) : super(id: details.name); /// A timer to periodically send the camera status to the dashboard. @@ -51,13 +47,7 @@ abstract class CameraIsolate extends IsolateChild } @override - void onData(VideoCommand data) { - if (data.details.interferesWithAutonomy) { - sendLog(LogLevel.error, "That would break autonomy"); - } else { - updateDetails(data.details); - } - } + void onData(VideoCommand data) => updateDetails(data.details); /// Updates the camera's [details], which will take effect on the next [sendFrame] call. void updateDetails(CameraDetails newDetails) { @@ -66,6 +56,9 @@ abstract class CameraIsolate extends IsolateChild start(); } + /// Disposes of this camera and all other resources. + /// + /// After running this, the camera should need to be opened again. void dispose() { disposeCamera(); frameTimer?.cancel(); @@ -73,14 +66,26 @@ abstract class CameraIsolate extends IsolateChild statusTimer?.cancel(); } + /// Initializes the camera and starts streaming. void initCamera(); + + /// Closes and releases the camera. + /// + /// This is separate from [dispose] so the isolate can keep reporting its status. void disposeCamera(); + + /// Reads frame/s from the camera and sends it/them. void sendFrames(); + /// Sends an individual frame to the dashboard. + /// + /// This function also checks if the frame is too big to send, and if so, + /// lowers the JPG quality by 1%. If the quality reaches 25% (visually noticeable), + /// an error is logged instead. void sendFrame(OpenCVImage image, {CameraDetails? detailsOverride}) { final details = detailsOverride ?? this.details; if (image.data.length < maxPacketLength) { // Frame can be sent - send(FramePayload(details: details, address: image.pointer.address, length: image.data.length)); + send(FramePayload(details: details, image: image)); } else if (details.quality > 25) { // Frame too large, lower quality sendLog(LogLevel.debug, "Lowering quality for $name from ${details.quality}"); details.quality -= 1; // maybe next frame can send diff --git a/lib/src/isolates/parent.dart b/lib/src/isolates/parent.dart index 8641a17..95d338c 100644 --- a/lib/src/isolates/parent.dart +++ b/lib/src/isolates/parent.dart @@ -40,7 +40,7 @@ class VideoController extends IsolateParent{ case DetailsPayload(): collection.videoServer.sendMessage(VideoData(details: data.details)); case FramePayload(): - final frame = data.getFrame(); + final frame = data.frame; collection.videoServer.sendMessage(VideoData(frame: frame.data, details: data.details)); frame.dispose(); case DepthFramePayload(): diff --git a/lib/src/isolates/payload.dart b/lib/src/isolates/payload.dart index 6ae8e0f..e6c73a2 100644 --- a/lib/src/isolates/payload.dart +++ b/lib/src/isolates/payload.dart @@ -35,13 +35,18 @@ class FramePayload extends IsolatePayload { final int length; /// A const constructor. - const FramePayload({required this.details, required this.address, required this.length}); + FramePayload({required this.details, required OpenCVImage image}) : + address = image.pointer.address, + length = image.data.length; /// The underlying data held at [address]. /// /// This cannot be a normal field as [Pointer]s cannot be sent across isolates, and this should /// not be a getter because the underlying memory needs to be freed and cannot be used again. - OpenCVImage getFrame() => OpenCVImage(pointer: Pointer.fromAddress(address), length: length); + OpenCVImage get frame => OpenCVImage(pointer: Pointer.fromAddress(address), length: length); + + /// Frees the data in this frame. + void dispose() => frame.dispose(); } /// A class to send log messages across isolates. The parent isolate is responsible for logging. @@ -54,10 +59,17 @@ class LogPayload extends IsolatePayload { const LogPayload({required this.level, required this.message}); } +/// A depth frame to be sent to the Autonomy program. class DepthFramePayload extends IsolatePayload { + /// The address of the data in memory, since pointers cannot be sent across isolates. final int address; - const DepthFramePayload(this.address); + /// Saves the address of the pointer to send across isolates. + DepthFramePayload(Pointer pointer) : + address = pointer.address; + /// The native frame being referenced by this pointer. Pointer get frame => Pointer.fromAddress(address); + + /// Frees the memory associated with the frame. void dispose() => frame.dispose(); } diff --git a/lib/src/isolates/realsense.dart b/lib/src/isolates/realsense.dart index cf1c9d1..f91fb0d 100644 --- a/lib/src/isolates/realsense.dart +++ b/lib/src/isolates/realsense.dart @@ -7,11 +7,35 @@ import "package:opencv_ffi/opencv_ffi.dart"; import "package:video/video.dart"; +extension on CameraDetails { + bool get interferesWithAutonomy => hasResolutionHeight() + || hasResolutionWidth() + || hasFps() + || hasStatus(); +} + +/// An isolate to read RGB, depth, and colorized frames from the RealSense. +/// +/// While using the RealSense SDK for depth streaming, OpenCV cannot access the standard RGB frames, +/// so it is necessary for this isolate to grab the RGB frames as well. +/// +/// Since the RealSense is being used for autonomy, certain settings that could interfere with the +/// autonomy program are not allowed to be changed, even for the RGB camera. class RealSenseIsolate extends CameraIsolate { + /// The native RealSense object. MUST be `late` so it isn't initialized on the parent isolate. late final RealSenseInterface camera = RealSenseInterface.forPlatform(); - bool hasError = false; + /// Creates an isolate to read from the RealSense camera. RealSenseIsolate({required super.details}); + @override + void onData(VideoCommand data) { + if (data.details.interferesWithAutonomy) { + sendLog(LogLevel.error, "That would break autonomy"); + } else { + super.onData(data); + } + } + @override void initCamera() { if (!camera.init()) { diff --git a/lib/src/realsense/ffi.dart b/lib/src/realsense/ffi.dart index 0a20dac..3c98d1f 100644 --- a/lib/src/realsense/ffi.dart +++ b/lib/src/realsense/ffi.dart @@ -5,18 +5,19 @@ import "dart:typed_data"; import "../generated/librealsense_ffi_bindings.dart"; export "../generated/librealsense_ffi_bindings.dart"; +/// Utils on a [Pointer] to [NativeFrames]. extension NativeFramesUtils on Pointer { + /// Frees the memory of all the frames. void dispose() => realsenseLib.NativeFrames_free(this); + /// The depth frame, as raw bytes. + /// + /// NOTE: The RealSense SDK returns [Uint16]s, but this is cast to a [Uint8] for UDP transfer. Be + /// sure to re-cast it on the processing side! Uint8List get depthFrame { final NativeFrames struct = ref; return struct.depth_data.asTypedList(struct.depth_length); } - - Uint8List get colorizedFrame { - final NativeFrames struct = ref; - return struct.colorized_data.asTypedList(struct.colorized_length); - } } String _getPath() { diff --git a/lib/src/realsense/interface.dart b/lib/src/realsense/interface.dart index b0be70f..f2ed13c 100644 --- a/lib/src/realsense/interface.dart +++ b/lib/src/realsense/interface.dart @@ -6,20 +6,32 @@ import "package:video/video.dart"; import "realsense_ffi.dart"; import "realsense_stub.dart"; +/// An interface for reading the RealSense camera. abstract class RealSenseInterface { - RealSenseInterface(); + /// A const constructor. + const RealSenseInterface(); + /// Decides which implementation to use depending on platform. factory RealSenseInterface.forPlatform() => Platform.isLinux ? RealSenseFFI() : RealSenseStub(); + /// Initializes the RealSense. Returns whether the initialization was successful. bool init(); + /// Releases the RealSense. Calling [init] again should re-open the device. void dispose(); + /// Starts the RealSense stream and waits for a valid frame. bool startStream(); + /// Stops the stream but keeps the device alive. void stopStream(); + /// The width of the frames. int get width; + /// The height of the frames. int get height; + /// The depth scale -- each pixel in the depth frame is an integer multiple of this, in meters. double get scale; + /// Gets the name and model of the camera. String getName(); + /// Gets the currently available frames. May return [nullptr]. Pointer getFrames(); } diff --git a/lib/src/realsense/realsense_ffi.dart b/lib/src/realsense/realsense_ffi.dart index a06981f..64b40f9 100644 --- a/lib/src/realsense/realsense_ffi.dart +++ b/lib/src/realsense/realsense_ffi.dart @@ -3,7 +3,9 @@ import "package:ffi/ffi.dart"; import "package:video/video.dart"; +/// An FFI implementation of [RealSenseInterface] using [librealsense](https://github.com/IntelRealSense/librealsense). class RealSenseFFI extends RealSenseInterface { + /// The native FFI device. final device = realsenseLib.RealSense_create(); @override late double scale; @override int height = 0; diff --git a/lib/src/realsense/realsense_stub.dart b/lib/src/realsense/realsense_stub.dart index f740dde..ee0149e 100644 --- a/lib/src/realsense/realsense_stub.dart +++ b/lib/src/realsense/realsense_stub.dart @@ -2,6 +2,7 @@ import "dart:ffi"; import "package:video/video.dart"; +/// A stub implementation for platforms or devices without the RealSense SDK. class RealSenseStub extends RealSenseInterface { @override bool init() => true; diff --git a/lib/src/server.dart b/lib/src/server.dart index 758a204..9273b53 100644 --- a/lib/src/server.dart +++ b/lib/src/server.dart @@ -4,6 +4,7 @@ import "package:burt_network/burt_network.dart"; import "collection.dart"; +/// The socket to send autonomy data to. final autonomySocket = SocketInfo(address: InternetAddress("192.168.1.30"), port: 8003); /// Class for the video program to interact with the dashboard @@ -24,6 +25,7 @@ class VideoServer extends RoverServer { collection.parent.send(data: command, id: command.details.name); } + /// Sends the depth frame to [autonomySocket]. void sendDepthFrame(VideoData frame) => sendMessage(frame, destinationOverride: autonomySocket);