Skip to content

Commit

Permalink
Split isolate payload into FramePayload, LogPayload, and `Details…
Browse files Browse the repository at this point in the history
…Payload` (#9)

* Documented everything

* Added lint explanation to deprecated_member_use

* Upgraded package:typed_isolate
  • Loading branch information
Levi-Lesches authored Jan 23, 2024
1 parent 628315d commit 8c23dcc
Show file tree
Hide file tree
Showing 9 changed files with 154 additions and 144 deletions.
1 change: 1 addition & 0 deletions analysis_options.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ linter:
sort_constructors_first: false # final properties, then constructor
avoid_dynamic_calls: false # this lint takes over errors in the IDE
omit_local_variable_types: false # it can be helpful sometimes to annotate types
cascade_invocations: false # Cascades are less readable

# Temporarily disabled until we are ready to document
# public_member_api_docs: false
58 changes: 0 additions & 58 deletions bin/example.dart

This file was deleted.

38 changes: 20 additions & 18 deletions lib/src/camera_isolate.dart
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import "periodic_timer.dart";
///
/// This class accepts [VideoCommand]s and calls [updateDetails] with the newly-received details.
/// When a frame is read, instead of sending the [VideoData], this class sends only the pointer
/// to the [OpenCVImage] via the [FrameData] class, and the image is read by the parent isolate.
class CameraIsolate extends IsolateChild<FrameData, VideoCommand>{
/// to the [OpenCVImage] via the [IsolatePayload] class, and the image is read by the parent isolate.
class CameraIsolate extends IsolateChild<IsolatePayload, VideoCommand>{
/// The native camera object from OpenCV.
late final Camera camera;
/// Holds the current details of the camera.
Expand All @@ -28,26 +28,28 @@ class CameraIsolate extends IsolateChild<FrameData, VideoCommand>{
/// Records how many FPS this camera is actually running at.
int fpsCount = 0;

/// The log level at which this isolate should be reporting.
LogLevel logLevel;

/// Creates a new manager for the given camera and default details.
CameraIsolate({required this.details, required this.logLevel}) : super(id: details.name);
CameraIsolate({required this.details}) : super(id: details.name);

/// The name of this camera (where it is on the rover).
CameraName get name => details.name;

/// Sends the current status to the dashboard (with an empty frame).
void sendStatus([_]) => send(FrameData(details: details, address: 0, length: 0));
void sendStatus([_]) => send(DetailsPayload(details));

/// Logs a message by sending a [LogPayload] to the parent isolate.
///
/// Note: it is important to _not_ log this message directly in _this_ isolate, as it will
/// not be configurable by the parent isolate and will not be sent to the Dashboard.
void log(LogLevel level, String message) => send(LogPayload(level: level, message: message));

@override
Future<void> run() async {
Logger.level = logLevel;
logger.debug("Initializing camera: $name");
log(LogLevel.debug, "Initializing camera: $name");
camera = getCamera(name);
statusTimer = Timer.periodic(const Duration(seconds: 5), sendStatus);
if (!camera.isOpened) {
logger.warning("Camera $name is not connected");
log(LogLevel.warning, "Camera $name is not connected");
updateDetails(CameraDetails(status: CameraStatus.CAMERA_DISCONNECTED));
}
start();
Expand All @@ -69,24 +71,24 @@ class CameraIsolate extends IsolateChild<FrameData, VideoCommand>{
frameTimer?.cancel();
fpsTimer?.cancel();
statusTimer?.cancel();
logger.info("Disposed camera $name");
log(LogLevel.info, "Disposed camera $name");
}

/// Starts the camera and timers.
void start() {
if (details.status != CameraStatus.CAMERA_ENABLED) return;
logger.debug("Starting camera $name. Status=${details.status}");
log(LogLevel.debug, "Starting camera $name. Status=${details.status}");
final interval = details.fps == 0 ? Duration.zero : Duration(milliseconds: 1000 ~/ details.fps);
frameTimer = PeriodicTimer(interval, sendFrame);
fpsTimer = Timer.periodic(const Duration(seconds: 5), (_) {
logger.trace("Camera $name sent ${fpsCount ~/ 5} frames");
log(LogLevel.trace, "Camera $name sent ${fpsCount ~/ 5} frames");
fpsCount = 0;
});
}

/// Cancels all timers and stops reading the camera.
void stop() {
logger.debug("Stopping camera $name");
log(LogLevel.debug, "Stopping camera $name");
frameTimer?.cancel();
fpsTimer?.cancel();
}
Expand All @@ -100,16 +102,16 @@ class CameraIsolate extends IsolateChild<FrameData, VideoCommand>{
Future<void> sendFrame() async {
final frame = camera.getJpg(quality: details.quality);
if (frame == null) { // Error getting the frame
logger.warning("Camera $name didn't respond");
log(LogLevel.warning, "Camera $name didn't respond");
updateDetails(CameraDetails(status: CameraStatus.CAMERA_NOT_RESPONDING));
} else if (frame.data.length < 60000) { // Frame can be sent
send(FrameData(address: frame.pointer.address, length: frame.data.length, details: details));
send(FramePayload(address: frame.pointer.address, length: frame.data.length, details: details));
fpsCount++;
} else if (details.quality > 25) { // Frame too large, try lowering quality
logger.debug("Lowering quality for $name from ${details.quality}");
log(LogLevel.debug, "Lowering quality for $name from ${details.quality}");
updateDetails(CameraDetails(quality: details.quality - 1));
} else { // Frame too large, cannot lower quality anymore
logger.warning("$name's frames are too large (${frame.data.length} bytes, quality=${details.quality})");
log(LogLevel.warning, "$name's frames are too large (${frame.data.length} bytes, quality=${details.quality})");
updateDetails(CameraDetails(status: CameraStatus.FRAME_TOO_LARGE));
}
}
Expand Down
15 changes: 14 additions & 1 deletion lib/src/collection.dart
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,22 @@ class Collection {
Future<void> init() async {
logger..trace("Running in trace mode")..debug("Running in debug mode");
await videoServer.init();
await parent.run();
await parent.init();
logger.info("Video program initialized");
}

/// Stops all cameras and disconnects from the hardware.
Future<void> dispose() async {
parent.stopAll();
parent.killAll();
await videoServer.dispose();
}

/// Restarts the video program.
Future<void> restart() async {
await dispose();
await init();
}
}

/// Holds all the devices connected
Expand Down
56 changes: 44 additions & 12 deletions lib/src/frame.dart
Original file line number Diff line number Diff line change
@@ -1,21 +1,53 @@
import "dart:ffi";

import "package:burt_network/burt_network.dart";
import "package:burt_network/logging.dart";
import "package:opencv_ffi/opencv_ffi.dart";

/// A payload containing some data to report back to the parent isolate.
///
/// Instead of having nullable fields on this class, we subclass it and provide
/// only the relevant fields for each subclass. That way, for example, you cannot
/// accidentally send a frame without a [CameraDetails].
sealed class IsolatePayload { const IsolatePayload(); }

/// A payload representing the status of the given camera.
class DetailsPayload extends IsolatePayload {
/// The details being sent.
final CameraDetails details;
/// A const constructor.
const DetailsPayload(this.details);
}

/// A container for a pointer to a native buffer that can be sent across isolates.
///
/// Sending a buffer across isolates would mean that data is copied, which is not ideal for
/// buffers containing an entire JPG image, from multiple isolates, multiple frames per second.
/// Since we cannot yet send FFI pointers across isolates, we have to send its raw address
/// instead.
class FrameData {
/// The [CameraDetails] for this frame.
/// Since we cannot yet send FFI pointers across isolates, we have to send its raw address.
class FramePayload extends IsolatePayload {
/// The details of the camera this frame came from.
final CameraDetails details;
/// The address of the FFI pointer containing the image.
final int? address;
/// The amount of bytes to read past [address].
final int? length;
/// The address in FFI memory this frame starts at.
final int address;
/// The length of this frame in bytes.
final int length;

/// A const constructor.
const FramePayload({required this.details, required this.address, required this.length});

/// The underlying data held at [address].
///
/// This cannot be a normal field as [Pointer]s cannot be sent across isolates, and this should
/// not be a getter because the underlying memory needs to be freed and cannot be used again.
OpenCVImage getFrame() => OpenCVImage(pointer: Pointer.fromAddress(address), length: length);
}

/// Creates a [FrameData] containing an actual frame.
FrameData({required this.details, required this.address, required this.length});
/// Creates a [FrameData] that only has [CameraDetails].
FrameData.details(this.details) : address = null, length = null;
/// A class to send log messages across isolates. The parent isolate is responsible for logging.
class LogPayload extends IsolatePayload {
/// The level to log this message.
final LogLevel level;
/// The message to log.
final String message;
/// A const constructor.
const LogPayload({required this.level, required this.message});
}
43 changes: 30 additions & 13 deletions lib/src/parent_isolate.dart
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import "dart:async";
import "dart:ffi";

import "package:opencv_ffi/opencv_ffi.dart";
import "package:typed_isolate/typed_isolate.dart";
Expand All @@ -13,32 +12,50 @@ import "camera_isolate.dart";
/// A parent isolate that spawns [CameraIsolate]s to manage the cameras.
///
/// With one isolate per camera, each camera can read in parallel. This class sends [VideoCommand]s
/// from the dashboard to the appropriate [CameraIsolate], and receives [FrameData]s which it uses
/// from the dashboard to the appropriate [CameraIsolate], and receives [IsolatePayload]s which it uses
/// to read an [OpenCVImage] from native memory and send to the dashboard. By not sending the frame
/// from child isolate to the parent (just the pointer), we save a whole JPG image's worth of bytes
/// from every camera, every frame, every second. That could be up to 5 MB per second of savings.
class VideoController extends IsolateParent<VideoCommand, FrameData>{
class VideoController extends IsolateParent<VideoCommand, IsolatePayload>{
@override
Future<void> run() async {
Future<void> init() async {
for (final name in CameraName.values) {
if (name == CameraName.CAMERA_NAME_UNDEFINED) continue;
await spawn(
CameraIsolate(
logLevel: Logger.level,
details: getDefaultDetails(name),
),
);
}
}

@override
void onData(FrameData data) {
if (data.address == null) {
collection.videoServer.sendMessage(VideoData(details: data.details));
} else {
final frame = OpenCVImage(pointer: Pointer.fromAddress(data.address!), length: data.length!);
collection.videoServer.sendMessage(VideoData(frame: frame.data, details: data.details));
frame.dispose();
void onData(IsolatePayload data, Object id) {
switch (data) {
case DetailsPayload():
collection.videoServer.sendMessage(VideoData(details: data.details));
case FramePayload():
final frame = data.getFrame();
collection.videoServer.sendMessage(VideoData(frame: frame.data, details: data.details));
frame.dispose();
case LogPayload(): switch (data.level) {
// Turns out using deprecated members when you *have* to still results in a lint.
// See https://github.com/dart-lang/linter/issues/4852 for why we ignore it.
case LogLevel.all: logger.info(data.message);
// ignore: deprecated_member_use
case LogLevel.verbose: logger.trace(data.message);
case LogLevel.trace: logger.trace(data.message);
case LogLevel.debug: logger.debug(data.message);
case LogLevel.info: logger.info(data.message);
case LogLevel.warning: logger.warning(data.message);
case LogLevel.error: logger.error(data.message);
// ignore: deprecated_member_use
case LogLevel.wtf: logger.info(data.message);
case LogLevel.fatal: logger.critical(data.message);
// ignore: deprecated_member_use
case LogLevel.nothing: logger.info(data.message);
case LogLevel.off: logger.info(data.message);
}
}
}

Expand All @@ -47,7 +64,7 @@ class VideoController extends IsolateParent<VideoCommand, FrameData>{
final command = VideoCommand(details: CameraDetails(status: CameraStatus.CAMERA_DISABLED));
for (final name in CameraName.values) {
if (name == CameraName.CAMERA_NAME_UNDEFINED) continue;
send(command, name);
send(data: command, id: name);
}
}
}
7 changes: 5 additions & 2 deletions lib/src/server.dart
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import "package:burt_network/burt_network.dart";
import "collection.dart";

/// Class for the video program to interact with the dashboard
class VideoServer extends ServerSocket {
class VideoServer extends RoverServer {
/// Requires a port to communicate through
VideoServer({required super.port}) : super(device: Device.VIDEO);

Expand All @@ -13,6 +13,9 @@ class VideoServer extends ServerSocket {
if (wrapper.name != VideoCommand().messageName) return;
final command = VideoCommand.fromBuffer(wrapper.data);
sendMessage(command); // Echo the request
collection.parent.send(command, command.details.name);
collection.parent.send(data: command, id: command.details.name);
}

@override
void restart() => collection.restart();
}
Loading

0 comments on commit 8c23dcc

Please sign in to comment.