From a1795c2d2e8a7a68a08eb9c73125cfc36c0d347a Mon Sep 17 00:00:00 2001 From: Paul Ampadu <89813338+pgrwe@users.noreply.github.com> Date: Wed, 22 May 2024 17:54:03 -0400 Subject: [PATCH] Dev - camera controls (#129) - Need to fix autofocus - Need to fix number of requests sent per second - Need to maintain state for settings panel Never again will I use the dart vscode plugin. --------- Co-authored-by: AndyZ54 Co-authored-by: Levi Lesches Co-authored-by: aidan ahram --- lib/src/models/data/video.dart | 7 +- .../models/view/builders/video_builder.dart | 16 +- lib/src/widgets/atomic/video_feed.dart | 316 ++++++++++++++---- pubspec.lock | 14 +- 4 files changed, 278 insertions(+), 75 deletions(-) diff --git a/lib/src/models/data/video.dart b/lib/src/models/data/video.dart index cdeb50cd38..6a9b9dc71d 100644 --- a/lib/src/models/data/video.dart +++ b/lib/src/models/data/video.dart @@ -116,10 +116,11 @@ class VideoModel extends Model { } /// Updates settings for the given camera. - Future updateCamera(String id, CameraDetails details) async { + Future updateCamera(String id, CameraDetails details, {bool verify = true}) async { _handshake = null; final command = VideoCommand(id: id, details: details); models.sockets.video.sendMessage(command); + if (!verify) return; await Future.delayed(const Duration(seconds: 2)); if (_handshake == null) throw RequestNotAccepted(); } @@ -144,11 +145,11 @@ class VideoModel extends Model { text: "Could not ${enable ? 'enable' : 'disable'} the ${name.humanName} camera", ); } - } + } } /// An exception thrown when the rover does not respond to a handshake. -/// +/// /// Certain changes require a handshake to ensure the rover has received and applied the change. /// If the rover fails to acknowledge or apply the change, a response will not be sent. Throw /// this error to indicate that. diff --git a/lib/src/models/view/builders/video_builder.dart b/lib/src/models/view/builders/video_builder.dart index b147e1d3b4..afe942c326 100644 --- a/lib/src/models/view/builders/video_builder.dart +++ b/lib/src/models/view/builders/video_builder.dart @@ -32,9 +32,12 @@ class CameraDetailsBuilder extends ValueBuilder { /// Whether changes are loading. bool isLoading = false; - /// The error that occurrec when changing these settings, if any. + /// The error that occurred when changing these settings, if any. String? error; + /// Current status of the camera's autofocus + bool autofocus = true; + @override List> get otherBuilders => [resolutionHeight, resolutionWidth, quality, fps]; @@ -45,7 +48,8 @@ class CameraDetailsBuilder extends ValueBuilder { quality = NumberBuilder(data.quality, min: 0, max: 100), fps = NumberBuilder(data.fps, min: 0, max: 60), name = data.name, - status = CameraStatus.CAMERA_ENABLED; + status = CameraStatus.CAMERA_ENABLED, + autofocus = data.autofocus; @override bool get isValid => resolutionHeight.isValid @@ -54,7 +58,8 @@ class CameraDetailsBuilder extends ValueBuilder { && fps.isValid && okStatuses.contains(status); - @override + + @override CameraDetails get value => CameraDetails( resolutionHeight: resolutionHeight.value, resolutionWidth: resolutionWidth.value, @@ -62,6 +67,11 @@ class CameraDetailsBuilder extends ValueBuilder { fps: fps.value, name: name, status: status, + focus: 0, + zoom: 100, + pan: 0, + tilt: 0, + autofocus: true, ); /// Updates the [status] field. diff --git a/lib/src/widgets/atomic/video_feed.dart b/lib/src/widgets/atomic/video_feed.dart index ee8bfa4b45..d0ea506dd1 100644 --- a/lib/src/widgets/atomic/video_feed.dart +++ b/lib/src/widgets/atomic/video_feed.dart @@ -13,7 +13,7 @@ import "package:rover_dashboard/widgets.dart"; /// - Call [load] with your image data /// - Pass [image] to a [RawImage] widget, if it isn't null /// - Call [dispose] to release all resources used by the image. -/// +/// /// It is safe to call [load] or [dispose] multiple times, and calling [load] /// will automatically call [dispose] on the existing resources. class ImageLoader { @@ -32,8 +32,8 @@ class ImageLoader { /// Processes the next frame and stores the result in [image]. Future load(List bytes) async { isLoading = true; - final ulist = Uint8List.fromList(bytes.toList()); - codec = await ui.instantiateImageCodec(ulist); + final buffer = Uint8List.fromList(bytes.toList()); + codec = await ui.instantiateImageCodec(buffer); final frame = await codec!.getNextFrame(); image = frame.image; isLoading = false; @@ -59,10 +59,50 @@ class VideoFeed extends StatefulWidget { VideoFeedState createState() => VideoFeedState(); } +/// Class that defines a slider for camera controls +class SliderSettings extends StatelessWidget { + /// Name of the slider + final String label; + + /// Value corresponding to the slider + final double value; + + /// Value to change the position of the slider + final ValueChanged onChanged; + + /// The min value on this slider. + final double min; + + /// The max value on this slider. + final double max; + + /// Constructor for SliderSettings + const SliderSettings({ + required this.label, + required this.value, + required this.onChanged, + this.min = 0, + this.max = 100, + }); + + @override + Widget build(BuildContext context) => Column( + children: [ + Text("$label: ${value.floor()}"), + Slider( + value: value, + onChanged: onChanged, + max: max, + min: min, + ), + ], + ); +} + /// The logic for updating a [VideoFeed]. /// /// This widget listens to [VideoModel.frameUpdater] to sync its framerate with other [VideoFeed]s. -/// On every update, this widget grabs the frame from [VideoData.frame], decodes it, renders it, +/// On every update, this widget grabs the frame from [VideoData.frame], decodes it, renders it, /// then replaces the old frame. The key is that all the image processing logic is done off-screen /// while the old frame remains on-screen. When the frame is processed, it quickly replaces the old /// frame. That way, the user sees one continuous video instead of a flickering image. @@ -73,6 +113,21 @@ class VideoFeedState extends State { /// A helper class responsible for managing and loading an image. final imageLoader = ImageLoader(); + /// Checks if the current slider for video camera is open + bool isOpened = false; + + /// Value for zoom + double zoom = 0; + + /// Value for pan + double pan = 0; + + /// Value for focus + double focus = 0; + + /// Value for brightness + double brightness = 0; + @override void initState() { super.initState(); @@ -91,68 +146,205 @@ class VideoFeedState extends State { Future updateImage() async { data = models.video.feeds[widget.name]!; if (data.details.status != CameraStatus.CAMERA_ENABLED) { - setState(() => imageLoader.image = null); + setState(() => imageLoader.image = null); } - setState(() { }); + setState(() {}); if (!data.hasFrame() || imageLoader.isLoading) return; await imageLoader.load(data.frame); - if (mounted) setState(() { }); + if (mounted) setState(() {}); } + /// Whether there is a frame ready to display. + bool get isReady => models.sockets.video.isConnected + && imageLoader.hasImage + && data.details.status == CameraStatus.CAMERA_ENABLED; + + /// Builds the inside of the video feed. Either a video frame or an error message. + Widget buildChild(BuildContext context) => isReady + ? Row( + children: [ + AnimatedContainer( + duration: const Duration(milliseconds: 250), + height: double.infinity, + width: isOpened ? 200 : 0, + child: VideoSettingsWidget( + id: data.id, + details: data.details, + ), + ), + Expanded(child: RawImage(image: imageLoader.image, fit: BoxFit.contain)), + ], + ) + : Center(child: Text(errorMessage, textAlign: TextAlign.center)); + + @override + Widget build(BuildContext context) => Container( + color: context.colorScheme.brightness == Brightness.light + ? Colors.blueGrey + : Colors.blueGrey[700], + child: Column( + children: [ + Row( + children: [ + IconButton( + onPressed: toggleSettings, + icon: const Icon(Icons.tune), + ), + Text(data.details.name.humanName), + const Spacer(), + Text("${models.video.networkFps[data.details.name]!} FPS"), + if (data.hasFrame()) + IconButton( + icon: const Icon(Icons.add_a_photo), + onPressed: () => models.video.saveFrame(widget.name), + ), + IconButton( + icon: const Icon(Icons.settings), + onPressed: () async => showDialog( + context: context, + builder: (_) => CameraDetailsEditor(data), + ), + ), + ViewsSelector(currentView: widget.name.humanName), + ], + ), + Expanded(child: buildChild(context)), + ], + ), + ); + + /// Opens or closes the settings panel. + void toggleSettings() => setState(() => isOpened = !isOpened); + + /// Displays an error message describing why `image == null`. + String get errorMessage { + if (!models.sockets.video.isConnected) return "The video program is not connected"; + switch (data.details.status) { + case CameraStatus.CAMERA_LOADING: + return "Camera is loading..."; + case CameraStatus.CAMERA_STATUS_UNDEFINED: + return "Unknown error"; + case CameraStatus.CAMERA_DISCONNECTED: + return "Camera is not connected"; + case CameraStatus.CAMERA_DISABLED: + return "Camera is disabled.\nClick the settings icon to enabled it."; + case CameraStatus.CAMERA_NOT_RESPONDING: + return "Camera is not responding"; + case CameraStatus.FRAME_TOO_LARGE: + return "Camera is reading too much detail\nReduce the quality or resolution"; + case CameraStatus.CAMERA_HAS_NO_NAME: + return "Camera has no name\nChange lib/constants.py on the video Pi"; + case CameraStatus.CAMERA_ENABLED: + if (data.hasFrame()) { + return "Loading feed..."; + } else { + return "Starting camera..."; + } + } + return "Unknown error"; + } +} + +/// A widget to edit camera settings. +class VideoSettingsWidget extends StatefulWidget { + /// The details being sent to the camera. + final CameraDetails details; + /// The ID of the camera being edited. + final String id; + /// Creates the video settings widget. + const VideoSettingsWidget({ + required this.details, + required this.id, + }); + + @override + VideoSettingsState createState() => VideoSettingsState(); +} + +/// A state for [VideoSettingsWidget]. +class VideoSettingsState extends State { + /// The zoom level. Camera-specific. + double zoom = 100; + /// The pan level, when zoomed in. + double pan = 0; + /// The tilt level, when zoomed in. + double tilt = 0; + /// The focus level, if autofocus is disabled. + double focus = 0; + /// Whether the camera should autofocus. + bool autofocus = true; + @override - Widget build(BuildContext context) => Stack( - children: [ - Container( - color: context.colorScheme.brightness == Brightness.light - ? Colors.blueGrey - : Colors.blueGrey[700], - height: double.infinity, - width: double.infinity, - padding: const EdgeInsets.all(4), - alignment: Alignment.center, - child: models.sockets.video.isConnected && imageLoader.hasImage && data.details.status == CameraStatus.CAMERA_ENABLED - ? Row(children: [ - Expanded(child: RawImage(image: imageLoader.image, fit: BoxFit.contain)), - ],) - : Text(errorMessage, textAlign: TextAlign.center), - ), - Row( - mainAxisAlignment: MainAxisAlignment.end, - children: [ - Text("${models.video.networkFps[data.details.name]!} FPS"), - if (data.hasFrame()) IconButton( - icon: const Icon(Icons.add_a_photo), - onPressed: () => models.video.saveFrame(widget.name), - ), - IconButton( - icon: const Icon(Icons.settings), - onPressed: () async => showDialog( - context: context, - builder: (_) => CameraDetailsEditor(data), - ), - ), - ViewsSelector(currentView: widget.name.humanName), - ], - ), - Positioned(left: 5, bottom: 5, child: Text(data.details.name.humanName)), - ], - ); - - /// Displays an error message describing why `image == null`. - String get errorMessage { - if (!models.sockets.video.isConnected) return "The video program is not connected"; - switch (data.details.status) { - case CameraStatus.CAMERA_LOADING: return "Camera is loading..."; - case CameraStatus.CAMERA_STATUS_UNDEFINED: return "Unknown error"; - case CameraStatus.CAMERA_DISCONNECTED: return "Camera is not connected"; - case CameraStatus.CAMERA_DISABLED: return "Camera is disabled.\nClick the settings icon to enabled it."; - case CameraStatus.CAMERA_NOT_RESPONDING: return "Camera is not responding"; - case CameraStatus.FRAME_TOO_LARGE: return "Camera is reading too much detail\nReduce the quality or resolution"; - case CameraStatus.CAMERA_HAS_NO_NAME: return "Camera has no name\nChange lib/constants.py on the video Pi"; - case CameraStatus.CAMERA_ENABLED: - if (data.hasFrame()) { return "Loading feed..."; } - else { return "Starting camera..."; } - } - return "Unknown error"; - } + Widget build(BuildContext context) => ListView( + children: [ + SliderSettings( + label: "Zoom", + value: zoom, + min: 100, + max: 800, + onChanged: (val) async { + setState(() => zoom = val); + await models.video.updateCamera( + verify: false, + widget.id, + CameraDetails(name: widget.details.name, zoom: val.round()), + ); + }, + ), + SliderSettings( + label: "Pan", + value: pan, + min: -180, + max: 180, + onChanged: (val) async { + setState(() => pan = val); + await models.video.updateCamera( + verify: false, + widget.id, + CameraDetails(name: widget.details.name, pan: val.round()), + ); + }, + ), + SliderSettings( + label: "Tilt", + value: tilt, + min: -180, + max: 180, + onChanged: (val) async { + setState(() => tilt = val); + await models.video.updateCamera( + verify: false, + widget.id, + CameraDetails(name: widget.details.name, tilt: val.round()), + ); + }, + ), + SliderSettings( + label: "Focus", + value: focus, + max: 255, + onChanged: (val) async { + setState(() => focus = val); + await models.video.updateCamera( + verify: false, + widget.id, + CameraDetails(name: widget.details.name, focus: val.round()), + ); + }, + ), + // Need to debug bool conversion to 1.0 and 2.0 + SwitchListTile( + title: const Text("Autofocus"), + value: autofocus, + onChanged: (bool value) async { + setState(() => autofocus = value); + await models.video.updateCamera( + verify: false, + widget.id, + CameraDetails(name: widget.details.name, autofocus: autofocus), + ); + }, + ), + ], + ); } diff --git a/pubspec.lock b/pubspec.lock index 097dadd7f2..eb59ed87ed 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -5,10 +5,10 @@ packages: dependency: transitive description: name: archive - sha256: ecf4273855368121b1caed0d10d4513c7241dfc813f7d3c8933b36622ae9b265 + sha256: "6bd38d335f0954f5fad9c79e614604fbf03a0e5b975923dd001b6ea965ef5b4b" url: "https://pub.dev" source: hosted - version: "3.5.1" + version: "3.6.0" args: dependency: transitive description: @@ -253,10 +253,10 @@ packages: dependency: transitive description: name: image - sha256: "4c68bfd5ae83e700b5204c1e74451e7bf3cf750e6843c6e158289cf56bda018e" + sha256: "2237616a36c0d69aef7549ab439b833fb7f9fb9fc861af2cc9ac3eedddd69ca8" url: "https://pub.dev" source: hosted - version: "4.1.7" + version: "4.2.0" json_annotation: dependency: transitive description: @@ -682,10 +682,10 @@ packages: dependency: transitive description: name: win32 - sha256: "0eaf06e3446824099858367950a813472af675116bf63f008a4c2a75ae13e9cb" + sha256: a79dbe579cb51ecd6d30b17e0cae4e0ea15e2c0e66f69ad4198f22a6789e94f4 url: "https://pub.dev" source: hosted - version: "5.5.0" + version: "5.5.1" win32_gamepad: dependency: "direct main" description: @@ -719,5 +719,5 @@ packages: source: hosted version: "3.1.2" sdks: - dart: ">=3.3.0 <4.0.0" + dart: ">=3.4.0 <4.0.0" flutter: ">=3.19.0"