From f6de9815708b7598cb71fda0d8f8ee86947a74d3 Mon Sep 17 00:00:00 2001 From: Stephen Higley Date: Thu, 7 Sep 2023 09:11:21 -0400 Subject: [PATCH 1/4] Wired up applyconstraints for zoomfactor --- ios/RCTWebRTC/VideoCaptureController.h | 1 + ios/RCTWebRTC/VideoCaptureController.m | 17 +++++++++++++++-- ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m | 9 +++++++++ src/MediaStreamTrack.ts | 12 ++++++++++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ios/RCTWebRTC/VideoCaptureController.h b/ios/RCTWebRTC/VideoCaptureController.h index 42ddea4d5..49852d9cf 100644 --- a/ios/RCTWebRTC/VideoCaptureController.h +++ b/ios/RCTWebRTC/VideoCaptureController.h @@ -13,5 +13,6 @@ - (void)startCapture; - (void)stopCapture; - (void)switchCamera; +- (void)applyZoomFactor:(CGFloat)zoomFactor; @end diff --git a/ios/RCTWebRTC/VideoCaptureController.m b/ios/RCTWebRTC/VideoCaptureController.m index f203afa06..fd8b31500 100644 --- a/ios/RCTWebRTC/VideoCaptureController.m +++ b/ios/RCTWebRTC/VideoCaptureController.m @@ -73,7 +73,7 @@ - (void)startCapture { return; } - + AVCaptureDeviceFormat *format = [self selectFormatForDevice:self.device withTargetWidth:self.width withTargetHeight:self.height]; @@ -84,7 +84,7 @@ - (void)startCapture { } self.selectedFormat = format; - + RCTLog(@"[VideoCaptureController] Capture will start"); // Starting the capture happens on another thread. Wait for it. @@ -135,6 +135,19 @@ - (void)switchCamera { [self startCapture]; } +- (void)applyZoomFactor:(CGFloat)zoomFactor { + if (!self.device) { + RCTLogWarn(@"[VideoCaptureController] No capture devices found!"); + return; + } + + CGFloat minZoomFactor = self.device.minAvailableVideoZoomFactor; + CGFloat maxZoomFactor = self.device.maxAvailableVideoZoomFactor; + CGFloat clampedZoomFactor = MAX(minZoomFactor, MIN(zoomFactor, maxZoomFactor)); + + [self.device setVideoZoomFactor:clampedZoomFactor]; +} + #pragma mark NSKeyValueObserving - (void)observeValueForKeyPath:(NSString *)keyPath diff --git a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m index b80663914..d3dbb2e28 100644 --- a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +++ b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m @@ -374,6 +374,15 @@ - (RTCVideoTrack *)createScreenCaptureVideoTrack { } } +RCT_EXPORT_METHOD(mediaStreamTrackApplyConstraints: (nonnull NSNumber *)pcId : (nonnull NSString *)trackID : (NSDictionary *)constraints) { + RTCMediaStreamTrack *track = [self trackForId:trackID pcId:pcId]; + if (track && [track.kind isEqualToString:@"video"]) { + CGFloat zoomFactor = [constraints[@"zoom"] floatValue]; + RTCVideoTrack *videoTrack = (RTCVideoTrack *)track; + [(VideoCaptureController *)videoTrack.captureController applyZoomFactor:zoomFactor]; + } +} + #pragma mark - Helpers - (RTCMediaStreamTrack *)trackForId:(nonnull NSString *)trackId pcId:(nonnull NSNumber *)pcId { diff --git a/src/MediaStreamTrack.ts b/src/MediaStreamTrack.ts index 574e028a5..979bc3c15 100644 --- a/src/MediaStreamTrack.ts +++ b/src/MediaStreamTrack.ts @@ -135,8 +135,16 @@ class MediaStreamTrack extends defineCustomEventTarget(...MEDIA_STREAM_TRACK_EVE WebRTCModule.mediaStreamTrackSetVolume(this.remote ? this._peerConnectionId : -1, this.id, volume); } - applyConstraints(): never { - throw new Error('Not implemented.'); + applyConstraints(constraints: object) { + if (this.kind !== 'video') { + throw new Error('Only implemented for video tracks'); + } + + if (!('zoom' in constraints)) { + log.warn('Only zoom is supported for applyConstraints on video tracks'); + } + + WebRTCModule.mediaStreamTrackApplyConstraints(this.remote ? this._peerConnectionId : -1, this.id, constraints); } clone(): never { From 9350a3faf84086b49dfc5856d06845b620434ef3 Mon Sep 17 00:00:00 2001 From: Stephen Higley Date: Mon, 2 Oct 2023 11:45:41 -0400 Subject: [PATCH 2/4] Mocked up android and lock device before applying zoom --- .../WebRTCModule/CameraCaptureController.java | 8 ++++++++ .../com/oney/WebRTCModule/WebRTCModule.java | 18 ++++++++++++++++++ ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m | 6 +++--- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/android/src/main/java/com/oney/WebRTCModule/CameraCaptureController.java b/android/src/main/java/com/oney/WebRTCModule/CameraCaptureController.java index 0faff5d44..1f37a4867 100644 --- a/android/src/main/java/com/oney/WebRTCModule/CameraCaptureController.java +++ b/android/src/main/java/com/oney/WebRTCModule/CameraCaptureController.java @@ -70,6 +70,14 @@ public void onCameraSwitchError(String s) { } } + public void applyZoomFactor(float zoomFactor) { + if (!(videoCapturer instanceof CameraVideoCapturer)) { + return; + } + + CameraVideoCapturer capturer = (CameraVideoCapturer) videoCapturer; + } + @Override protected VideoCapturer createVideoCapturer() { String deviceId = ReactBridgeUtil.getMapStrValue(this.constraints, "deviceId"); diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java index 4483f64dc..566556e27 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java @@ -837,6 +837,24 @@ public void mediaStreamTrackSetVolume(int pcId, String id, double volume) { }); } + @ReactMethod + public void mediaStreamTrackApplyConstraints(int pcId, String id, ReadableMap constraints) { + ThreadUtils.runOnExecutor(() -> { + MediaStreamTrack track = getTrack(pcId, id); + if (track == null) { + Log.d(TAG, "mediaStreamTrackApplyConstraints() could not find track " + id); + return; + } + + if (!(track instanceof VideoTrack)) { + Log.d(TAG, "mediaStreamTrackApplyConstraints() track is not a VideoTrack!"); + return; + } + + VideoTrack videoTrack = (VideoTrack)track; + }); + } + /** * This serializes the transceivers current direction and mid and returns them * for update when an sdp negotiation/renegotiation happens diff --git a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m index d3dbb2e28..931ee1449 100644 --- a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +++ b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m @@ -56,7 +56,7 @@ - (NSArray *)createMediaStream:(NSArray *)tracks { NSString *mediaStreamId = [[NSUUID UUID] UUIDString]; RTCMediaStream *mediaStream = [self.peerConnectionFactory mediaStreamWithStreamId:mediaStreamId]; NSMutableArray *trackInfos = [NSMutableArray array]; - + for (RTCMediaStreamTrack *track in tracks) { if ([track.kind isEqualToString:@"audio"]) { [mediaStream addAudioTrack:(RTCAudioTrack *)track]; @@ -182,7 +182,7 @@ - (RTCVideoTrack *)createScreenCaptureVideoTrack { if (constraints[@"video"]) { videoTrack = [self createVideoTrack:constraints]; } - + if (audioTrack == nil && videoTrack == nil) { // Fail with DOMException with name AbortError as per: // https://www.w3.org/TR/mediacapture-streams/#dom-mediadevices-getusermedia @@ -205,7 +205,7 @@ - (RTCVideoTrack *)createScreenCaptureVideoTrack { } else if ([track.kind isEqualToString:@"video"]) { [mediaStream addVideoTrack:(RTCVideoTrack *)track]; } - + NSString *trackId = track.trackId; self.localTracks[trackId] = track; From da57c87702e77658e327acc6c657606659d48137 Mon Sep 17 00:00:00 2001 From: Stephen Higley Date: Mon, 2 Oct 2023 11:54:26 -0400 Subject: [PATCH 3/4] actually commit code for locking device before applying zoom --- ios/RCTWebRTC/VideoCaptureController.m | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ios/RCTWebRTC/VideoCaptureController.m b/ios/RCTWebRTC/VideoCaptureController.m index fd8b31500..9abf06c17 100644 --- a/ios/RCTWebRTC/VideoCaptureController.m +++ b/ios/RCTWebRTC/VideoCaptureController.m @@ -145,7 +145,16 @@ - (void)applyZoomFactor:(CGFloat)zoomFactor { CGFloat maxZoomFactor = self.device.maxAvailableVideoZoomFactor; CGFloat clampedZoomFactor = MAX(minZoomFactor, MIN(zoomFactor, maxZoomFactor)); + NSError *error = nil; + [self.device lockForConfiguration:&error]; + if (error) { + RCTLog(@"[VideoCaptureController] Could not lock device for configuration: %@", error); + return; + } + [self.device setVideoZoomFactor:clampedZoomFactor]; + + [self.device unlockForConfiguration]; } #pragma mark NSKeyValueObserving From 9061213e7a8b15f0651e9a5aa2f05e472a53ae20 Mon Sep 17 00:00:00 2001 From: Stephen Higley Date: Mon, 4 Dec 2023 13:56:42 -0500 Subject: [PATCH 4/4] added ability to get zoom range via getCapabilities on the videoTrack --- ios/RCTWebRTC/VideoCaptureController.h | 1 + ios/RCTWebRTC/VideoCaptureController.m | 13 +++++++++++++ ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m | 19 +++++++++++++++++++ src/MediaStreamTrack.ts | 9 +++++++-- 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/ios/RCTWebRTC/VideoCaptureController.h b/ios/RCTWebRTC/VideoCaptureController.h index 49852d9cf..c52002c0b 100644 --- a/ios/RCTWebRTC/VideoCaptureController.h +++ b/ios/RCTWebRTC/VideoCaptureController.h @@ -14,5 +14,6 @@ - (void)stopCapture; - (void)switchCamera; - (void)applyZoomFactor:(CGFloat)zoomFactor; +- (NSDictionary *)getZoomCapabilities; @end diff --git a/ios/RCTWebRTC/VideoCaptureController.m b/ios/RCTWebRTC/VideoCaptureController.m index 9abf06c17..4ced05f85 100644 --- a/ios/RCTWebRTC/VideoCaptureController.m +++ b/ios/RCTWebRTC/VideoCaptureController.m @@ -157,6 +157,19 @@ - (void)applyZoomFactor:(CGFloat)zoomFactor { [self.device unlockForConfiguration]; } +- (NSDictionary *)getZoomCapabilities { + if (!self.device) { + RCTLogWarn(@"[VideoCaptureController] No capture devices found!"); + return @{}; + } + + return @{ + @"min" : @(self.device.minAvailableVideoZoomFactor), + @"max" : @(self.device.maxAvailableVideoZoomFactor), + @"step" : @(0.1) + }; +} + #pragma mark NSKeyValueObserving - (void)observeValueForKeyPath:(NSString *)keyPath diff --git a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m index 931ee1449..84a304972 100644 --- a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +++ b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m @@ -1,4 +1,5 @@ #import +#import #import #import @@ -383,6 +384,24 @@ - (RTCVideoTrack *)createScreenCaptureVideoTrack { } } +RCT_EXPORT_METHOD(mediaStreamTrackGetCapabilities: (nonnull NSNumber *)pcId : (nonnull NSString *)trackID : + (RCTResponseSenderBlock)callback) { + RTCMediaStreamTrack *track = [self trackForId:trackID pcId:pcId]; + + if (track && [track.kind isEqualToString:@"video"]) { + RTCVideoTrack *videoTrack = (RTCVideoTrack*)track; + NSDictionary *zoomCapabilities = [(VideoCaptureController *)videoTrack.captureController getZoomCapabilities]; + + callback(@[@{ + @"zoom" : zoomCapabilities + }]); + return; + } + + RCTLogWarn(@"getCapabilities only supported for video track"); + callback(@[]); +} + #pragma mark - Helpers - (RTCMediaStreamTrack *)trackForId:(nonnull NSString *)trackId pcId:(nonnull NSNumber *)pcId { diff --git a/src/MediaStreamTrack.ts b/src/MediaStreamTrack.ts index 979bc3c15..1ad10aeb3 100644 --- a/src/MediaStreamTrack.ts +++ b/src/MediaStreamTrack.ts @@ -144,6 +144,7 @@ class MediaStreamTrack extends defineCustomEventTarget(...MEDIA_STREAM_TRACK_EVE log.warn('Only zoom is supported for applyConstraints on video tracks'); } + this._constraints = Object.assign(this._constraints, constraints); WebRTCModule.mediaStreamTrackApplyConstraints(this.remote ? this._peerConnectionId : -1, this.id, constraints); } @@ -151,8 +152,12 @@ class MediaStreamTrack extends defineCustomEventTarget(...MEDIA_STREAM_TRACK_EVE throw new Error('Not implemented.'); } - getCapabilities(): never { - throw new Error('Not implemented.'); + getCapabilities(): Promise { + if (this.kind !== 'video') { + throw new Error('Only implemented for video tracks'); + } + + return new Promise((resolve) => WebRTCModule.mediaStreamTrackGetCapabilities(this.remote ? this._peerConnectionId : -1, this.id, resolve)); } getConstraints() {