From bc161921a48c752c4143e9ca4527c57807ed0049 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Thu, 18 Apr 2024 13:11:45 -0700 Subject: [PATCH 01/15] feat: add no light challenge implementation (#127) * feat: add no light challenge implementation * update package.swift for CI build * Fix unit tests * Address review comments --- HostApp/HostApp.xcodeproj/project.pbxproj | 2 - .../xcshareddata/swiftpm/Package.resolved | 8 +- .../HostApp/Views/ExampleLivenessView.swift | 3 +- Package.resolved | 8 +- Package.swift | 3 +- .../BlazeFace/DetectedFace.swift | 6 +- .../FaceDetectorShortRange+Model.swift | 15 +++- .../FaceDetection/FaceDetector.swift | 5 ++ .../Views/GetReadyPage/GetReadyPageView.swift | 46 ++++++++---- .../InstructionContainerView.swift | 27 +++++-- .../Liveness/FaceLivenessDetectionView.swift | 75 ++++++++++++++----- ...ViewModel+FaceDetectionResultHandler.swift | 34 ++++++--- ...ctionViewModel+VideoSegmentProcessor.swift | 4 +- .../FaceLivenessDetectionViewModel.swift | 38 ++++++++-- .../FaceLivenessViewControllerPresenter.swift | 1 + .../Views/Liveness/LivenessStateMachine.swift | 6 ++ .../Liveness/LivenessViewController.swift | 7 ++ .../Views/LoadingPage/LoadingPageView.swift | 27 +++++++ .../FaceLivenessTests/DetectedFaceTests.swift | 26 ++++++- Tests/FaceLivenessTests/LivenessTests.swift | 33 ++++++-- .../MockLivenessService.swift | 12 ++- 21 files changed, 305 insertions(+), 81 deletions(-) create mode 100644 Sources/FaceLiveness/Views/LoadingPage/LoadingPageView.swift diff --git a/HostApp/HostApp.xcodeproj/project.pbxproj b/HostApp/HostApp.xcodeproj/project.pbxproj index f318bc2d..81ef8e10 100644 --- a/HostApp/HostApp.xcodeproj/project.pbxproj +++ b/HostApp/HostApp.xcodeproj/project.pbxproj @@ -308,8 +308,6 @@ Base, ); mainGroup = 9070FF97285112B4009867D5; - packageReferences = ( - ); productRefGroup = 9070FFA1285112B4009867D5 /* Products */; projectDirPath = ""; projectRoot = ""; diff --git a/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 61218e39..cddb5116 100644 --- a/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { - "revision" : "7846328106dba471b3fb35170155e92aad50d427", - "version" : "2.33.3" + "branch" : "feat/no-light-support", + "revision" : "7c1fa2f7a766208f5af69ca8dce5fd02e6de4db6" } }, { @@ -50,8 +50,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/stephencelis/SQLite.swift.git", "state" : { - "revision" : "e78ae0220e17525a15ac68c697a155eb7a672a8e", - "version" : "0.15.0" + "revision" : "5f5ad81ac0d0a0f3e56e39e646e8423c617df523", + "version" : "0.13.2" } }, { diff --git a/HostApp/HostApp/Views/ExampleLivenessView.swift b/HostApp/HostApp/Views/ExampleLivenessView.swift index 39407c89..85eaf8cc 100644 --- a/HostApp/HostApp/Views/ExampleLivenessView.swift +++ b/HostApp/HostApp/Views/ExampleLivenessView.swift @@ -22,7 +22,8 @@ struct ExampleLivenessView: View { case .liveness: FaceLivenessDetectorView( sessionID: viewModel.sessionID, - region: "us-east-1", + // TODO: Change before merging to main + region: "us-west-2", isPresented: Binding( get: { viewModel.presentationState == .liveness }, set: { _ in } diff --git a/Package.resolved b/Package.resolved index e9515eee..a046c257 100644 --- a/Package.resolved +++ b/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { - "revision" : "dbc4a0412f4b5cd96f3e756e78bbd1e8e0a35a2f", - "version" : "2.35.4" + "branch" : "feat/no-light-support", + "revision" : "7c1fa2f7a766208f5af69ca8dce5fd02e6de4db6" } }, { @@ -50,8 +50,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/stephencelis/SQLite.swift.git", "state" : { - "revision" : "a95fc6df17d108bd99210db5e8a9bac90fe984b8", - "version" : "0.15.3" + "revision" : "5f5ad81ac0d0a0f3e56e39e646e8423c617df523", + "version" : "0.13.2" } }, { diff --git a/Package.swift b/Package.swift index ee5353c6..446f12c0 100644 --- a/Package.swift +++ b/Package.swift @@ -13,7 +13,8 @@ let package = Package( targets: ["FaceLiveness"]), ], dependencies: [ - .package(url: "https://github.com/aws-amplify/amplify-swift", exact: "2.35.4") + // TODO: Change this before merge to main + .package(url: "https://github.com/aws-amplify/amplify-swift", branch: "feat/no-light-support") ], targets: [ .target( diff --git a/Sources/FaceLiveness/FaceDetection/BlazeFace/DetectedFace.swift b/Sources/FaceLiveness/FaceDetection/BlazeFace/DetectedFace.swift index d6879848..1d62b263 100644 --- a/Sources/FaceLiveness/FaceDetection/BlazeFace/DetectedFace.swift +++ b/Sources/FaceLiveness/FaceDetection/BlazeFace/DetectedFace.swift @@ -6,6 +6,7 @@ // import Foundation +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin struct DetectedFace { var boundingBox: CGRect @@ -19,7 +20,8 @@ struct DetectedFace { let confidence: Float - func boundingBoxFromLandmarks(ovalRect: CGRect) -> CGRect { + func boundingBoxFromLandmarks(ovalRect: CGRect, + ovalMatchChallenge: FaceLivenessSession.OvalMatchChallenge) -> CGRect { let alpha = 2.0 let gamma = 1.8 let ow = (alpha * pupilDistance + gamma * faceHeight) / 2 @@ -34,7 +36,7 @@ struct DetectedFace { } let faceWidth = ow - let faceHeight = 1.618 * faceWidth + let faceHeight = ovalMatchChallenge.oval.heightWidthRatio * faceWidth let faceBoxBottom = boundingBox.maxY let faceBoxTop = faceBoxBottom - faceHeight let faceBoxLeft = min(cx - ow / 2, rightEar.x) diff --git a/Sources/FaceLiveness/FaceDetection/BlazeFace/FaceDetectorShortRange+Model.swift b/Sources/FaceLiveness/FaceDetection/BlazeFace/FaceDetectorShortRange+Model.swift index d9430720..100f0418 100644 --- a/Sources/FaceLiveness/FaceDetection/BlazeFace/FaceDetectorShortRange+Model.swift +++ b/Sources/FaceLiveness/FaceDetection/BlazeFace/FaceDetectorShortRange+Model.swift @@ -12,6 +12,7 @@ import Accelerate import CoreGraphics import CoreImage import VideoToolbox +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin enum FaceDetectorShortRange {} @@ -33,11 +34,16 @@ extension FaceDetectorShortRange { ) } + weak var faceDetectionSessionConfiguration: FaceDetectionSessionConfigurationWrapper? weak var detectionResultHandler: FaceDetectionResultHandler? func setResultHandler(detectionResultHandler: FaceDetectionResultHandler) { self.detectionResultHandler = detectionResultHandler } + + func setFaceDetectionSessionConfigurationWrapper(configuration: FaceDetectionSessionConfigurationWrapper) { + self.faceDetectionSessionConfiguration = configuration + } func detectFaces(from buffer: CVPixelBuffer) { let faces = prediction(for: buffer) @@ -105,10 +111,17 @@ extension FaceDetectorShortRange { count: confidenceScoresCapacity ) ) + + let blazeFaceDetectionThreshold: Float + if let sessionConfiguration = faceDetectionSessionConfiguration?.sessionConfiguration { + blazeFaceDetectionThreshold = Float(sessionConfiguration.ovalMatchChallenge.faceDetectionThreshold) + } else { + blazeFaceDetectionThreshold = confidenceScoreThreshold + } var passingConfidenceScoresIndices = confidenceScores .enumerated() - .filter { $0.element >= confidenceScoreThreshold } + .filter { $0.element >= blazeFaceDetectionThreshold} .sorted(by: { $0.element > $1.element }) diff --git a/Sources/FaceLiveness/FaceDetection/FaceDetector.swift b/Sources/FaceLiveness/FaceDetection/FaceDetector.swift index 3801eeab..1afb90c1 100644 --- a/Sources/FaceLiveness/FaceDetection/FaceDetector.swift +++ b/Sources/FaceLiveness/FaceDetection/FaceDetector.swift @@ -6,6 +6,7 @@ // import AVFoundation +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin protocol FaceDetector { func detectFaces(from buffer: CVPixelBuffer) @@ -16,6 +17,10 @@ protocol FaceDetectionResultHandler: AnyObject { func process(newResult: FaceDetectionResult) } +protocol FaceDetectionSessionConfigurationWrapper: AnyObject { + var sessionConfiguration: FaceLivenessSession.SessionConfiguration? { get } +} + enum FaceDetectionResult { case noFace case singleFace(DetectedFace) diff --git a/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift b/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift index 00ecb9b7..dadb2076 100644 --- a/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift +++ b/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift @@ -6,35 +6,49 @@ // import SwiftUI +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin struct GetReadyPageView: View { let beginCheckButtonDisabled: Bool let onBegin: () -> Void - + let challenge: Challenge + init( onBegin: @escaping () -> Void, - beginCheckButtonDisabled: Bool = false + beginCheckButtonDisabled: Bool = false, + challenge: Challenge ) { self.onBegin = onBegin self.beginCheckButtonDisabled = beginCheckButtonDisabled + self.challenge = challenge } var body: some View { VStack { ZStack { CameraPreviewView() - VStack { - WarningBox( - titleText: LocalizedStrings.get_ready_photosensitivity_title, - bodyText: LocalizedStrings.get_ready_photosensitivity_description, - popoverContent: { photosensitivityWarningPopoverContent } - ) - .accessibilityElement(children: .combine) - Text(LocalizedStrings.preview_center_your_face_text) - .font(.title) - .multilineTextAlignment(.center) - Spacer() - }.padding() + switch self.challenge.type { + case .faceMovementChallenge: + VStack { + Text(LocalizedStrings.preview_center_your_face_text) + .font(.title) + .multilineTextAlignment(.center) + Spacer() + }.padding() + case . faceMovementAndLightChallenge: + VStack { + WarningBox( + titleText: LocalizedStrings.get_ready_photosensitivity_title, + bodyText: LocalizedStrings.get_ready_photosensitivity_description, + popoverContent: { photosensitivityWarningPopoverContent } + ) + .accessibilityElement(children: .combine) + Text(LocalizedStrings.preview_center_your_face_text) + .font(.title) + .multilineTextAlignment(.center) + Spacer() + }.padding() + } } beginCheckButton } @@ -72,6 +86,8 @@ struct GetReadyPageView: View { struct GetReadyPageView_Previews: PreviewProvider { static var previews: some View { - GetReadyPageView(onBegin: {}) + GetReadyPageView(onBegin: {}, + challenge: .init(version: "2.0.0", + type: .faceMovementAndLightChallenge)) } } diff --git a/Sources/FaceLiveness/Views/Instruction/InstructionContainerView.swift b/Sources/FaceLiveness/Views/Instruction/InstructionContainerView.swift index ff02a3d6..5ed45ae7 100644 --- a/Sources/FaceLiveness/Views/Instruction/InstructionContainerView.swift +++ b/Sources/FaceLiveness/Views/Instruction/InstructionContainerView.swift @@ -7,6 +7,7 @@ import SwiftUI import Combine +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin struct InstructionContainerView: View { @ObservedObject var viewModel: FaceLivenessDetectionViewModel @@ -97,13 +98,29 @@ struct InstructionContainerView: View { argument: LocalizedStrings.challenge_verifying ) } - case .faceMatched: + case .completedNoLightCheck: InstructionView( - text: LocalizedStrings.challenge_instruction_hold_still, - backgroundColor: .livenessPrimaryBackground, - textColor: .livenessPrimaryLabel, - font: .title + text: LocalizedStrings.challenge_verifying, + backgroundColor: .livenessBackground ) + .onAppear { + UIAccessibility.post( + notification: .announcement, + argument: LocalizedStrings.challenge_verifying + ) + } + case .faceMatched: + if let challenge = viewModel.challenge, + case .faceMovementAndLightChallenge = challenge.type { + InstructionView( + text: LocalizedStrings.challenge_instruction_hold_still, + backgroundColor: .livenessPrimaryBackground, + textColor: .livenessPrimaryLabel, + font: .title + ) + } else { + EmptyView() + } default: EmptyView() } diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index bc4e47e8..eff46ec7 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -16,10 +16,11 @@ import Amplify public struct FaceLivenessDetectorView: View { @StateObject var viewModel: FaceLivenessDetectionViewModel @Binding var isPresented: Bool - @State var displayState: DisplayState = .awaitingCameraPermission + @State var displayState: DisplayState = .awaitingChallengeType @State var displayingCameraPermissionsNeededAlert = false let disableStartView: Bool + let facelivenessDetectorViewId: String let onCompletion: (Result) -> Void let sessionTask: Task @@ -32,6 +33,8 @@ public struct FaceLivenessDetectorView: View { isPresented: Binding, onCompletion: @escaping (Result) -> Void ) { + let viewId = UUID().uuidString + self.facelivenessDetectorViewId = viewId self.disableStartView = disableStartView self._isPresented = isPresented self.onCompletion = onCompletion @@ -41,7 +44,8 @@ public struct FaceLivenessDetectorView: View { withID: sessionID, credentialsProvider: credentialsProvider, region: region, - options: .init(), + options: .init(faceLivenessDetectorViewId: viewId, + preCheckViewEnabled: !disableStartView), completion: map(detectionCompletion: onCompletion) ) return session @@ -82,6 +86,8 @@ public struct FaceLivenessDetectorView: View { sessionID: sessionID ) ) + + faceDetector.setFaceDetectionSessionConfigurationWrapper(configuration: viewModel) } init( @@ -93,6 +99,8 @@ public struct FaceLivenessDetectorView: View { onCompletion: @escaping (Result) -> Void, captureSession: LivenessCaptureSession ) { + let viewId = UUID().uuidString + self.facelivenessDetectorViewId = viewId self.disableStartView = disableStartView self._isPresented = isPresented self.onCompletion = onCompletion @@ -102,7 +110,8 @@ public struct FaceLivenessDetectorView: View { withID: sessionID, credentialsProvider: credentialsProvider, region: region, - options: .init(), + options: .init(faceLivenessDetectorViewId: viewId, + preCheckViewEnabled: !disableStartView), completion: map(detectionCompletion: onCompletion) ) return session @@ -126,32 +135,44 @@ public struct FaceLivenessDetectorView: View { public var body: some View { switch displayState { - case .awaitingLivenessSession: + case .awaitingChallengeType: + LoadingPageView() + .onAppear { + Task { + do { + let session = try await sessionTask.value + viewModel.livenessService = session + viewModel.registerServiceEvents(onChallengeTypeReceived: { challenge in + self.displayState = DisplayState.awaitingLivenessSession(challenge) + }) + viewModel.initializeLivenessStream() + } catch { + throw FaceLivenessDetectionError.accessDenied + } + } + } + case .awaitingLivenessSession(let challenge): Color.clear .onAppear { Task { do { let newState = disableStartView ? DisplayState.displayingLiveness - : DisplayState.displayingGetReadyView + : DisplayState.displayingGetReadyView(challenge) guard self.displayState != newState else { return } - let session = try await sessionTask.value - viewModel.livenessService = session - viewModel.registerServiceEvents() self.displayState = newState - } catch { - throw FaceLivenessDetectionError.accessDenied } } } - case .displayingGetReadyView: + case .displayingGetReadyView(let challenge): GetReadyPageView( onBegin: { guard displayState != .displayingLiveness else { return } displayState = .displayingLiveness }, - beginCheckButtonDisabled: false + beginCheckButtonDisabled: false, + challenge: challenge ) .onAppear { DispatchQueue.main.async { @@ -215,7 +236,8 @@ public struct FaceLivenessDetectorView: View { for: .video, completionHandler: { accessGranted in guard accessGranted == true else { return } - displayState = .awaitingLivenessSession + guard let challenge = viewModel.challenge else { return } + displayState = .awaitingLivenessSession(challenge) } ) @@ -233,18 +255,37 @@ public struct FaceLivenessDetectorView: View { case .restricted, .denied: alertCameraAccessNeeded() case .authorized: - displayState = .awaitingLivenessSession + guard let challenge = viewModel.challenge else { return } + displayState = .awaitingLivenessSession(challenge) @unknown default: break } } } -enum DisplayState { - case awaitingLivenessSession - case displayingGetReadyView +enum DisplayState: Equatable { + case awaitingChallengeType + case awaitingLivenessSession(Challenge) + case displayingGetReadyView(Challenge) case displayingLiveness case awaitingCameraPermission + + static func == (lhs: DisplayState, rhs: DisplayState) -> Bool { + switch (lhs, rhs) { + case (.awaitingChallengeType, .awaitingChallengeType): + return true + case (let .awaitingLivenessSession(c1), let .awaitingLivenessSession(c2)): + return c1.type == c2.type && c1.version == c2.version + case (let .displayingGetReadyView(c1), let .displayingGetReadyView(c2)): + return c1.type == c2.type && c1.version == c2.version + case (.displayingLiveness, .displayingLiveness): + return true + case (.awaitingCameraPermission, .awaitingCameraPermission): + return true + default: + return false + } + } } enum InstructionState { diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+FaceDetectionResultHandler.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+FaceDetectionResultHandler.swift index 99e92ee2..edb96e80 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+FaceDetectionResultHandler.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+FaceDetectionResultHandler.swift @@ -28,14 +28,15 @@ extension FaceLivenessDetectionViewModel: FaceDetectionResultHandler { } case .singleFace(let face): var normalizedFace = normalizeFace(face) - normalizedFace.boundingBox = normalizedFace.boundingBoxFromLandmarks(ovalRect: ovalRect) + guard let sessionConfiguration = sessionConfiguration else { return } + normalizedFace.boundingBox = normalizedFace.boundingBoxFromLandmarks(ovalRect: ovalRect, + ovalMatchChallenge: sessionConfiguration.ovalMatchChallenge) switch livenessState.state { case .pendingFacePreparedConfirmation: - if face.faceDistance <= initialFaceDistanceThreshold { + if face.faceDistance <= sessionConfiguration.ovalMatchChallenge.face.distanceThreshold { DispatchQueue.main.async { self.livenessState.awaitingRecording() - self.initializeLivenessStream() } DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { self.livenessState.beginRecording() @@ -55,7 +56,6 @@ extension FaceLivenessDetectionViewModel: FaceDetectionResultHandler { ) }) case .recording(ovalDisplayed: true): - guard let sessionConfiguration = sessionConfiguration else { return } let instruction = faceInOvalMatching.faceMatchState( for: normalizedFace.boundingBox, in: ovalRect, @@ -64,18 +64,18 @@ extension FaceLivenessDetectionViewModel: FaceDetectionResultHandler { handleInstruction( instruction, - colorSequences: sessionConfiguration.colorChallenge.colors + colorSequences: sessionConfiguration.colorChallenge?.colors ) case .awaitingFaceInOvalMatch: - guard let sessionConfiguration = sessionConfiguration else { return } let instruction = faceInOvalMatching.faceMatchState( for: normalizedFace.boundingBox, in: ovalRect, challengeConfig: sessionConfiguration.ovalMatchChallenge ) + handleInstruction( instruction, - colorSequences: sessionConfiguration.colorChallenge.colors + colorSequences: sessionConfiguration.colorChallenge?.colors ) default: break @@ -104,16 +104,30 @@ extension FaceLivenessDetectionViewModel: FaceDetectionResultHandler { func handleInstruction( _ instruction: Instructor.Instruction, - colorSequences: [FaceLivenessSession.DisplayColor] + colorSequences: [FaceLivenessSession.DisplayColor]? ) { DispatchQueue.main.async { switch instruction { case .match: self.livenessState.faceMatched() self.faceMatchedTimestamp = Date().timestampMilliseconds - DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { - self.livenessViewControllerDelegate?.displayFreshness(colorSequences: colorSequences) + + // next step after face match + switch self.challenge?.type { + case .faceMovementAndLightChallenge: + if let colorSequences = colorSequences { + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { + self.livenessViewControllerDelegate?.displayFreshness(colorSequences: colorSequences) + } + } + case .faceMovementChallenge: + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { + self.livenessViewControllerDelegate?.completeNoLightCheck() + } + default: + break } + let generator = UINotificationFeedbackGenerator() generator.notificationOccurred(.success) self.noFitStartTime = nil diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+VideoSegmentProcessor.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+VideoSegmentProcessor.swift index c2ed2b39..d2f88343 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+VideoSegmentProcessor.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+VideoSegmentProcessor.swift @@ -11,8 +11,8 @@ extension FaceLivenessDetectionViewModel: VideoSegmentProcessor { func process(initalSegment: Data, currentSeparableSegment: Data) { let chunk = chunk(initial: initalSegment, current: currentSeparableSegment) sendVideoEvent(data: chunk, videoEventTime: .zero) - if !hasSentFinalVideoEvent, - case .completedDisplayingFreshness = livenessState.state { + if !hasSentFinalVideoEvent && + (livenessState.state == .completedDisplayingFreshness || livenessState.state == .completedNoLightCheck) { DispatchQueue.global(qos: .default).asyncAfter(deadline: .now() + 0.9) { self.sendFinalVideoEvent() } diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift index b1a95f36..352b1855 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift @@ -33,6 +33,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { var hasSentFirstVideo = false var layerRectConverted: (CGRect) -> CGRect = { $0 } var sessionConfiguration: FaceLivenessSession.SessionConfiguration? + var challenge: Challenge? var normalizeFace: (DetectedFace) -> DetectedFace = { $0 } var provideSingleFrame: ((UIImage) -> Void)? var cameraViewRect = CGRect.zero @@ -89,7 +90,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { NotificationCenter.default.removeObserver(self) } - func registerServiceEvents() { + func registerServiceEvents(onChallengeTypeReceived: @escaping (Challenge) -> Void) { livenessService?.register(onComplete: { [weak self] reason in self?.stopRecording() @@ -112,6 +113,13 @@ class FaceLivenessDetectionViewModel: ObservableObject { }, on: .challenge ) + + livenessService?.register( + listener: { [weak self] _challenge in + self?.challenge = _challenge + onChallengeTypeReceived(_challenge) + }, + on: .challenge) } @objc func willResignActive(_ notification: Notification) { @@ -178,7 +186,11 @@ class FaceLivenessDetectionViewModel: ObservableObject { func initializeLivenessStream() { do { - try livenessService?.initializeLivenessStream( + guard let livenessSession = livenessService as? FaceLivenessSession else { + throw FaceLivenessDetectionError.unknown + } + + try livenessSession.initializeLivenessStream( withSessionID: sessionID, userAgent: UserAgentValues.standard().userAgentString ) @@ -226,6 +238,8 @@ class FaceLivenessDetectionViewModel: ObservableObject { videoStartTime: UInt64 ) { guard initialClientEvent == nil else { return } + guard let challenge else { return } + videoChunker.start() let initialFace = FaceDetection( @@ -243,7 +257,9 @@ class FaceLivenessDetectionViewModel: ObservableObject { do { try livenessService?.send( - .initialFaceDetected(event: _initialClientEvent), + .initialFaceDetected(event: _initialClientEvent, + challenge: .init(version: challenge.version, + type: challenge.type)), eventDate: { .init() } ) } catch { @@ -261,7 +277,8 @@ class FaceLivenessDetectionViewModel: ObservableObject { guard let sessionConfiguration, let initialClientEvent, - let faceMatchedTimestamp + let faceMatchedTimestamp, + let challenge else { return } let finalClientEvent = FinalClientEvent( @@ -275,7 +292,9 @@ class FaceLivenessDetectionViewModel: ObservableObject { do { try livenessService?.send( - .final(event: finalClientEvent), + .final(event: finalClientEvent, + challenge: .init(version: challenge.version, + type: challenge.type)), eventDate: { .init() } ) @@ -310,6 +329,13 @@ class FaceLivenessDetectionViewModel: ObservableObject { self.faceGuideRect = faceGuide } } + + func completeNoLightCheck(faceGuide: CGRect) { + DispatchQueue.main.async { + self.livenessState.completedNoLightCheck() + self.faceGuideRect = faceGuide + } + } func sendVideoEvent(data: Data, videoEventTime: UInt64) { guard !hasSentFinalVideoEvent else { return } @@ -362,3 +388,5 @@ class FaceLivenessDetectionViewModel: ObservableObject { return data } } + +extension FaceLivenessDetectionViewModel: FaceDetectionSessionConfigurationWrapper { } diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessViewControllerPresenter.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessViewControllerPresenter.swift index 5786620b..8fff8b9f 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessViewControllerPresenter.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessViewControllerPresenter.swift @@ -12,4 +12,5 @@ protocol FaceLivenessViewControllerPresenter: AnyObject { func drawOvalInCanvas(_ ovalRect: CGRect) func displayFreshness(colorSequences: [FaceLivenessSession.DisplayColor]) func displaySingleFrame(uiImage: UIImage) + func completeNoLightCheck() } diff --git a/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift b/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift index 872c7ee6..fa66ffb0 100644 --- a/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift +++ b/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift @@ -76,6 +76,10 @@ struct LivenessStateMachine { mutating func completedDisplayingFreshness() { state = .completedDisplayingFreshness } + + mutating func completedNoLightCheck() { + state = .completedNoLightCheck + } mutating func displayingFreshness() { state = .displayingFreshness @@ -95,6 +99,7 @@ struct LivenessStateMachine { enum State: Equatable { case initial + case awaitingChallengeType case pendingFacePreparedConfirmation(FaceNotPreparedReason) case recording(ovalDisplayed: Bool) case awaitingFaceInOvalMatch(FaceNotPreparedReason, Double) @@ -102,6 +107,7 @@ struct LivenessStateMachine { case initialClientInfoEventSent case displayingFreshness case completedDisplayingFreshness + case completedNoLightCheck case completed case awaitingDisconnectEvent case disconnectEventReceived diff --git a/Sources/FaceLiveness/Views/Liveness/LivenessViewController.swift b/Sources/FaceLiveness/Views/Liveness/LivenessViewController.swift index 5e698f23..0435a862 100644 --- a/Sources/FaceLiveness/Views/Liveness/LivenessViewController.swift +++ b/Sources/FaceLiveness/Views/Liveness/LivenessViewController.swift @@ -169,4 +169,11 @@ extension _LivenessViewController: FaceLivenessViewControllerPresenter { self.ovalExists = true } } + + func completeNoLightCheck() { + guard let faceGuideRect = self.faceGuideRect else { return } + self.viewModel.completeNoLightCheck( + faceGuide: faceGuideRect + ) + } } diff --git a/Sources/FaceLiveness/Views/LoadingPage/LoadingPageView.swift b/Sources/FaceLiveness/Views/LoadingPage/LoadingPageView.swift new file mode 100644 index 00000000..e02b4e79 --- /dev/null +++ b/Sources/FaceLiveness/Views/LoadingPage/LoadingPageView.swift @@ -0,0 +1,27 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +import SwiftUI + +struct LoadingPageView: View { + + var body: some View { + VStack { + HStack(spacing: 5) { + ProgressView() + Text(LocalizedStrings.challenge_connecting) + } + + } + } +} + +struct LoadingPageView_Previews: PreviewProvider { + static var previews: some View { + LoadingPageView() + } +} diff --git a/Tests/FaceLivenessTests/DetectedFaceTests.swift b/Tests/FaceLivenessTests/DetectedFaceTests.swift index 4bee8292..6d538e33 100644 --- a/Tests/FaceLivenessTests/DetectedFaceTests.swift +++ b/Tests/FaceLivenessTests/DetectedFaceTests.swift @@ -7,7 +7,7 @@ import XCTest @testable import FaceLiveness - +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin final class DetectedFaceTests: XCTestCase { var detectedFace: DetectedFace! @@ -104,7 +104,29 @@ final class DetectedFaceTests: XCTestCase { width: 0.6240418540649166, height: 0.8144985824018897 ) - let boundingBox = detectedFace.boundingBoxFromLandmarks(ovalRect: ovalRect) + + let face = FaceLivenessSession.OvalMatchChallenge.Face( + distanceThreshold: 0.1, + distanceThresholdMax: 0.1, + distanceThresholdMin: 0.1, + iouWidthThreshold: 0.1, + iouHeightThreshold: 0.1 + ) + + let oval = FaceLivenessSession.OvalMatchChallenge.Oval(boundingBox: .init(x: 0.1, + y: 0.1, + width: 0.1, + height: 0.1), + heightWidthRatio: 1.618, + iouThreshold: 0.1, + iouWidthThreshold: 0.1, + iouHeightThreshold: 0.1, + ovalFitTimeout: 1) + + let boundingBox = detectedFace.boundingBoxFromLandmarks(ovalRect: ovalRect, + ovalMatchChallenge: .init(faceDetectionThreshold: 0.7, + face: face, + oval: oval)) XCTAssertEqual(boundingBox.origin.x, expectedBoundingBox.origin.x) XCTAssertEqual(boundingBox.origin.y, expectedBoundingBox.origin.y) XCTAssertEqual(boundingBox.width, expectedBoundingBox.width) diff --git a/Tests/FaceLivenessTests/LivenessTests.swift b/Tests/FaceLivenessTests/LivenessTests.swift index da063930..c4b95d02 100644 --- a/Tests/FaceLivenessTests/LivenessTests.swift +++ b/Tests/FaceLivenessTests/LivenessTests.swift @@ -69,6 +69,7 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { /// Then: The end state of this flow is `.faceMatched` func testHappyPathToMatchedFace() async throws { viewModel.livenessService = self.livenessService + viewModel.challenge = Challenge(version: "2.0.0", type: .faceMovementAndLightChallenge) viewModel.livenessState.checkIsFacePrepared() XCTAssertEqual(viewModel.livenessState.state, .pendingFacePreparedConfirmation(.pendingCheck)) @@ -103,16 +104,37 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { XCTAssertEqual(faceDetector.interactions, [ "setResultHandler(detectionResultHandler:) (FaceLivenessDetectionViewModel)" ]) - XCTAssertEqual(livenessService.interactions, [ - "initializeLivenessStream(withSessionID:userAgent:)" - ]) + XCTAssertEqual(livenessService.interactions, []) } /// Given: A `FaceLivenessDetectionViewModel` /// When: The viewModel is processes a single face result with a face distance less than the inital face distance - /// Then: The end state of this flow is `.recording(ovalDisplayed: false)` and initializeLivenessStream(withSessionID:userAgent:) is called + /// Then: The end state of this flow is `.recording(ovalDisplayed: false)` func testTransitionToRecordingState() async throws { viewModel.livenessService = self.livenessService + viewModel.challenge = Challenge(version: "2.0.0", type: .faceMovementAndLightChallenge) + + let face = FaceLivenessSession.OvalMatchChallenge.Face( + distanceThreshold: 0.32, + distanceThresholdMax: 0.1, + distanceThresholdMin: 0.1, + iouWidthThreshold: 0.1, + iouHeightThreshold: 0.1 + ) + + let oval = FaceLivenessSession.OvalMatchChallenge.Oval(boundingBox: .init(x: 0.1, + y: 0.1, + width: 0.1, + height: 0.1), + heightWidthRatio: 1.618, + iouThreshold: 0.1, + iouWidthThreshold: 0.1, + iouHeightThreshold: 0.1, + ovalFitTimeout: 1) + + viewModel.sessionConfiguration = .init(ovalMatchChallenge: .init(faceDetectionThreshold: 0.7, + face: face, + oval: oval)) viewModel.livenessState.checkIsFacePrepared() XCTAssertEqual(viewModel.livenessState.state, .pendingFacePreparedConfirmation(.pendingCheck)) @@ -136,9 +158,6 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { XCTAssertEqual(faceDetector.interactions, [ "setResultHandler(detectionResultHandler:) (FaceLivenessDetectionViewModel)" ]) - XCTAssertEqual(livenessService.interactions, [ - "initializeLivenessStream(withSessionID:userAgent:)" - ]) } /// Given: A `FaceLivenessDetectionViewModel` diff --git a/Tests/FaceLivenessTests/MockLivenessService.swift b/Tests/FaceLivenessTests/MockLivenessService.swift index 2b4633d1..942f7488 100644 --- a/Tests/FaceLivenessTests/MockLivenessService.swift +++ b/Tests/FaceLivenessTests/MockLivenessService.swift @@ -18,7 +18,7 @@ class MockLivenessService { var onFinalClientEvent: (LivenessEvent, Date) -> Void = { _, _ in } var onFreshnessEvent: (LivenessEvent, Date) -> Void = { _, _ in } var onVideoEvent: (LivenessEvent, Date) -> Void = { _, _ in } - var onInitializeLivenessStream: (String, String) -> Void = { _, _ in } + var onInitializeLivenessStream: (String, String,[Challenge]?) -> Void = { _, _, _ in } var onServiceException: (FaceLivenessSessionError) -> Void = { _ in } var onCloseSocket: (URLSessionWebSocketTask.CloseCode) -> Void = { _ in } } @@ -44,10 +44,12 @@ extension MockLivenessService: LivenessService { } func initializeLivenessStream( - withSessionID sessionID: String, userAgent: String + withSessionID sessionID: String, + userAgent: String, + challenges: [Challenge] ) throws { interactions.append(#function) - onInitializeLivenessStream(sessionID, userAgent) + onInitializeLivenessStream(sessionID, userAgent, challenges) } func register( @@ -62,6 +64,10 @@ extension MockLivenessService: LivenessService { ) { interactions.append(#function) } + + func register(listener: @escaping (Challenge) -> Void, on event: LivenessEventKind.Server) { + interactions.append(#function) + } func closeSocket(with code: URLSessionWebSocketTask.CloseCode) { interactions.append(#function) From e8643b766a386a05f9f0131eeb5f43225ee373ee Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Mon, 6 May 2024 10:14:55 -0700 Subject: [PATCH 02/15] chore: Add attempt count changes (#137) * chore: Add attempt count changes * Fix unit tests * add unit tests * Update region for example liveness view * Update amplify-swift dependency --- .../xcshareddata/swiftpm/Package.resolved | 2 +- HostApp/HostApp/Model/LivenessResult.swift | 4 ++ .../HostApp/Views/ExampleLivenessView.swift | 3 +- .../LivenessResultContentView+Result.swift | 4 ++ .../Views/LivenessResultContentView.swift | 64 +++++++++++++------ Package.resolved | 2 +- .../Views/GetReadyPage/GetReadyPageView.swift | 35 ++++------ .../Liveness/FaceLivenessDetectionView.swift | 17 ++--- ...ViewModel+FaceDetectionResultHandler.swift | 4 +- .../FaceLivenessDetectionViewModel.swift | 24 +++++-- .../CredentialsProviderTestCase.swift | 3 +- Tests/FaceLivenessTests/LivenessTests.swift | 58 ++++++++++++++++- .../MockLivenessService.swift | 7 +- 13 files changed, 155 insertions(+), 72 deletions(-) diff --git a/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index cddb5116..51a33701 100644 --- a/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -6,7 +6,7 @@ "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { "branch" : "feat/no-light-support", - "revision" : "7c1fa2f7a766208f5af69ca8dce5fd02e6de4db6" + "revision" : "22e02fa21399122aac1d8b4f6ab23c242c79dae6" } }, { diff --git a/HostApp/HostApp/Model/LivenessResult.swift b/HostApp/HostApp/Model/LivenessResult.swift index 226bc30f..3a36f089 100644 --- a/HostApp/HostApp/Model/LivenessResult.swift +++ b/HostApp/HostApp/Model/LivenessResult.swift @@ -6,11 +6,13 @@ // import Foundation +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin struct LivenessResult: Codable { let auditImageBytes: String? let confidenceScore: Double let isLive: Bool + let challenge: Challenge? } extension LivenessResult: CustomDebugStringConvertible { @@ -20,6 +22,8 @@ extension LivenessResult: CustomDebugStringConvertible { - confidenceScore: \(confidenceScore) - isLive: \(isLive) - auditImageBytes: \(auditImageBytes == nil ? "nil" : "") + - challengeType: \(String(describing: challenge?.type)) + - challengeVersion: \(String(describing: challenge?.version)) """ } } diff --git a/HostApp/HostApp/Views/ExampleLivenessView.swift b/HostApp/HostApp/Views/ExampleLivenessView.swift index 85eaf8cc..39407c89 100644 --- a/HostApp/HostApp/Views/ExampleLivenessView.swift +++ b/HostApp/HostApp/Views/ExampleLivenessView.swift @@ -22,8 +22,7 @@ struct ExampleLivenessView: View { case .liveness: FaceLivenessDetectorView( sessionID: viewModel.sessionID, - // TODO: Change before merging to main - region: "us-west-2", + region: "us-east-1", isPresented: Binding( get: { viewModel.presentationState == .liveness }, set: { _ in } diff --git a/HostApp/HostApp/Views/LivenessResultContentView+Result.swift b/HostApp/HostApp/Views/LivenessResultContentView+Result.swift index 3f57982f..0b18eaab 100644 --- a/HostApp/HostApp/Views/LivenessResultContentView+Result.swift +++ b/HostApp/HostApp/Views/LivenessResultContentView+Result.swift @@ -6,6 +6,7 @@ // import SwiftUI +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin extension LivenessResultContentView { struct Result { @@ -15,6 +16,7 @@ extension LivenessResultContentView { let valueBackgroundColor: Color let auditImage: Data? let isLive: Bool + let challenge: Challenge? init(livenessResult: LivenessResult) { guard livenessResult.confidenceScore > 0 else { @@ -24,6 +26,7 @@ extension LivenessResultContentView { valueBackgroundColor = .clear auditImage = nil isLive = false + challenge = nil return } isLive = livenessResult.isLive @@ -41,6 +44,7 @@ extension LivenessResultContentView { auditImage = livenessResult.auditImageBytes.flatMap{ Data(base64Encoded: $0) } + challenge = livenessResult.challenge } } diff --git a/HostApp/HostApp/Views/LivenessResultContentView.swift b/HostApp/HostApp/Views/LivenessResultContentView.swift index de2ecff7..51660f55 100644 --- a/HostApp/HostApp/Views/LivenessResultContentView.swift +++ b/HostApp/HostApp/Views/LivenessResultContentView.swift @@ -6,9 +6,10 @@ // import SwiftUI +@_spi(PredictionsFaceLiveness) import AWSPredictionsPlugin struct LivenessResultContentView: View { - @State var result: Result = .init(livenessResult: .init(auditImageBytes: nil, confidenceScore: -1, isLive: false)) + @State var result: Result = .init(livenessResult: .init(auditImageBytes: nil, confidenceScore: -1, isLive: false, challenge: nil)) let fetchResults: () async throws -> Result var body: some View { @@ -67,26 +68,48 @@ struct LivenessResultContentView: View { } } + func step(number: Int, text: String) -> some View { + HStack(alignment: .top) { + Text("\(number).") + Text(text) + } + } + + @ViewBuilder private func steps() -> some View { - func step(number: Int, text: String) -> some View { - HStack(alignment: .top) { - Text("\(number).") - Text(text) + switch result.challenge?.type { + case .faceMovementChallenge: + VStack( + alignment: .leading, + spacing: 8 + ) { + Text("Tips to pass the video check:") + .fontWeight(.semibold) + + Text("Remove sunglasses, mask, hat, or anything blocking your face.") + .accessibilityElement(children: .combine) + } + case .faceMovementAndLightChallenge: + VStack( + alignment: .leading, + spacing: 8 + ) { + Text("Tips to pass the video check:") + .fontWeight(.semibold) + + step(number: 1, text: "Avoid very bright lighting conditions, such as direct sunlight.") + .accessibilityElement(children: .combine) + + step(number: 2, text: "Remove sunglasses, mask, hat, or anything blocking your face.") + .accessibilityElement(children: .combine) + } + case .none: + VStack( + alignment: .leading, + spacing: 8 + ) { + EmptyView() } - } - - return VStack( - alignment: .leading, - spacing: 8 - ) { - Text("Tips to pass the video check:") - .fontWeight(.semibold) - - step(number: 1, text: "Avoid very bright lighting conditions, such as direct sunlight.") - .accessibilityElement(children: .combine) - - step(number: 2, text: "Remove sunglasses, mask, hat, or anything blocking your face.") - .accessibilityElement(children: .combine) } } } @@ -99,7 +122,8 @@ extension LivenessResultContentView { livenessResult: .init( auditImageBytes: nil, confidenceScore: 99.8329, - isLive: true + isLive: true, + challenge: nil ) ) } diff --git a/Package.resolved b/Package.resolved index a046c257..a61605c0 100644 --- a/Package.resolved +++ b/Package.resolved @@ -6,7 +6,7 @@ "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { "branch" : "feat/no-light-support", - "revision" : "7c1fa2f7a766208f5af69ca8dce5fd02e6de4db6" + "revision" : "22e02fa21399122aac1d8b4f6ab23c242c79dae6" } }, { diff --git a/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift b/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift index dadb2076..0c52ccff 100644 --- a/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift +++ b/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift @@ -27,28 +27,19 @@ struct GetReadyPageView: View { VStack { ZStack { CameraPreviewView() - switch self.challenge.type { - case .faceMovementChallenge: - VStack { - Text(LocalizedStrings.preview_center_your_face_text) - .font(.title) - .multilineTextAlignment(.center) - Spacer() - }.padding() - case . faceMovementAndLightChallenge: - VStack { - WarningBox( - titleText: LocalizedStrings.get_ready_photosensitivity_title, - bodyText: LocalizedStrings.get_ready_photosensitivity_description, - popoverContent: { photosensitivityWarningPopoverContent } - ) - .accessibilityElement(children: .combine) - Text(LocalizedStrings.preview_center_your_face_text) - .font(.title) - .multilineTextAlignment(.center) - Spacer() - }.padding() - } + VStack { + WarningBox( + titleText: LocalizedStrings.get_ready_photosensitivity_title, + bodyText: LocalizedStrings.get_ready_photosensitivity_description, + popoverContent: { photosensitivityWarningPopoverContent } + ) + .accessibilityElement(children: .combine) + .opacity(challenge.type == .faceMovementAndLightChallenge ? 1.0 : 0.0) + Text(LocalizedStrings.preview_center_your_face_text) + .font(.title) + .multilineTextAlignment(.center) + Spacer() + }.padding() } beginCheckButton } diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index eff46ec7..f31d2390 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -20,7 +20,6 @@ public struct FaceLivenessDetectorView: View { @State var displayingCameraPermissionsNeededAlert = false let disableStartView: Bool - let facelivenessDetectorViewId: String let onCompletion: (Result) -> Void let sessionTask: Task @@ -32,9 +31,7 @@ public struct FaceLivenessDetectorView: View { disableStartView: Bool = false, isPresented: Binding, onCompletion: @escaping (Result) -> Void - ) { - let viewId = UUID().uuidString - self.facelivenessDetectorViewId = viewId + ) { self.disableStartView = disableStartView self._isPresented = isPresented self.onCompletion = onCompletion @@ -44,8 +41,6 @@ public struct FaceLivenessDetectorView: View { withID: sessionID, credentialsProvider: credentialsProvider, region: region, - options: .init(faceLivenessDetectorViewId: viewId, - preCheckViewEnabled: !disableStartView), completion: map(detectionCompletion: onCompletion) ) return session @@ -83,7 +78,8 @@ public struct FaceLivenessDetectorView: View { captureSession: captureSession, videoChunker: videoChunker, closeButtonAction: { onCompletion(.failure(.userCancelled)) }, - sessionID: sessionID + sessionID: sessionID, + isPreviewScreenEnabled: !disableStartView ) ) @@ -99,8 +95,6 @@ public struct FaceLivenessDetectorView: View { onCompletion: @escaping (Result) -> Void, captureSession: LivenessCaptureSession ) { - let viewId = UUID().uuidString - self.facelivenessDetectorViewId = viewId self.disableStartView = disableStartView self._isPresented = isPresented self.onCompletion = onCompletion @@ -110,8 +104,6 @@ public struct FaceLivenessDetectorView: View { withID: sessionID, credentialsProvider: credentialsProvider, region: region, - options: .init(faceLivenessDetectorViewId: viewId, - preCheckViewEnabled: !disableStartView), completion: map(detectionCompletion: onCompletion) ) return session @@ -128,7 +120,8 @@ public struct FaceLivenessDetectorView: View { captureSession: captureSession, videoChunker: captureSession.outputSampleBufferCapturer!.videoChunker, closeButtonAction: { onCompletion(.failure(.userCancelled)) }, - sessionID: sessionID + sessionID: sessionID, + isPreviewScreenEnabled: !disableStartView ) ) } diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+FaceDetectionResultHandler.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+FaceDetectionResultHandler.swift index edb96e80..0e43de2a 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+FaceDetectionResultHandler.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel+FaceDetectionResultHandler.swift @@ -121,9 +121,7 @@ extension FaceLivenessDetectionViewModel: FaceDetectionResultHandler { } } case .faceMovementChallenge: - DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { - self.livenessViewControllerDelegate?.completeNoLightCheck() - } + self.livenessViewControllerDelegate?.completeNoLightCheck() default: break } diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift index 352b1855..e83c7c22 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift @@ -12,6 +12,7 @@ import AVFoundation fileprivate let videoSize: CGSize = .init(width: 480, height: 640) fileprivate let defaultNoFitTimeoutInterval: TimeInterval = 7 +fileprivate let defaultAttemptCountResetInterval: TimeInterval = 300.0 @MainActor class FaceLivenessDetectionViewModel: ObservableObject { @@ -28,6 +29,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { let faceDetector: FaceDetector let faceInOvalMatching: FaceInOvalMatching let challengeID: String = UUID().uuidString + let isPreviewScreenEnabled : Bool var colorSequences: [ColorSequence] = [] var hasSentFinalVideoEvent = false var hasSentFirstVideo = false @@ -43,6 +45,9 @@ class FaceLivenessDetectionViewModel: ObservableObject { var faceMatchedTimestamp: UInt64? var noFitStartTime: Date? + static var attemptCount: Int = 0 + static var attemptIdTimeStamp: Date = Date() + var noFitTimeoutInterval: TimeInterval { if let sessionTimeoutMilliSec = sessionConfiguration?.ovalMatchChallenge.oval.ovalFitTimeout { return TimeInterval(sessionTimeoutMilliSec/1_000) @@ -58,7 +63,8 @@ class FaceLivenessDetectionViewModel: ObservableObject { videoChunker: VideoChunker, stateMachine: LivenessStateMachine = .init(state: .initial), closeButtonAction: @escaping () -> Void, - sessionID: String + sessionID: String, + isPreviewScreenEnabled: Bool ) { self.closeButtonAction = closeButtonAction self.videoChunker = videoChunker @@ -67,6 +73,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { self.captureSession = captureSession self.faceDetector = faceDetector self.faceInOvalMatching = faceInOvalMatching + self.isPreviewScreenEnabled = isPreviewScreenEnabled self.closeButtonAction = { [weak self] in guard let self else { return } @@ -186,13 +193,20 @@ class FaceLivenessDetectionViewModel: ObservableObject { func initializeLivenessStream() { do { - guard let livenessSession = livenessService as? FaceLivenessSession else { - throw FaceLivenessDetectionError.unknown + if (abs(Self.attemptIdTimeStamp.timeIntervalSinceNow) > defaultAttemptCountResetInterval) { + Self.attemptCount = 1 + } else { + Self.attemptCount += 1 } + Self.attemptIdTimeStamp = Date() - try livenessSession.initializeLivenessStream( + try livenessService?.initializeLivenessStream( withSessionID: sessionID, - userAgent: UserAgentValues.standard().userAgentString + userAgent: UserAgentValues.standard().userAgentString, + challenges: FaceLivenessSession.supportedChallenges, + options: .init( + attemptCount: Self.attemptCount, + preCheckViewEnabled: isPreviewScreenEnabled) ) } catch { DispatchQueue.main.async { diff --git a/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift b/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift index 7d69251b..3c1dabbf 100644 --- a/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift +++ b/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift @@ -41,7 +41,8 @@ final class CredentialsProviderTestCase: XCTestCase { captureSession: captureSession, videoChunker: videoChunker, closeButtonAction: {}, - sessionID: UUID().uuidString + sessionID: UUID().uuidString, + isPreviewScreenEnabled: false ) self.videoChunker = videoChunker diff --git a/Tests/FaceLivenessTests/LivenessTests.swift b/Tests/FaceLivenessTests/LivenessTests.swift index c4b95d02..5603914a 100644 --- a/Tests/FaceLivenessTests/LivenessTests.swift +++ b/Tests/FaceLivenessTests/LivenessTests.swift @@ -32,7 +32,8 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { captureSession: captureSession, videoChunker: videoChunker, closeButtonAction: {}, - sessionID: UUID().uuidString + sessionID: UUID().uuidString, + isPreviewScreenEnabled: false ) self.videoChunker = videoChunker @@ -104,7 +105,9 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { XCTAssertEqual(faceDetector.interactions, [ "setResultHandler(detectionResultHandler:) (FaceLivenessDetectionViewModel)" ]) - XCTAssertEqual(livenessService.interactions, []) + XCTAssertEqual(livenessService.interactions, [ + "initializeLivenessStream(withSessionID:userAgent:challenges:options:)" + ]) } /// Given: A `FaceLivenessDetectionViewModel` @@ -193,4 +196,55 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { try await Task.sleep(seconds: 1) XCTAssertEqual(self.viewModel.livenessState.state, .encounteredUnrecoverableError(.timedOut)) } + + /// Given: A `FaceLivenessDetectionViewModel` + /// When: The initializeLivenessStream() is called for the first time and then called again after 3 seconds + /// Then: The attempt count is incremented + func testAttemptCountIncrementFirstTime() async throws { + viewModel.livenessService = self.livenessService + self.viewModel.initializeLivenessStream() + XCTAssertEqual(livenessService.interactions, [ + "initializeLivenessStream(withSessionID:userAgent:challenges:options:)" + ]) + + XCTAssertEqual(FaceLivenessDetectionViewModel.attemptCount, 1) + try await Task.sleep(seconds: 3) + + self.viewModel.initializeLivenessStream() + XCTAssertEqual(livenessService.interactions, [ + "initializeLivenessStream(withSessionID:userAgent:challenges:options:)", + "initializeLivenessStream(withSessionID:userAgent:challenges:options:)" + ]) + XCTAssertEqual(FaceLivenessDetectionViewModel.attemptCount, 2) + } + + /// Given: A `FaceLivenessDetectionViewModel` + /// When: The attempt count is 4, last attempt time was < 5 minutes and initializeLivenessStream() is called + /// Then: The attempt count is incremented + func testAttemptCountIncrement() async throws { + viewModel.livenessService = self.livenessService + FaceLivenessDetectionViewModel.attemptCount = 4 + FaceLivenessDetectionViewModel.attemptIdTimeStamp = Date().addingTimeInterval(-180) + self.viewModel.initializeLivenessStream() + XCTAssertEqual(livenessService.interactions, [ + "initializeLivenessStream(withSessionID:userAgent:challenges:options:)" + ]) + + XCTAssertEqual(FaceLivenessDetectionViewModel.attemptCount, 5) + } + + /// Given: A `FaceLivenessDetectionViewModel` + /// When: The attempt count is 4, last attempt time was > 5 minutes and initializeLivenessStream() is called + /// Then: The attempt count is not incremented and reset to 1 + func testAttemptCountReset() async throws { + viewModel.livenessService = self.livenessService + FaceLivenessDetectionViewModel.attemptCount = 4 + FaceLivenessDetectionViewModel.attemptIdTimeStamp = Date().addingTimeInterval(-305) + self.viewModel.initializeLivenessStream() + XCTAssertEqual(livenessService.interactions, [ + "initializeLivenessStream(withSessionID:userAgent:challenges:options:)" + ]) + + XCTAssertEqual(FaceLivenessDetectionViewModel.attemptCount, 1) + } } diff --git a/Tests/FaceLivenessTests/MockLivenessService.swift b/Tests/FaceLivenessTests/MockLivenessService.swift index 942f7488..d3e43a8d 100644 --- a/Tests/FaceLivenessTests/MockLivenessService.swift +++ b/Tests/FaceLivenessTests/MockLivenessService.swift @@ -18,7 +18,7 @@ class MockLivenessService { var onFinalClientEvent: (LivenessEvent, Date) -> Void = { _, _ in } var onFreshnessEvent: (LivenessEvent, Date) -> Void = { _, _ in } var onVideoEvent: (LivenessEvent, Date) -> Void = { _, _ in } - var onInitializeLivenessStream: (String, String,[Challenge]?) -> Void = { _, _, _ in } + var onInitializeLivenessStream: (String, String,[Challenge]?,FaceLivenessSession.Options) -> Void = { _, _, _, _ in } var onServiceException: (FaceLivenessSessionError) -> Void = { _ in } var onCloseSocket: (URLSessionWebSocketTask.CloseCode) -> Void = { _ in } } @@ -46,10 +46,11 @@ extension MockLivenessService: LivenessService { func initializeLivenessStream( withSessionID sessionID: String, userAgent: String, - challenges: [Challenge] + challenges: [Challenge], + options: FaceLivenessSession.Options ) throws { interactions.append(#function) - onInitializeLivenessStream(sessionID, userAgent, challenges) + onInitializeLivenessStream(sessionID, userAgent, challenges, options) } func register( From b22ef51b38c71645f98dc581532ee878d102a422 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Mon, 1 Jul 2024 11:01:28 -0700 Subject: [PATCH 03/15] chore: update dependencies after rebase --- Package.resolved | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Package.resolved b/Package.resolved index a61605c0..f850e8a9 100644 --- a/Package.resolved +++ b/Package.resolved @@ -6,7 +6,7 @@ "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { "branch" : "feat/no-light-support", - "revision" : "22e02fa21399122aac1d8b4f6ab23c242c79dae6" + "revision" : "1a5386d8d8e8e1edf631625c7bb6e003b2c0c821" } }, { @@ -50,8 +50,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/stephencelis/SQLite.swift.git", "state" : { - "revision" : "5f5ad81ac0d0a0f3e56e39e646e8423c617df523", - "version" : "0.13.2" + "revision" : "a95fc6df17d108bd99210db5e8a9bac90fe984b8", + "version" : "0.15.3" } }, { From 49b620c93536cab7646ed463f1d6facb89f2b47e Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Tue, 2 Jul 2024 11:29:13 -0700 Subject: [PATCH 04/15] fix: handle error on loading view (#154) --- .../Liveness/FaceLivenessDetectionView.swift | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index f31d2390..309c3e81 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -82,8 +82,6 @@ public struct FaceLivenessDetectorView: View { isPreviewScreenEnabled: !disableStartView ) ) - - faceDetector.setFaceDetectionSessionConfigurationWrapper(configuration: viewModel) } init( @@ -142,6 +140,23 @@ public struct FaceLivenessDetectorView: View { } catch { throw FaceLivenessDetectionError.accessDenied } + + DispatchQueue.main.async { + if let faceDetector = viewModel.faceDetector as? FaceDetectorShortRange.Model { + faceDetector.setFaceDetectionSessionConfigurationWrapper(configuration: viewModel) + } + } + } + } + .onReceive(viewModel.$livenessState) { output in + switch output.state { + case .encounteredUnrecoverableError(let error): + let closeCode = error.webSocketCloseCode ?? .normalClosure + viewModel.livenessService?.closeSocket(with: closeCode) + isPresented = false + onCompletion(.failure(mapError(error))) + default: + break } } case .awaitingLivenessSession(let challenge): From f2e8a7e2e9598344df1f59a1be9840e84e196a54 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Mon, 1 Jul 2024 12:01:49 -0700 Subject: [PATCH 05/15] update test branch dependency and add code for choosing challenge type --- .../xcshareddata/swiftpm/Package.resolved | 12 ++++++------ Package.resolved | 4 ++-- Package.swift | 2 +- .../Liveness/FaceLivenessDetectionViewModel.swift | 7 +++++++ 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 51a33701..2e1322ac 100644 --- a/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { - "branch" : "feat/no-light-support", - "revision" : "22e02fa21399122aac1d8b4f6ab23c242c79dae6" + "branch" : "test/no-light-support", + "revision" : "6aabc94f91193397b4c57e2b2823dbca4dbc0477" } }, { @@ -50,8 +50,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/stephencelis/SQLite.swift.git", "state" : { - "revision" : "5f5ad81ac0d0a0f3e56e39e646e8423c617df523", - "version" : "0.13.2" + "revision" : "a95fc6df17d108bd99210db5e8a9bac90fe984b8", + "version" : "0.15.3" } }, { @@ -59,8 +59,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "e97a6fcb1ab07462881ac165fdbb37f067e205d5", - "version" : "1.5.4" + "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537", + "version" : "1.6.1" } } ], diff --git a/Package.resolved b/Package.resolved index f850e8a9..2e1322ac 100644 --- a/Package.resolved +++ b/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { - "branch" : "feat/no-light-support", - "revision" : "1a5386d8d8e8e1edf631625c7bb6e003b2c0c821" + "branch" : "test/no-light-support", + "revision" : "6aabc94f91193397b4c57e2b2823dbca4dbc0477" } }, { diff --git a/Package.swift b/Package.swift index 446f12c0..01a52b9a 100644 --- a/Package.swift +++ b/Package.swift @@ -14,7 +14,7 @@ let package = Package( ], dependencies: [ // TODO: Change this before merge to main - .package(url: "https://github.com/aws-amplify/amplify-swift", branch: "feat/no-light-support") + .package(url: "https://github.com/aws-amplify/amplify-swift", branch: "test/no-light-support") ], targets: [ .target( diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift index e83c7c22..76585726 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift @@ -204,6 +204,13 @@ class FaceLivenessDetectionViewModel: ObservableObject { withSessionID: sessionID, userAgent: UserAgentValues.standard().userAgentString, challenges: FaceLivenessSession.supportedChallenges, + + // light challenge + // challenges: [.init(version: "2.0.0", type: .faceMovementAndLightChallenge)], + + // no light challenge + // challenges: [.init(version: "1.0.0", type: .faceMovementChallenge)], + options: .init( attemptCount: Self.attemptCount, preCheckViewEnabled: isPreviewScreenEnabled) From 7bc6928d38eac5e6f61950ca26228f75141a35f0 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Wed, 24 Jul 2024 10:48:09 -0700 Subject: [PATCH 06/15] chore: back camera support --- .../HostApp/Views/ExampleLivenessView.swift | 1 + .../AV/LivenessCaptureDevice.swift | 5 +++++ .../GetReadyPage/CameraPreviewView.swift | 2 +- .../GetReadyPage/CameraPreviewViewModel.swift | 7 +++++-- .../Views/GetReadyPage/GetReadyPageView.swift | 10 +++++++--- .../Liveness/FaceLivenessDetectionView.swift | 20 ++++++++++++------- .../CredentialsProviderTestCase.swift | 2 ++ 7 files changed, 34 insertions(+), 13 deletions(-) diff --git a/HostApp/HostApp/Views/ExampleLivenessView.swift b/HostApp/HostApp/Views/ExampleLivenessView.swift index 39407c89..eeb34272 100644 --- a/HostApp/HostApp/Views/ExampleLivenessView.swift +++ b/HostApp/HostApp/Views/ExampleLivenessView.swift @@ -23,6 +23,7 @@ struct ExampleLivenessView: View { FaceLivenessDetectorView( sessionID: viewModel.sessionID, region: "us-east-1", + cameraPosition: .front, isPresented: Binding( get: { viewModel.presentationState == .liveness }, set: { _ in } diff --git a/Sources/FaceLiveness/AV/LivenessCaptureDevice.swift b/Sources/FaceLiveness/AV/LivenessCaptureDevice.swift index 1d67e913..3935459e 100644 --- a/Sources/FaceLiveness/AV/LivenessCaptureDevice.swift +++ b/Sources/FaceLiveness/AV/LivenessCaptureDevice.swift @@ -37,3 +37,8 @@ struct LivenessCaptureDevice { } } } + +public enum LivenessCaptureDevicePosition { + case front + case back +} diff --git a/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewView.swift b/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewView.swift index 2e8530f3..19e2f483 100644 --- a/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewView.swift +++ b/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewView.swift @@ -15,7 +15,7 @@ struct CameraPreviewView: View { @StateObject var model: CameraPreviewViewModel - init(model: CameraPreviewViewModel = CameraPreviewViewModel()) { + init(model: CameraPreviewViewModel = CameraPreviewViewModel(cameraPosition: .front)) { self._model = StateObject(wrappedValue: model) } diff --git a/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewViewModel.swift b/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewViewModel.swift index b50173b0..f40645b5 100644 --- a/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewViewModel.swift +++ b/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewViewModel.swift @@ -16,15 +16,18 @@ class CameraPreviewViewModel: NSObject, ObservableObject { @Published var buffer: CVPixelBuffer? var previewCaptureSession: LivenessCaptureSession? + let cameraPosition: LivenessCaptureDevicePosition - override init() { + init(cameraPosition: LivenessCaptureDevicePosition) { + self.cameraPosition = cameraPosition + super.init() setupSubscriptions() let avCaptureDevice = AVCaptureDevice.DiscoverySession( deviceTypes: [.builtInWideAngleCamera], mediaType: .video, - position: .front + position: cameraPosition == .front ? .front : .back ).devices.first let outputDelegate = CameraPreviewOutputSampleBufferDelegate { [weak self] buffer in diff --git a/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift b/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift index 0c52ccff..d5a87b5b 100644 --- a/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift +++ b/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift @@ -12,21 +12,24 @@ struct GetReadyPageView: View { let beginCheckButtonDisabled: Bool let onBegin: () -> Void let challenge: Challenge + let cameraPosition: LivenessCaptureDevicePosition init( onBegin: @escaping () -> Void, beginCheckButtonDisabled: Bool = false, - challenge: Challenge + challenge: Challenge, + cameraPosition: LivenessCaptureDevicePosition ) { self.onBegin = onBegin self.beginCheckButtonDisabled = beginCheckButtonDisabled self.challenge = challenge + self.cameraPosition = cameraPosition } var body: some View { VStack { ZStack { - CameraPreviewView() + CameraPreviewView(model: CameraPreviewViewModel(cameraPosition: cameraPosition)) VStack { WarningBox( titleText: LocalizedStrings.get_ready_photosensitivity_title, @@ -79,6 +82,7 @@ struct GetReadyPageView_Previews: PreviewProvider { static var previews: some View { GetReadyPageView(onBegin: {}, challenge: .init(version: "2.0.0", - type: .faceMovementAndLightChallenge)) + type: .faceMovementAndLightChallenge), + cameraPosition: .front) } } diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index 309c3e81..f4f91a35 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -20,6 +20,7 @@ public struct FaceLivenessDetectorView: View { @State var displayingCameraPermissionsNeededAlert = false let disableStartView: Bool + let cameraPosition: LivenessCaptureDevicePosition let onCompletion: (Result) -> Void let sessionTask: Task @@ -29,11 +30,13 @@ public struct FaceLivenessDetectorView: View { credentialsProvider: AWSCredentialsProvider? = nil, region: String, disableStartView: Bool = false, + cameraPosition: LivenessCaptureDevicePosition, isPresented: Binding, onCompletion: @escaping (Result) -> Void ) { self.disableStartView = disableStartView self._isPresented = isPresented + self.cameraPosition = cameraPosition self.onCompletion = onCompletion self.sessionTask = Task { @@ -60,7 +63,7 @@ public struct FaceLivenessDetectorView: View { let avCpatureDevice = AVCaptureDevice.DiscoverySession( deviceTypes: [.builtInWideAngleCamera], mediaType: .video, - position: .front + position: cameraPosition == .front ? .front : .back ).devices.first let captureSession = LivenessCaptureSession( @@ -89,6 +92,7 @@ public struct FaceLivenessDetectorView: View { credentialsProvider: AWSCredentialsProvider? = nil, region: String, disableStartView: Bool = false, + cameraPosition: LivenessCaptureDevicePosition, isPresented: Binding, onCompletion: @escaping (Result) -> Void, captureSession: LivenessCaptureSession @@ -96,6 +100,7 @@ public struct FaceLivenessDetectorView: View { self.disableStartView = disableStartView self._isPresented = isPresented self.onCompletion = onCompletion + self.cameraPosition = cameraPosition self.sessionTask = Task { let session = try await AWSPredictionsPlugin.startFaceLivenessSession( @@ -166,21 +171,22 @@ public struct FaceLivenessDetectorView: View { do { let newState = disableStartView ? DisplayState.displayingLiveness - : DisplayState.displayingGetReadyView(challenge) + : DisplayState.displayingGetReadyView(challenge, cameraPosition) guard self.displayState != newState else { return } self.displayState = newState } } } - case .displayingGetReadyView(let challenge): + case .displayingGetReadyView(let challenge, let cameraPosition): GetReadyPageView( onBegin: { guard displayState != .displayingLiveness else { return } displayState = .displayingLiveness }, beginCheckButtonDisabled: false, - challenge: challenge + challenge: challenge, + cameraPosition: cameraPosition ) .onAppear { DispatchQueue.main.async { @@ -274,7 +280,7 @@ public struct FaceLivenessDetectorView: View { enum DisplayState: Equatable { case awaitingChallengeType case awaitingLivenessSession(Challenge) - case displayingGetReadyView(Challenge) + case displayingGetReadyView(Challenge, LivenessCaptureDevicePosition) case displayingLiveness case awaitingCameraPermission @@ -284,8 +290,8 @@ enum DisplayState: Equatable { return true case (let .awaitingLivenessSession(c1), let .awaitingLivenessSession(c2)): return c1.type == c2.type && c1.version == c2.version - case (let .displayingGetReadyView(c1), let .displayingGetReadyView(c2)): - return c1.type == c2.type && c1.version == c2.version + case (let .displayingGetReadyView(c1, position1), let .displayingGetReadyView(c2, position2)): + return c1.type == c2.type && c1.version == c2.version && position1 == position2 case (.displayingLiveness, .displayingLiveness): return true case (.awaitingCameraPermission, .awaitingCameraPermission): diff --git a/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift b/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift index 3c1dabbf..59eb19eb 100644 --- a/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift +++ b/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift @@ -66,6 +66,7 @@ final class CredentialsProviderTestCase: XCTestCase { sessionID: UUID().uuidString, credentialsProvider: credentialsProvider, region: "us-east-1", + cameraPosition: .front, isPresented: .constant(true), onCompletion: { _ in } ) @@ -102,6 +103,7 @@ final class CredentialsProviderTestCase: XCTestCase { sessionID: UUID().uuidString, credentialsProvider: credentialsProvider, region: "us-east-1", + cameraPosition: .front, isPresented: .constant(true), onCompletion: { _ in } ) From 6d8243457ea19eadf6ca33abe438c89c98c66417 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Thu, 22 Aug 2024 12:01:14 -0700 Subject: [PATCH 07/15] Add code for error scenarios --- HostApp/HostApp.xcodeproj/project.pbxproj | 10 ++++++++ .../HostApp/Views/ExampleLivenessView.swift | 6 ++++- .../Views/LivenessCheckErrorContentView.swift | 6 +++++ .../Liveness/FaceLivenessDetectionError.swift | 6 +++++ .../Liveness/FaceLivenessDetectionView.swift | 23 +++++++++++-------- .../FaceLivenessDetectionViewModel.swift | 18 +++++++++++++-- .../Views/Liveness/LivenessStateMachine.swift | 1 + .../CredentialsProviderTestCase.swift | 3 ++- Tests/FaceLivenessTests/LivenessTests.swift | 3 ++- 9 files changed, 61 insertions(+), 15 deletions(-) diff --git a/HostApp/HostApp.xcodeproj/project.pbxproj b/HostApp/HostApp.xcodeproj/project.pbxproj index 81ef8e10..8206ed1c 100644 --- a/HostApp/HostApp.xcodeproj/project.pbxproj +++ b/HostApp/HostApp.xcodeproj/project.pbxproj @@ -132,6 +132,7 @@ 9070FFBD285112B5009867D5 /* HostAppUITests */, 9070FFA1285112B4009867D5 /* Products */, 90215EED291E9FB60050F2AD /* Frameworks */, + A5A9AF5054D0FF13505B212A /* AmplifyConfig */, ); sourceTree = ""; }; @@ -214,6 +215,15 @@ path = Model; sourceTree = ""; }; + A5A9AF5054D0FF13505B212A /* AmplifyConfig */ = { + isa = PBXGroup; + children = ( + 973619242BA378690003A590 /* awsconfiguration.json */, + 973619232BA378690003A590 /* amplifyconfiguration.json */, + ); + name = AmplifyConfig; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ diff --git a/HostApp/HostApp/Views/ExampleLivenessView.swift b/HostApp/HostApp/Views/ExampleLivenessView.swift index eeb34272..8401b833 100644 --- a/HostApp/HostApp/Views/ExampleLivenessView.swift +++ b/HostApp/HostApp/Views/ExampleLivenessView.swift @@ -23,7 +23,7 @@ struct ExampleLivenessView: View { FaceLivenessDetectorView( sessionID: viewModel.sessionID, region: "us-east-1", - cameraPosition: .front, + cameraPosition: .back, isPresented: Binding( get: { viewModel.presentationState == .liveness }, set: { _ in } @@ -46,6 +46,8 @@ struct ExampleLivenessView: View { viewModel.presentationState = .error(.countdownFaceTooClose) case .failure(.invalidSignature): viewModel.presentationState = .error(.invalidSignature) + case .failure(.invalidCameraPositionSelected): + viewModel.presentationState = .error(.invalidCameraPositionSelected) default: viewModel.presentationState = .liveness } @@ -75,6 +77,8 @@ struct ExampleLivenessView: View { LivenessCheckErrorContentView.failedDuringCountdown case .invalidSignature: LivenessCheckErrorContentView.invalidSignature + case . invalidCameraPositionSelected: + LivenessCheckErrorContentView.invalidCameraPositionSelected default: LivenessCheckErrorContentView.unexpected } diff --git a/HostApp/HostApp/Views/LivenessCheckErrorContentView.swift b/HostApp/HostApp/Views/LivenessCheckErrorContentView.swift index 3357aa7a..deed2cf5 100644 --- a/HostApp/HostApp/Views/LivenessCheckErrorContentView.swift +++ b/HostApp/HostApp/Views/LivenessCheckErrorContentView.swift @@ -55,6 +55,12 @@ extension LivenessCheckErrorContentView { name: "The signature on the request is invalid.", description: "Ensure the device time is correct and try again." ) + + static let invalidCameraPositionSelected = LivenessCheckErrorContentView( + name: "The camera position selected is incompatible with the liveness challenge type requested.", + description: "Please ensure the camera position is supported for the liveness challenge type requested." + ) + } struct LivenessCheckErrorContentView_Previews: PreviewProvider { diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift index a19cfd57..8ed31176 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift @@ -125,6 +125,12 @@ public struct FaceLivenessDetectionError: Error, Equatable { message: "The signature on the request is invalid.", recoverySuggestion: "Ensure the device time is correct and try again." ) + + public static let invalidCameraPositionSelected = FaceLivenessDetectionError( + code: 18, + message: "The camera position selected is incompatible with the liveness challenge type requested.", + recoverySuggestion: "Please ensure the camera position is supported for the liveness challenge type requested." + ) public static func == (lhs: FaceLivenessDetectionError, rhs: FaceLivenessDetectionError) -> Bool { lhs.code == rhs.code diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index f4f91a35..33a05940 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -30,7 +30,7 @@ public struct FaceLivenessDetectorView: View { credentialsProvider: AWSCredentialsProvider? = nil, region: String, disableStartView: Bool = false, - cameraPosition: LivenessCaptureDevicePosition, + cameraPosition: LivenessCaptureDevicePosition = .front, isPresented: Binding, onCompletion: @escaping (Result) -> Void ) { @@ -59,15 +59,14 @@ public struct FaceLivenessDetectorView: View { assetWriterDelegate: VideoChunker.AssetWriterDelegate(), assetWriterInput: LivenessAVAssetWriterInput() ) - - let avCpatureDevice = AVCaptureDevice.DiscoverySession( - deviceTypes: [.builtInWideAngleCamera], - mediaType: .video, - position: cameraPosition == .front ? .front : .back - ).devices.first + + let avCaptureDevice = AVCaptureDevice.default( + .builtInWideAngleCamera, + for: .video, + position: cameraPosition == .front ? .front : .back) let captureSession = LivenessCaptureSession( - captureDevice: .init(avCaptureDevice: avCpatureDevice), + captureDevice: .init(avCaptureDevice: avCaptureDevice), outputDelegate: OutputSampleBufferCapturer( faceDetector: faceDetector, videoChunker: videoChunker @@ -82,7 +81,8 @@ public struct FaceLivenessDetectorView: View { videoChunker: videoChunker, closeButtonAction: { onCompletion(.failure(.userCancelled)) }, sessionID: sessionID, - isPreviewScreenEnabled: !disableStartView + isPreviewScreenEnabled: !disableStartView, + cameraPosition: cameraPosition ) ) } @@ -124,7 +124,8 @@ public struct FaceLivenessDetectorView: View { videoChunker: captureSession.outputSampleBufferCapturer!.videoChunker, closeButtonAction: { onCompletion(.failure(.userCancelled)) }, sessionID: sessionID, - isPreviewScreenEnabled: !disableStartView + isPreviewScreenEnabled: !disableStartView, + cameraPosition: cameraPosition ) ) } @@ -240,6 +241,8 @@ public struct FaceLivenessDetectorView: View { return .faceInOvalMatchExceededTimeLimitError case .socketClosed: return .socketClosed + case .invalidCameraPositionSelecteed: + return .invalidCameraPositionSelected default: return .cameraPermissionDenied } diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift index 76585726..dcee0b6b 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift @@ -44,6 +44,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { var initialClientEvent: InitialClientEvent? var faceMatchedTimestamp: UInt64? var noFitStartTime: Date? + let cameraPosition: LivenessCaptureDevicePosition static var attemptCount: Int = 0 static var attemptIdTimeStamp: Date = Date() @@ -64,7 +65,8 @@ class FaceLivenessDetectionViewModel: ObservableObject { stateMachine: LivenessStateMachine = .init(state: .initial), closeButtonAction: @escaping () -> Void, sessionID: String, - isPreviewScreenEnabled: Bool + isPreviewScreenEnabled: Bool, + cameraPosition: LivenessCaptureDevicePosition ) { self.closeButtonAction = closeButtonAction self.videoChunker = videoChunker @@ -74,6 +76,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { self.faceDetector = faceDetector self.faceInOvalMatching = faceInOvalMatching self.isPreviewScreenEnabled = isPreviewScreenEnabled + self.cameraPosition = cameraPosition self.closeButtonAction = { [weak self] in guard let self else { return } @@ -124,7 +127,18 @@ class FaceLivenessDetectionViewModel: ObservableObject { livenessService?.register( listener: { [weak self] _challenge in self?.challenge = _challenge - onChallengeTypeReceived(_challenge) + guard _challenge.type == .faceMovementAndLightChallenge, + self?.cameraPosition == .back else { + onChallengeTypeReceived(_challenge) + return + } + + // incompatible camera position with challenge type + // return error + DispatchQueue.main.async { + self?.livenessState + .unrecoverableStateEncountered(.invalidCameraPositionSelecteed) + } }, on: .challenge) } diff --git a/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift b/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift index fa66ffb0..77a95c1a 100644 --- a/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift +++ b/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift @@ -165,6 +165,7 @@ struct LivenessStateMachine { static let couldNotOpenStream = LivenessError(code: 5, webSocketCloseCode: .unexpectedRuntimeError) static let socketClosed = LivenessError(code: 6, webSocketCloseCode: .normalClosure) static let viewResignation = LivenessError(code: 8, webSocketCloseCode: .viewClosure) + static let invalidCameraPositionSelecteed = LivenessError(code: 9, webSocketCloseCode: .unexpectedRuntimeError) static func == (lhs: LivenessError, rhs: LivenessError) -> Bool { lhs.code == rhs.code diff --git a/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift b/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift index 59eb19eb..5b82119a 100644 --- a/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift +++ b/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift @@ -42,7 +42,8 @@ final class CredentialsProviderTestCase: XCTestCase { videoChunker: videoChunker, closeButtonAction: {}, sessionID: UUID().uuidString, - isPreviewScreenEnabled: false + isPreviewScreenEnabled: false, + cameraPosition: .front ) self.videoChunker = videoChunker diff --git a/Tests/FaceLivenessTests/LivenessTests.swift b/Tests/FaceLivenessTests/LivenessTests.swift index 5603914a..45b090c4 100644 --- a/Tests/FaceLivenessTests/LivenessTests.swift +++ b/Tests/FaceLivenessTests/LivenessTests.swift @@ -33,7 +33,8 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { videoChunker: videoChunker, closeButtonAction: {}, sessionID: UUID().uuidString, - isPreviewScreenEnabled: false + isPreviewScreenEnabled: false, + cameraPosition: .front ) self.videoChunker = videoChunker From ef85f6c4f44b21243b925f7287d6b3f344336723 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Thu, 22 Aug 2024 13:48:31 -0700 Subject: [PATCH 08/15] chore: update amplify dependency and add sample code to choose challenge type --- Package.resolved | 4 ++-- Package.swift | 2 +- .../Views/Liveness/FaceLivenessDetectionViewModel.swift | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Package.resolved b/Package.resolved index 13524dd4..02973f5a 100644 --- a/Package.resolved +++ b/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { - "branch" : "feat/no-light-support", - "revision" : "614be628cb01188e519bb0e9e4d90bd83703d139" + "branch" : "test/no-light-support", + "revision" : "cdee9437c8bae4be8198a9860d09cd79fdb044ba" } }, { diff --git a/Package.swift b/Package.swift index 446f12c0..01a52b9a 100644 --- a/Package.swift +++ b/Package.swift @@ -14,7 +14,7 @@ let package = Package( ], dependencies: [ // TODO: Change this before merge to main - .package(url: "https://github.com/aws-amplify/amplify-swift", branch: "feat/no-light-support") + .package(url: "https://github.com/aws-amplify/amplify-swift", branch: "test/no-light-support") ], targets: [ .target( diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift index 42e7149a..d00f07a8 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionViewModel.swift @@ -204,6 +204,13 @@ class FaceLivenessDetectionViewModel: ObservableObject { withSessionID: sessionID, userAgent: UserAgentValues.standard().userAgentString, challenges: FaceLivenessSession.supportedChallenges, + + // light challenge + // challenges: [.init(version: "2.0.0", type: .faceMovementAndLightChallenge)], + + // no light challenge + // challenges: [.init(version: "1.0.0", type: .faceMovementChallenge)], + options: .init( attemptCount: Self.attemptCount, preCheckViewEnabled: isPreviewScreenEnabled) From 861c3680d430f8309963628cbf21956ef63e9a6d Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Thu, 22 Aug 2024 14:15:51 -0700 Subject: [PATCH 09/15] update amplify dependency --- Package.resolved | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.resolved b/Package.resolved index 2e1322ac..02973f5a 100644 --- a/Package.resolved +++ b/Package.resolved @@ -6,7 +6,7 @@ "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { "branch" : "test/no-light-support", - "revision" : "6aabc94f91193397b4c57e2b2823dbca4dbc0477" + "revision" : "cdee9437c8bae4be8198a9860d09cd79fdb044ba" } }, { From 3e479bd58834e6cc51e1b295c05a13ff0e399fb0 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Thu, 22 Aug 2024 14:29:58 -0700 Subject: [PATCH 10/15] fix build --- .../FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index 47b560b8..f09c39d4 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -165,7 +165,6 @@ public struct FaceLivenessDetectorView: View { break } } - case .awaitingLivenessSession(let challenge): case .awaitingChallengeType: LoadingPageView() .onAppear { From d74d878b4f4e3326d2651b54208a5a6a5ad6eb06 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Thu, 22 Aug 2024 14:31:58 -0700 Subject: [PATCH 11/15] fix build --- .../Liveness/FaceLivenessDetectionView.swift | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index f09c39d4..a004b730 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -165,39 +165,6 @@ public struct FaceLivenessDetectorView: View { break } } - case .awaitingChallengeType: - LoadingPageView() - .onAppear { - Task { - do { - let session = try await sessionTask.value - viewModel.livenessService = session - viewModel.registerServiceEvents(onChallengeTypeReceived: { challenge in - self.displayState = DisplayState.awaitingLivenessSession(challenge) - }) - viewModel.initializeLivenessStream() - } catch { - throw FaceLivenessDetectionError.accessDenied - } - - DispatchQueue.main.async { - if let faceDetector = viewModel.faceDetector as? FaceDetectorShortRange.Model { - faceDetector.setFaceDetectionSessionConfigurationWrapper(configuration: viewModel) - } - } - } - } - .onReceive(viewModel.$livenessState) { output in - switch output.state { - case .encounteredUnrecoverableError(let error): - let closeCode = error.webSocketCloseCode ?? .normalClosure - viewModel.livenessService?.closeSocket(with: closeCode) - isPresented = false - onCompletion(.failure(mapError(error))) - default: - break - } - } case .awaitingLivenessSession(let challenge): Color.clear .onAppear { From 8e6c9d57b6b9d96b766e164a82f4948df2ae9664 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Thu, 22 Aug 2024 15:00:23 -0700 Subject: [PATCH 12/15] update error codes and message --- .../xcshareddata/swiftpm/Package.resolved | 2 +- .../Liveness/FaceLivenessDetectionError.swift | 14 +++++++------- .../Views/Liveness/LivenessStateMachine.swift | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 2e1322ac..02973f5a 100644 --- a/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/HostApp/HostApp.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -6,7 +6,7 @@ "location" : "https://github.com/aws-amplify/amplify-swift", "state" : { "branch" : "test/no-light-support", - "revision" : "6aabc94f91193397b4c57e2b2823dbca4dbc0477" + "revision" : "cdee9437c8bae4be8198a9860d09cd79fdb044ba" } }, { diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift index f24afd5b..5a626572 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift @@ -126,16 +126,16 @@ public struct FaceLivenessDetectionError: Error, Equatable { recoverySuggestion: "Ensure the device time is correct and try again." ) - public static let invalidCameraPositionSelected = FaceLivenessDetectionError( - code: 18, - message: "The camera position selected is incompatible with the liveness challenge type requested.", - recoverySuggestion: "Please ensure the camera position is supported for the liveness challenge type requested." - ) - public static let cameraNotAvailable = FaceLivenessDetectionError( code: 18, message: "The camera is not available.", - recoverySuggestion: "There might be a hardware issue." + recoverySuggestion: "There might be a hardware issue or the selected camera is not available." + ) + + public static let invalidCameraPositionSelected = FaceLivenessDetectionError( + code: 19, + message: "The camera position selected is incompatible with the liveness challenge type requested.", + recoverySuggestion: "Please ensure the camera position is supported for the liveness challenge type requested." ) public static func == (lhs: FaceLivenessDetectionError, rhs: FaceLivenessDetectionError) -> Bool { diff --git a/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift b/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift index 7c64a360..5df7ec63 100644 --- a/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift +++ b/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift @@ -165,7 +165,7 @@ struct LivenessStateMachine { static let couldNotOpenStream = LivenessError(code: 5, webSocketCloseCode: .unexpectedRuntimeError) static let socketClosed = LivenessError(code: 6, webSocketCloseCode: .normalClosure) static let viewResignation = LivenessError(code: 8, webSocketCloseCode: .viewClosure) - static let cameraNotAvailable = LivenessError(code: 9, webSocketCloseCode: .missingVideoPermission) + static let cameraNotAvailable = LivenessError(code: 9, webSocketCloseCode: .unexpectedRuntimeError) static let invalidCameraPositionSelecteed = LivenessError(code: 10, webSocketCloseCode: .unexpectedRuntimeError) static func == (lhs: LivenessError, rhs: LivenessError) -> Bool { From b9a3aec12714e10556046d8f1742c3e7c94e31f7 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Fri, 30 Aug 2024 12:39:59 -0700 Subject: [PATCH 13/15] Add challengeOption parameter and remove error codes --- .../HostApp/Views/ExampleLivenessView.swift | 6 +-- .../AV/LivenessCaptureDevice.swift | 5 -- .../GetReadyPage/CameraPreviewViewModel.swift | 4 +- .../Views/GetReadyPage/GetReadyPageView.swift | 4 +- .../Liveness/FaceLivenessDetectionError.swift | 8 +-- .../Liveness/FaceLivenessDetectionView.swift | 50 +++++++++++++++---- .../FaceLivenessDetectionViewModel.swift | 29 +++-------- .../Views/Liveness/LivenessStateMachine.swift | 1 - 8 files changed, 53 insertions(+), 54 deletions(-) diff --git a/HostApp/HostApp/Views/ExampleLivenessView.swift b/HostApp/HostApp/Views/ExampleLivenessView.swift index 1163a1e9..ea0c5291 100644 --- a/HostApp/HostApp/Views/ExampleLivenessView.swift +++ b/HostApp/HostApp/Views/ExampleLivenessView.swift @@ -23,7 +23,7 @@ struct ExampleLivenessView: View { FaceLivenessDetectorView( sessionID: viewModel.sessionID, region: "us-east-1", - cameraPosition: .back, + challengeOption: .faceMovementAndLightChallenge, isPresented: Binding( get: { viewModel.presentationState == .liveness }, set: { _ in } @@ -49,8 +49,6 @@ struct ExampleLivenessView: View { viewModel.presentationState = .error(.invalidSignature) case .failure(.cameraNotAvailable): viewModel.presentationState = .error(.cameraNotAvailable) - case .failure(.invalidCameraPositionSelected): - viewModel.presentationState = .error(.invalidCameraPositionSelected) default: viewModel.presentationState = .liveness } @@ -83,8 +81,6 @@ struct ExampleLivenessView: View { LivenessCheckErrorContentView.invalidSignature case .cameraNotAvailable: LivenessCheckErrorContentView.cameraNotAvailable - case . invalidCameraPositionSelected: - LivenessCheckErrorContentView.invalidCameraPositionSelected default: LivenessCheckErrorContentView.unexpected } diff --git a/Sources/FaceLiveness/AV/LivenessCaptureDevice.swift b/Sources/FaceLiveness/AV/LivenessCaptureDevice.swift index 3935459e..1d67e913 100644 --- a/Sources/FaceLiveness/AV/LivenessCaptureDevice.swift +++ b/Sources/FaceLiveness/AV/LivenessCaptureDevice.swift @@ -37,8 +37,3 @@ struct LivenessCaptureDevice { } } } - -public enum LivenessCaptureDevicePosition { - case front - case back -} diff --git a/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewViewModel.swift b/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewViewModel.swift index f40645b5..a46dbaa8 100644 --- a/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewViewModel.swift +++ b/Sources/FaceLiveness/Views/GetReadyPage/CameraPreviewViewModel.swift @@ -16,9 +16,9 @@ class CameraPreviewViewModel: NSObject, ObservableObject { @Published var buffer: CVPixelBuffer? var previewCaptureSession: LivenessCaptureSession? - let cameraPosition: LivenessCaptureDevicePosition + let cameraPosition: LivenessCamera - init(cameraPosition: LivenessCaptureDevicePosition) { + init(cameraPosition: LivenessCamera) { self.cameraPosition = cameraPosition super.init() diff --git a/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift b/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift index d5a87b5b..806aa25e 100644 --- a/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift +++ b/Sources/FaceLiveness/Views/GetReadyPage/GetReadyPageView.swift @@ -12,13 +12,13 @@ struct GetReadyPageView: View { let beginCheckButtonDisabled: Bool let onBegin: () -> Void let challenge: Challenge - let cameraPosition: LivenessCaptureDevicePosition + let cameraPosition: LivenessCamera init( onBegin: @escaping () -> Void, beginCheckButtonDisabled: Bool = false, challenge: Challenge, - cameraPosition: LivenessCaptureDevicePosition + cameraPosition: LivenessCamera ) { self.onBegin = onBegin self.beginCheckButtonDisabled = beginCheckButtonDisabled diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift index 5a626572..19ced079 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionError.swift @@ -129,13 +129,7 @@ public struct FaceLivenessDetectionError: Error, Equatable { public static let cameraNotAvailable = FaceLivenessDetectionError( code: 18, message: "The camera is not available.", - recoverySuggestion: "There might be a hardware issue or the selected camera is not available." - ) - - public static let invalidCameraPositionSelected = FaceLivenessDetectionError( - code: 19, - message: "The camera position selected is incompatible with the liveness challenge type requested.", - recoverySuggestion: "Please ensure the camera position is supported for the liveness challenge type requested." + recoverySuggestion: "There might be a hardware issue." ) public static func == (lhs: FaceLivenessDetectionError, rhs: FaceLivenessDetectionError) -> Bool { diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index a004b730..af975ec6 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -20,7 +20,7 @@ public struct FaceLivenessDetectorView: View { @State var displayingCameraPermissionsNeededAlert = false let disableStartView: Bool - let cameraPosition: LivenessCaptureDevicePosition + let cameraPosition: LivenessCamera let onCompletion: (Result) -> Void let sessionTask: Task @@ -30,13 +30,13 @@ public struct FaceLivenessDetectorView: View { credentialsProvider: AWSCredentialsProvider? = nil, region: String, disableStartView: Bool = false, - cameraPosition: LivenessCaptureDevicePosition = .front, + challengeOption: ChallengeOption, isPresented: Binding, onCompletion: @escaping (Result) -> Void ) { self.disableStartView = disableStartView self._isPresented = isPresented - self.cameraPosition = cameraPosition + self.cameraPosition = challengeOption.camera self.onCompletion = onCompletion self.sessionTask = Task { @@ -82,7 +82,8 @@ public struct FaceLivenessDetectorView: View { closeButtonAction: { onCompletion(.failure(.userCancelled)) }, sessionID: sessionID, isPreviewScreenEnabled: !disableStartView, - cameraPosition: cameraPosition + cameraPosition: challengeOption.camera, + challengeOption: challengeOption ) ) } @@ -92,7 +93,7 @@ public struct FaceLivenessDetectorView: View { credentialsProvider: AWSCredentialsProvider? = nil, region: String, disableStartView: Bool = false, - cameraPosition: LivenessCaptureDevicePosition, + challengeOption: ChallengeOption, isPresented: Binding, onCompletion: @escaping (Result) -> Void, captureSession: LivenessCaptureSession @@ -100,7 +101,7 @@ public struct FaceLivenessDetectorView: View { self.disableStartView = disableStartView self._isPresented = isPresented self.onCompletion = onCompletion - self.cameraPosition = cameraPosition + self.cameraPosition = challengeOption.camera self.sessionTask = Task { let session = try await AWSPredictionsPlugin.startFaceLivenessSession( @@ -125,7 +126,8 @@ public struct FaceLivenessDetectorView: View { closeButtonAction: { onCompletion(.failure(.userCancelled)) }, sessionID: sessionID, isPreviewScreenEnabled: !disableStartView, - cameraPosition: cameraPosition + cameraPosition: challengeOption.camera, + challengeOption: challengeOption ) ) } @@ -243,8 +245,6 @@ public struct FaceLivenessDetectorView: View { return .socketClosed case .cameraNotAvailable: return .cameraNotAvailable - case .invalidCameraPositionSelecteed: - return .invalidCameraPositionSelected default: return .cameraPermissionDenied } @@ -285,7 +285,7 @@ public struct FaceLivenessDetectorView: View { enum DisplayState: Equatable { case awaitingChallengeType case awaitingLivenessSession(Challenge) - case displayingGetReadyView(Challenge, LivenessCaptureDevicePosition) + case displayingGetReadyView(Challenge, LivenessCamera) case displayingLiveness case awaitingCameraPermission @@ -340,3 +340,33 @@ private func map(detectionCompletion: @escaping (Result Void, sessionID: String, isPreviewScreenEnabled: Bool, - cameraPosition: LivenessCaptureDevicePosition + cameraPosition: LivenessCamera, + challengeOption: ChallengeOption ) { self.closeButtonAction = closeButtonAction self.videoChunker = videoChunker @@ -77,6 +79,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { self.faceInOvalMatching = faceInOvalMatching self.isPreviewScreenEnabled = isPreviewScreenEnabled self.cameraPosition = cameraPosition + self.challengeOption = challengeOption self.closeButtonAction = { [weak self] in guard let self else { return } @@ -127,18 +130,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { livenessService?.register( listener: { [weak self] _challenge in self?.challenge = _challenge - guard _challenge.type == .faceMovementAndLightChallenge, - self?.cameraPosition == .back else { - onChallengeTypeReceived(_challenge) - return - } - - // incompatible camera position with challenge type - // return error - DispatchQueue.main.async { - self?.livenessState - .unrecoverableStateEncountered(.invalidCameraPositionSelecteed) - } + onChallengeTypeReceived(_challenge) }, on: .challenge) } @@ -217,14 +209,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { try livenessService?.initializeLivenessStream( withSessionID: sessionID, userAgent: UserAgentValues.standard().userAgentString, - challenges: FaceLivenessSession.supportedChallenges, - - // light challenge - // challenges: [.init(version: "2.0.0", type: .faceMovementAndLightChallenge)], - - // no light challenge - // challenges: [.init(version: "1.0.0", type: .faceMovementChallenge)], - + challenges: [challengeOption.challenge], options: .init( attemptCount: Self.attemptCount, preCheckViewEnabled: isPreviewScreenEnabled) diff --git a/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift b/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift index 5df7ec63..62e563ef 100644 --- a/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift +++ b/Sources/FaceLiveness/Views/Liveness/LivenessStateMachine.swift @@ -166,7 +166,6 @@ struct LivenessStateMachine { static let socketClosed = LivenessError(code: 6, webSocketCloseCode: .normalClosure) static let viewResignation = LivenessError(code: 8, webSocketCloseCode: .viewClosure) static let cameraNotAvailable = LivenessError(code: 9, webSocketCloseCode: .unexpectedRuntimeError) - static let invalidCameraPositionSelecteed = LivenessError(code: 10, webSocketCloseCode: .unexpectedRuntimeError) static func == (lhs: LivenessError, rhs: LivenessError) -> Bool { lhs.code == rhs.code From b7463241c56eee36d4d7d8952cdc5dd3dc47c036 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Tue, 10 Sep 2024 13:31:29 -0700 Subject: [PATCH 14/15] Update ChallengeOptions and use camera position based on challenge type received --- .../HostApp/Views/ExampleLivenessView.swift | 7 +- .../Views/LivenessCheckErrorContentView.swift | 5 -- .../InstructionContainerView.swift | 2 +- .../Liveness/FaceLivenessDetectionView.swift | 85 +++++++++---------- ...ViewModel+FaceDetectionResultHandler.swift | 4 +- .../FaceLivenessDetectionViewModel.swift | 62 +++++++++----- .../CredentialsProviderTestCase.swift | 10 ++- Tests/FaceLivenessTests/LivenessTests.swift | 10 +-- 8 files changed, 102 insertions(+), 83 deletions(-) diff --git a/HostApp/HostApp/Views/ExampleLivenessView.swift b/HostApp/HostApp/Views/ExampleLivenessView.swift index ea0c5291..27ab8429 100644 --- a/HostApp/HostApp/Views/ExampleLivenessView.swift +++ b/HostApp/HostApp/Views/ExampleLivenessView.swift @@ -23,7 +23,8 @@ struct ExampleLivenessView: View { FaceLivenessDetectorView( sessionID: viewModel.sessionID, region: "us-east-1", - challengeOption: .faceMovementAndLightChallenge, + challengeOptions: .init(faceMovementChallengeOption: FaceMovementChallengeOption(camera: .front), + faceMovementAndLightChallengeOption: FaceMovementAndLightChallengeOption()), isPresented: Binding( get: { viewModel.presentationState == .liveness }, set: { _ in } @@ -47,6 +48,10 @@ struct ExampleLivenessView: View { viewModel.presentationState = .error(.countdownFaceTooClose) case .failure(.invalidSignature): viewModel.presentationState = .error(.invalidSignature) + case .failure(.faceInOvalMatchExceededTimeLimitError): + viewModel.presentationState = .error(.faceInOvalMatchExceededTimeLimitError) + case .failure(.internalServer): + viewModel.presentationState = .error(.internalServer) case .failure(.cameraNotAvailable): viewModel.presentationState = .error(.cameraNotAvailable) default: diff --git a/HostApp/HostApp/Views/LivenessCheckErrorContentView.swift b/HostApp/HostApp/Views/LivenessCheckErrorContentView.swift index 78998a7d..b8df88f1 100644 --- a/HostApp/HostApp/Views/LivenessCheckErrorContentView.swift +++ b/HostApp/HostApp/Views/LivenessCheckErrorContentView.swift @@ -60,11 +60,6 @@ extension LivenessCheckErrorContentView { name: "The camera could not be started.", description: "There might be a hardware issue with the camera." ) - - static let invalidCameraPositionSelected = LivenessCheckErrorContentView( - name: "The camera position selected is incompatible with the liveness challenge type requested.", - description: "Please ensure the camera position is supported for the liveness challenge type requested." - ) } diff --git a/Sources/FaceLiveness/Views/Instruction/InstructionContainerView.swift b/Sources/FaceLiveness/Views/Instruction/InstructionContainerView.swift index 5ed45ae7..01dcd37f 100644 --- a/Sources/FaceLiveness/Views/Instruction/InstructionContainerView.swift +++ b/Sources/FaceLiveness/Views/Instruction/InstructionContainerView.swift @@ -110,7 +110,7 @@ struct InstructionContainerView: View { ) } case .faceMatched: - if let challenge = viewModel.challenge, + if let challenge = viewModel.challengeReceived, case .faceMovementAndLightChallenge = challenge.type { InstructionView( text: LocalizedStrings.challenge_instruction_hold_still, diff --git a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift index af975ec6..40e2504d 100644 --- a/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift +++ b/Sources/FaceLiveness/Views/Liveness/FaceLivenessDetectionView.swift @@ -20,7 +20,7 @@ public struct FaceLivenessDetectorView: View { @State var displayingCameraPermissionsNeededAlert = false let disableStartView: Bool - let cameraPosition: LivenessCamera + let challengeOptions: ChallengeOptions let onCompletion: (Result) -> Void let sessionTask: Task @@ -30,14 +30,14 @@ public struct FaceLivenessDetectorView: View { credentialsProvider: AWSCredentialsProvider? = nil, region: String, disableStartView: Bool = false, - challengeOption: ChallengeOption, + challengeOptions: ChallengeOptions, isPresented: Binding, onCompletion: @escaping (Result) -> Void ) { self.disableStartView = disableStartView self._isPresented = isPresented - self.cameraPosition = challengeOption.camera self.onCompletion = onCompletion + self.challengeOptions = challengeOptions self.sessionTask = Task { let session = try await AWSPredictionsPlugin.startFaceLivenessSession( @@ -59,31 +59,16 @@ public struct FaceLivenessDetectorView: View { assetWriterDelegate: VideoChunker.AssetWriterDelegate(), assetWriterInput: LivenessAVAssetWriterInput() ) - - let avCaptureDevice = AVCaptureDevice.default( - .builtInWideAngleCamera, - for: .video, - position: cameraPosition == .front ? .front : .back) - - let captureSession = LivenessCaptureSession( - captureDevice: .init(avCaptureDevice: avCaptureDevice), - outputDelegate: OutputSampleBufferCapturer( - faceDetector: faceDetector, - videoChunker: videoChunker - ) - ) self._viewModel = StateObject( wrappedValue: .init( faceDetector: faceDetector, faceInOvalMatching: faceInOvalStateMatching, - captureSession: captureSession, videoChunker: videoChunker, closeButtonAction: { onCompletion(.failure(.userCancelled)) }, sessionID: sessionID, isPreviewScreenEnabled: !disableStartView, - cameraPosition: challengeOption.camera, - challengeOption: challengeOption + challengeOptions: challengeOptions ) ) } @@ -93,7 +78,7 @@ public struct FaceLivenessDetectorView: View { credentialsProvider: AWSCredentialsProvider? = nil, region: String, disableStartView: Bool = false, - challengeOption: ChallengeOption, + challengeOptions: ChallengeOptions, isPresented: Binding, onCompletion: @escaping (Result) -> Void, captureSession: LivenessCaptureSession @@ -101,7 +86,7 @@ public struct FaceLivenessDetectorView: View { self.disableStartView = disableStartView self._isPresented = isPresented self.onCompletion = onCompletion - self.cameraPosition = challengeOption.camera + self.challengeOptions = challengeOptions self.sessionTask = Task { let session = try await AWSPredictionsPlugin.startFaceLivenessSession( @@ -121,13 +106,11 @@ public struct FaceLivenessDetectorView: View { wrappedValue: .init( faceDetector: captureSession.outputSampleBufferCapturer!.faceDetector, faceInOvalMatching: faceInOvalStateMatching, - captureSession: captureSession, videoChunker: captureSession.outputSampleBufferCapturer!.videoChunker, closeButtonAction: { onCompletion(.failure(.userCancelled)) }, sessionID: sessionID, isPreviewScreenEnabled: !disableStartView, - cameraPosition: challengeOption.camera, - challengeOption: challengeOption + challengeOptions: challengeOptions ) ) } @@ -172,6 +155,14 @@ public struct FaceLivenessDetectorView: View { .onAppear { Task { do { + let cameraPosition: LivenessCamera + switch challenge.type { + case .faceMovementAndLightChallenge: + cameraPosition = challengeOptions.faceMovementAndLightChallengeOption.camera + case .faceMovementChallenge: + cameraPosition = challengeOptions.faceMovementChallengeOption.camera + } + let newState = disableStartView ? DisplayState.displayingLiveness : DisplayState.displayingGetReadyView(challenge, cameraPosition) @@ -255,7 +246,7 @@ public struct FaceLivenessDetectorView: View { for: .video, completionHandler: { accessGranted in guard accessGranted == true else { return } - guard let challenge = viewModel.challenge else { return } + guard let challenge = viewModel.challengeReceived else { return } displayState = .awaitingLivenessSession(challenge) } ) @@ -274,7 +265,7 @@ public struct FaceLivenessDetectorView: View { case .restricted, .denied: alertCameraAccessNeeded() case .authorized: - guard let challenge = viewModel.challenge else { return } + guard let challenge = viewModel.challengeReceived else { return } displayState = .awaitingLivenessSession(challenge) @unknown default: break @@ -341,32 +332,38 @@ private func map(detectionCompletion: @escaping (Result Void let videoChunker: VideoChunker let sessionID: String @@ -35,7 +35,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { var hasSentFirstVideo = false var layerRectConverted: (CGRect) -> CGRect = { $0 } var sessionConfiguration: FaceLivenessSession.SessionConfiguration? - var challenge: Challenge? + var challengeReceived: Challenge? var normalizeFace: (DetectedFace) -> DetectedFace = { $0 } var provideSingleFrame: ((UIImage) -> Void)? var cameraViewRect = CGRect.zero @@ -44,8 +44,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { var initialClientEvent: InitialClientEvent? var faceMatchedTimestamp: UInt64? var noFitStartTime: Date? - let cameraPosition: LivenessCamera - let challengeOption: ChallengeOption + let challengeOptions: ChallengeOptions static var attemptCount: Int = 0 static var attemptIdTimeStamp: Date = Date() @@ -61,25 +60,21 @@ class FaceLivenessDetectionViewModel: ObservableObject { init( faceDetector: FaceDetector, faceInOvalMatching: FaceInOvalMatching, - captureSession: LivenessCaptureSession, videoChunker: VideoChunker, stateMachine: LivenessStateMachine = .init(state: .initial), closeButtonAction: @escaping () -> Void, sessionID: String, isPreviewScreenEnabled: Bool, - cameraPosition: LivenessCamera, - challengeOption: ChallengeOption + challengeOptions: ChallengeOptions ) { self.closeButtonAction = closeButtonAction self.videoChunker = videoChunker self.livenessState = stateMachine self.sessionID = sessionID - self.captureSession = captureSession self.faceDetector = faceDetector self.faceInOvalMatching = faceInOvalMatching self.isPreviewScreenEnabled = isPreviewScreenEnabled - self.cameraPosition = cameraPosition - self.challengeOption = challengeOption + self.challengeOptions = challengeOptions self.closeButtonAction = { [weak self] in guard let self else { return } @@ -129,7 +124,8 @@ class FaceLivenessDetectionViewModel: ObservableObject { livenessService?.register( listener: { [weak self] _challenge in - self?.challenge = _challenge + self?.challengeReceived = _challenge + self?.configureCaptureSession(challenge: _challenge) onChallengeTypeReceived(_challenge) }, on: .challenge) @@ -144,16 +140,16 @@ class FaceLivenessDetectionViewModel: ObservableObject { } func startSession() { - captureSession.startSession() + captureSession?.startSession() } func stopRecording() { - captureSession.stopRunning() + captureSession?.stopRunning() } func configureCamera(withinFrame frame: CGRect) -> CALayer? { do { - let avLayer = try captureSession.configureCamera(frame: frame) + let avLayer = try captureSession?.configureCamera(frame: frame) DispatchQueue.main.async { self.livenessState.checkIsFacePrepared() } @@ -209,7 +205,8 @@ class FaceLivenessDetectionViewModel: ObservableObject { try livenessService?.initializeLivenessStream( withSessionID: sessionID, userAgent: UserAgentValues.standard().userAgentString, - challenges: [challengeOption.challenge], + challenges: [challengeOptions.faceMovementChallengeOption.challenge, + challengeOptions.faceMovementAndLightChallengeOption.challenge], options: .init( attemptCount: Self.attemptCount, preCheckViewEnabled: isPreviewScreenEnabled) @@ -258,7 +255,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { videoStartTime: UInt64 ) { guard initialClientEvent == nil else { return } - guard let challenge else { return } + guard let challengeReceived else { return } videoChunker.start() @@ -278,8 +275,8 @@ class FaceLivenessDetectionViewModel: ObservableObject { do { try livenessService?.send( .initialFaceDetected(event: _initialClientEvent, - challenge: .init(version: challenge.version, - type: challenge.type)), + challenge: .init(version: challengeReceived.version, + type: challengeReceived.type)), eventDate: { .init() } ) } catch { @@ -298,7 +295,7 @@ class FaceLivenessDetectionViewModel: ObservableObject { let sessionConfiguration, let initialClientEvent, let faceMatchedTimestamp, - let challenge + let challengeReceived else { return } let finalClientEvent = FinalClientEvent( @@ -313,8 +310,8 @@ class FaceLivenessDetectionViewModel: ObservableObject { do { try livenessService?.send( .final(event: finalClientEvent, - challenge: .init(version: challenge.version, - type: challenge.type)), + challenge: .init(version: challengeReceived.version, + type: challengeReceived.type)), eventDate: { .init() } ) @@ -407,6 +404,29 @@ class FaceLivenessDetectionViewModel: ObservableObject { } return data } + + func configureCaptureSession(challenge: Challenge) { + let cameraPosition: LivenessCamera + switch challenge.type { + case .faceMovementChallenge: + cameraPosition = challengeOptions.faceMovementChallengeOption.camera + case .faceMovementAndLightChallenge: + cameraPosition = challengeOptions.faceMovementAndLightChallengeOption.camera + } + + let avCaptureDevice = AVCaptureDevice.default( + .builtInWideAngleCamera, + for: .video, + position: cameraPosition == .front ? .front : .back) + + self.captureSession = LivenessCaptureSession( + captureDevice: .init(avCaptureDevice: avCaptureDevice), + outputDelegate: OutputSampleBufferCapturer( + faceDetector: self.faceDetector, + videoChunker: self.videoChunker + ) + ) + } } extension FaceLivenessDetectionViewModel: FaceDetectionSessionConfigurationWrapper { } diff --git a/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift b/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift index 5b82119a..396ef60a 100644 --- a/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift +++ b/Tests/FaceLivenessTests/CredentialsProviderTestCase.swift @@ -38,12 +38,12 @@ final class CredentialsProviderTestCase: XCTestCase { let viewModel = FaceLivenessDetectionViewModel( faceDetector: faceDetector, faceInOvalMatching: .init(instructor: .init()), - captureSession: captureSession, videoChunker: videoChunker, closeButtonAction: {}, sessionID: UUID().uuidString, isPreviewScreenEnabled: false, - cameraPosition: .front + challengeOptions: .init(faceMovementChallengeOption: .init(camera: .front), + faceMovementAndLightChallengeOption: .init()) ) self.videoChunker = videoChunker @@ -67,7 +67,8 @@ final class CredentialsProviderTestCase: XCTestCase { sessionID: UUID().uuidString, credentialsProvider: credentialsProvider, region: "us-east-1", - cameraPosition: .front, + challengeOptions: .init(faceMovementChallengeOption: .init(camera: .front), + faceMovementAndLightChallengeOption: .init()), isPresented: .constant(true), onCompletion: { _ in } ) @@ -104,7 +105,8 @@ final class CredentialsProviderTestCase: XCTestCase { sessionID: UUID().uuidString, credentialsProvider: credentialsProvider, region: "us-east-1", - cameraPosition: .front, + challengeOptions: .init(faceMovementChallengeOption: .init(camera: .front), + faceMovementAndLightChallengeOption: .init()), isPresented: .constant(true), onCompletion: { _ in } ) diff --git a/Tests/FaceLivenessTests/LivenessTests.swift b/Tests/FaceLivenessTests/LivenessTests.swift index d1f7944a..0307124f 100644 --- a/Tests/FaceLivenessTests/LivenessTests.swift +++ b/Tests/FaceLivenessTests/LivenessTests.swift @@ -29,12 +29,12 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { let viewModel = FaceLivenessDetectionViewModel( faceDetector: faceDetector, faceInOvalMatching: .init(instructor: .init()), - captureSession: captureSession, videoChunker: videoChunker, closeButtonAction: {}, sessionID: UUID().uuidString, isPreviewScreenEnabled: false, - cameraPosition: .front + challengeOptions: .init(faceMovementChallengeOption: .init(camera: .front), + faceMovementAndLightChallengeOption: .init()) ) self.videoChunker = videoChunker @@ -71,8 +71,8 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { /// Then: The end state of this flow is `.faceMatched` func testHappyPathToMatchedFace() async throws { viewModel.livenessService = self.livenessService - viewModel.challenge = Challenge(version: "2.0.0", type: .faceMovementAndLightChallenge) - viewModel.challenge = Challenge(version: "2.0.0", type: .faceMovementAndLightChallenge) + viewModel.challengeReceived = Challenge(version: "2.0.0", type: .faceMovementAndLightChallenge) + viewModel.challengeReceived = Challenge(version: "2.0.0", type: .faceMovementAndLightChallenge) viewModel.livenessState.checkIsFacePrepared() XCTAssertEqual(viewModel.livenessState.state, .pendingFacePreparedConfirmation(.pendingCheck)) @@ -117,7 +117,7 @@ final class FaceLivenessDetectionViewModelTestCase: XCTestCase { /// Then: The end state of this flow is `.recording(ovalDisplayed: false)` func testTransitionToRecordingState() async throws { viewModel.livenessService = self.livenessService - viewModel.challenge = Challenge(version: "2.0.0", type: .faceMovementAndLightChallenge) + viewModel.challengeReceived = Challenge(version: "2.0.0", type: .faceMovementAndLightChallenge) let face = FaceLivenessSession.OvalMatchChallenge.Face( distanceThreshold: 0.32, From bf516df7f19c2818412db3a544fd1c1860019e60 Mon Sep 17 00:00:00 2001 From: Abhash Kumar Singh Date: Mon, 16 Sep 2024 13:00:09 -0700 Subject: [PATCH 15/15] Add UI changes for selecting back camera in HostApp --- .../HostApp/Views/ExampleLivenessView.swift | 34 +++++++++------- .../Views/ExampleLivenessViewModel.swift | 7 ++-- HostApp/HostApp/Views/RootView.swift | 17 +++++--- HostApp/HostApp/Views/StartSessionView.swift | 39 +++++++++++++++++-- 4 files changed, 70 insertions(+), 27 deletions(-) diff --git a/HostApp/HostApp/Views/ExampleLivenessView.swift b/HostApp/HostApp/Views/ExampleLivenessView.swift index 27ab8429..5152ca17 100644 --- a/HostApp/HostApp/Views/ExampleLivenessView.swift +++ b/HostApp/HostApp/Views/ExampleLivenessView.swift @@ -9,24 +9,28 @@ import SwiftUI import FaceLiveness struct ExampleLivenessView: View { - @Binding var isPresented: Bool + @Binding var containerViewState: ContainerViewState @ObservedObject var viewModel: ExampleLivenessViewModel - init(sessionID: String, isPresented: Binding) { - self.viewModel = .init(sessionID: sessionID) - self._isPresented = isPresented + init(sessionID: String, containerViewState: Binding) { + self._containerViewState = containerViewState + if case let .liveness(selectedCamera) = _containerViewState.wrappedValue { + self.viewModel = .init(sessionID: sessionID, presentationState: .liveness(selectedCamera)) + } else { + self.viewModel = .init(sessionID: sessionID) + } } var body: some View { switch viewModel.presentationState { - case .liveness: + case .liveness(let camera): FaceLivenessDetectorView( sessionID: viewModel.sessionID, region: "us-east-1", - challengeOptions: .init(faceMovementChallengeOption: FaceMovementChallengeOption(camera: .front), + challengeOptions: .init(faceMovementChallengeOption: FaceMovementChallengeOption(camera: camera), faceMovementAndLightChallengeOption: FaceMovementAndLightChallengeOption()), isPresented: Binding( - get: { viewModel.presentationState == .liveness }, + get: { viewModel.presentationState == .liveness(camera) }, set: { _ in } ), onCompletion: { result in @@ -35,11 +39,11 @@ struct ExampleLivenessView: View { case .success: withAnimation { viewModel.presentationState = .result } case .failure(.sessionNotFound), .failure(.cameraPermissionDenied), .failure(.accessDenied): - viewModel.presentationState = .liveness - isPresented = false + viewModel.presentationState = .liveness(camera) + containerViewState = .startSession case .failure(.userCancelled): - viewModel.presentationState = .liveness - isPresented = false + viewModel.presentationState = .liveness(camera) + containerViewState = .startSession case .failure(.sessionTimedOut): viewModel.presentationState = .error(.sessionTimedOut) case .failure(.socketClosed): @@ -55,16 +59,16 @@ struct ExampleLivenessView: View { case .failure(.cameraNotAvailable): viewModel.presentationState = .error(.cameraNotAvailable) default: - viewModel.presentationState = .liveness + viewModel.presentationState = .liveness(camera) } } } ) - .id(isPresented) + .id(containerViewState) case .result: LivenessResultView( sessionID: viewModel.sessionID, - onTryAgain: { isPresented = false }, + onTryAgain: { containerViewState = .startSession }, content: { LivenessResultContentView(fetchResults: viewModel.fetchLivenessResult) } @@ -73,7 +77,7 @@ struct ExampleLivenessView: View { case .error(let detectionError): LivenessResultView( sessionID: viewModel.sessionID, - onTryAgain: { isPresented = false }, + onTryAgain: { containerViewState = .startSession }, content: { switch detectionError { case .socketClosed: diff --git a/HostApp/HostApp/Views/ExampleLivenessViewModel.swift b/HostApp/HostApp/Views/ExampleLivenessViewModel.swift index a04571bc..7dade2fb 100644 --- a/HostApp/HostApp/Views/ExampleLivenessViewModel.swift +++ b/HostApp/HostApp/Views/ExampleLivenessViewModel.swift @@ -10,11 +10,12 @@ import FaceLiveness import Amplify class ExampleLivenessViewModel: ObservableObject { - @Published var presentationState = PresentationState.liveness + @Published var presentationState: PresentationState = .liveness(.front) let sessionID: String - init(sessionID: String) { + init(sessionID: String, presentationState: PresentationState = .liveness(.front)) { self.sessionID = sessionID + self.presentationState = presentationState } func fetchLivenessResult() async throws -> LivenessResultContentView.Result { @@ -30,6 +31,6 @@ class ExampleLivenessViewModel: ObservableObject { } enum PresentationState: Equatable { - case liveness, result, error(FaceLivenessDetectionError) + case liveness(LivenessCamera), result, error(FaceLivenessDetectionError) } } diff --git a/HostApp/HostApp/Views/RootView.swift b/HostApp/HostApp/Views/RootView.swift index 7600f1b4..59a3c815 100644 --- a/HostApp/HostApp/Views/RootView.swift +++ b/HostApp/HostApp/Views/RootView.swift @@ -6,25 +6,32 @@ // import SwiftUI +import FaceLiveness struct RootView: View { @EnvironmentObject var sceneDelegate: SceneDelegate @State var sessionID = "" - @State var isPresentingContainerView = false + @State var containerViewState = ContainerViewState.startSession var body: some View { - if isPresentingContainerView { + switch containerViewState { + case .liveness: ExampleLivenessView( sessionID: sessionID, - isPresented: $isPresentingContainerView + containerViewState: $containerViewState ) - } else { + case .startSession: StartSessionView( sessionID: $sessionID, - isPresentingContainerView: $isPresentingContainerView + containerViewState: $containerViewState ) .background(Color.dynamicColors(light: .white, dark: .secondarySystemBackground)) .edgesIgnoringSafeArea(.all) } } } + +enum ContainerViewState: Hashable { + case liveness(LivenessCamera) + case startSession +} diff --git a/HostApp/HostApp/Views/StartSessionView.swift b/HostApp/HostApp/Views/StartSessionView.swift index 42f64401..6905e9b2 100644 --- a/HostApp/HostApp/Views/StartSessionView.swift +++ b/HostApp/HostApp/Views/StartSessionView.swift @@ -12,7 +12,7 @@ struct StartSessionView: View { @EnvironmentObject var sceneDelegate: SceneDelegate @ObservedObject var viewModel = StartSessionViewModel() @Binding var sessionID: String - @Binding var isPresentingContainerView: Bool + @Binding var containerViewState: ContainerViewState @State private var showAlert = false var body: some View { @@ -26,7 +26,7 @@ struct StartSessionView: View { ) button( - text: "Create Liveness Session", + text: "Create Liveness Session (front camera)", backgroundColor: .dynamicColors( light: .hex("#047D95"), dark: .hex("#7dd6e8") @@ -35,7 +35,7 @@ struct StartSessionView: View { viewModel.createSession { sessionId, err in if let sessionId = sessionId { sessionID = sessionId - isPresentingContainerView = true + containerViewState = .liveness(.front) } showAlert = err != nil @@ -50,7 +50,38 @@ struct StartSessionView: View { dismissButton: .default( Text("OK"), action: { - isPresentingContainerView = false + containerViewState = .startSession + } + ) + ) + } + + button( + text: "Create Liveness Session (back camera)", + backgroundColor: .dynamicColors( + light: .hex("#047D95"), + dark: .hex("#7dd6e8") + ), + action: { + viewModel.createSession { sessionId, err in + if let sessionId = sessionId { + sessionID = sessionId + containerViewState = .liveness(.back) + } + + showAlert = err != nil + } + }, + enabled: viewModel.isSignedIn + ) + .alert(isPresented: $showAlert) { + Alert( + title: Text("Error Creating Liveness Session"), + message: Text("Unable to create a liveness session id. Please try again."), + dismissButton: .default( + Text("OK"), + action: { + containerViewState = .startSession } ) )