First cut of MRTD (passport/id card) reader library. Scans passport a…

…nd reads data from it through NFC. (#461)

Signed-off-by: Peter Sorotokin <[email protected]>

gradle/libs.versions.toml

@@ -35,6 +35,12 @@
     junit-jupiter = "5.10.0"
     truth = "1.1.5"
     navigation-compose = "2.7.5"
+    camera-version = "1.3.1"
+    scuba-version = "0.0.25"
+    jmrtd-version = "0.7.30"
+    mlkit-version = "16.0.0"
+    kotlinx-coroutines = "1.8.0-RC2"
+    junit = "4.13.2"
 
 [libraries]
     androidx-core-ktx = { module = "androidx.core:core-ktx", version.ref = "core-ktx" }
@@ -54,13 +60,17 @@
     androidx-navigation-ktx = { module = "androidx.navigation:navigation-fragment-ktx", version.ref = "navigation" }
     androidx-navigation-ui-ktx = { module = "androidx.navigation:navigation-ui-ktx", version.ref = "navigation" }
 
+    kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-android", version.ref="kotlinx-coroutines" }
+    kotlinx-coroutines-guava = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-guava", version.ref="kotlinx-coroutines" }
+
     compose-bom = { module = "androidx.compose:compose-bom", version.ref="compose-bom" }
     compose-ui = { module = "androidx.compose.ui:ui" }
     compose-foundation = { module = "androidx.compose.foundation:foundation" }
     compose-material = { module = "androidx.compose.material3:material3" }
     compose-ui-tooling = { module = "androidx.compose.ui:ui-tooling-preview" }
     compose-preview = { module = "androidx.compose.material:material-icons-extended" }
     compose-icons = { module = "androidx.compose.ui:ui-tooling" }
+    compose-runtime-livedata = { module = "androidx.compose.runtime:runtime-livedata" }
 
     volley = { module = "com.android.volley:volley", version.ref = "volley" }
 
@@ -88,6 +98,14 @@
     truth = { module = "com.google.truth:truth", version.ref = "truth" }
     androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigation-compose" }
 
+    androidx-camera-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "camera-version" }
+    androidx-camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "camera-version" }
+    androidx-camera-camera2 = { group = "androidx.camera", name = "camera-camera2", version.ref = "camera-version" }
+    net-sf-scuba-scuba-sc-android = { group = "net.sf.scuba", name = "scuba-sc-android", version.ref="scuba-version" }
+    org-jmrtd-jmrtd = { group = "org.jmrtd", name ="jmrtd", version.ref="jmrtd-version" }
+    com-google-mlkit-text-recognition = { group = "com.google.mlkit", name = "text-recognition", version.ref="mlkit-version" }
+junit = { group = "junit", name = "junit", version.ref = "junit" }
+
 [bundles]
     androidx-core = ["androidx-core-ktx", "androidx-appcompat", "androidx-material", "androidx-contraint-layout", "androidx-fragment-ktx", "androidx-legacy-v4", "androidx-preference-ktx", "androidx-work"]
     androidx-lifecycle = ["androidx-lifecycle-extensions", "androidx-lifecycle-livedata", "androidx-lifecycle-viewmodel"]

mrtd-reader/.gitignore

@@ -0,0 +1 @@
+/build

mrtd-reader/build.gradle

@@ -0,0 +1,54 @@
+plugins {
+    alias(libs.plugins.android.library)
+    alias(libs.plugins.kotlin.android)
+}
+
+android {
+    namespace 'com.android.identity_credential.icao_reader'
+    compileSdk 34
+
+    defaultConfig {
+        minSdk 27
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+        consumerProguardFiles "consumer-rules.pro"
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_11
+        targetCompatibility JavaVersion.VERSION_11
+    }
+    kotlinOptions {
+        jvmTarget = '11'
+    }
+}
+
+dependencies {
+
+    implementation libs.androidx.core.ktx
+    implementation libs.androidx.appcompat
+    implementation libs.androidx.material
+
+    implementation libs.androidx.camera.view
+    implementation libs.androidx.camera.camera2
+    implementation libs.androidx.camera.lifecycle
+    implementation libs.net.sf.scuba.scuba.sc.android
+    implementation libs.org.jmrtd.jmrtd
+    implementation libs.com.google.mlkit.text.recognition
+
+    implementation libs.kotlinx.coroutines.guava
+
+    androidTestImplementation libs.androidx.test.ext.junit
+    androidTestImplementation libs.androidx.test.espresso
+    androidTestImplementation platform(libs.compose.bom)
+    testImplementation libs.bundles.unit.testing
+    testRuntimeOnly libs.junit.jupiter.engine
+
+    testImplementation project(":identity-android")
+}

mrtd-reader/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

mrtd-reader/src/main/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+</manifest>

mrtd-reader/src/main/java/com/android/identity_credential/mrtd/MrtdDecodedData.kt

@@ -0,0 +1,17 @@
+package com.android.identity_credential.mrtd
+
+import android.graphics.Bitmap
+
+/**
+ * Data read from the passport or ID card.
+ *
+ * This data is typically produced from [MrtdNfcData] using [MrtdNfcDataDecoder].
+ */
+data class MrtdDecodedData(
+    val firstName: String,
+    val lastName: String,
+    val state: String,
+    val nationality: String,
+    val gender: String,
+    val photo: Bitmap?
+)

mrtd-reader/src/main/java/com/android/identity_credential/mrtd/MrtdMrzChecksumValidator.kt

@@ -0,0 +1,38 @@
+package com.android.identity_credential.mrtd
+
+private val checksumWeights = arrayOf(7, 3, 1)
+
+/**
+ * Validates that a data field in Machine Readable Zone (MRZ) was scanned and OCRed correctly from
+ * an ICAO Machine-readable Travel Document (MRTD).
+ *
+ * See [ICAO Spec](https://www.icao.int/publications/Documents/9303_p3_cons_en.pdf) for details.
+ */
+class MrtdMrzChecksumValidator(
+    private val checksumRanges: List<Range>,
+    private val checksumDigitIndex: Int) {
+
+    data class Range(val start: Int, val end: Int)
+
+    fun validate(line: String): Boolean {
+        val checkChar = line[checksumDigitIndex]
+        if (checkChar < '0' || checkChar > '9') {
+            return false
+        }
+        var checksum = 0
+        var i = 0
+        for (range in checksumRanges) {
+            for (index in range.start..range.end) {
+                val value = when (val character = line[index]) {
+                    in '0'..'9' -> character.code - '0'.code
+                    in 'A'..'Z' -> character.code - 'A'.code + 10
+                    '<' -> 0
+                    else -> return false
+                }
+                checksum += value * checksumWeights[i % checksumWeights.size]
+                i++
+            }
+        }
+        return checkChar.code - '0'.code == checksum % 10
+    }
+}

mrtd-reader/src/main/java/com/android/identity_credential/mrtd/MrtdMrzData.kt

@@ -0,0 +1,15 @@
+package com.android.identity_credential.mrtd
+
+/**
+ * Represents a small set of data that is typically captured by OCR from Machine Readable Zone (MRZ)
+ * on a passport (or other MRTD).
+ *
+ * These particular field are important as they provide one way to gain access to the NFC chip in
+ * an MRTD. Other info in MRZ (and more) can then be captured in a much better way from the NFC
+ * chip.
+ */
+data class MrtdMrzData(
+    val documentNumber: String,
+    val dateOfBirth: String,
+    val dateOfExpiration: String
+)

mrtd-reader/src/main/java/com/android/identity_credential/mrtd/MrtdMrzScanner.kt

@@ -0,0 +1,105 @@
+package com.android.identity_credential.mrtd
+
+import android.os.Handler
+import android.util.Log
+import android.util.Size
+import androidx.activity.ComponentActivity
+import androidx.camera.core.CameraSelector
+import androidx.camera.core.ExperimentalGetImage
+import androidx.camera.core.ImageAnalysis
+import androidx.camera.core.ImageProxy
+import androidx.camera.core.Preview
+import androidx.camera.lifecycle.ProcessCameraProvider
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.text.TextRecognition
+import com.google.mlkit.vision.text.latin.TextRecognizerOptions
+import kotlinx.coroutines.guava.await
+import java.util.concurrent.Executors
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
+import kotlin.coroutines.suspendCoroutine
+
+/**
+ * OCR scanner that extracts [MrtdMrzData] from camera feed.
+ *
+ * TODO: hook cancellation
+ */
+class MrtdMrzScanner(private val mActivity: ComponentActivity) {
+    companion object {
+        private const val TAG = "MrtdMrzScanner"
+    }
+
+    /**
+     * Starts camera and scans camera feed until it finds passport/ID image that contains valid
+     * data.
+     */
+    suspend fun readFromCamera(surfaceProvider: Preview.SurfaceProvider): MrtdMrzData {
+        val provider = ProcessCameraProvider.getInstance(mActivity).await()
+        return onCameraReady(provider, surfaceProvider)
+    }
+
+    private suspend fun onCameraReady(
+        cameraProvider: ProcessCameraProvider, surfaceProvider: Preview.SurfaceProvider
+    ): MrtdMrzData {
+        val mainHandler = Handler(mActivity.mainLooper)
+
+        // Preview
+        val previewUseCase = Preview.Builder().build()
+        previewUseCase.setSurfaceProvider(surfaceProvider)
+
+        val analysisBuilder = ImageAnalysis.Builder()
+        // we take the portrait format of the Image.
+        analysisBuilder.setTargetResolution(Size(4 * 480, 4 * 640))
+        val analysisUseCase = analysisBuilder.build()
+        val executor = Executors.newFixedThreadPool(1)!!
+        return suspendCoroutine { continuation ->
+            analysisUseCase.setAnalyzer(
+                executor
+            ) { image ->
+                try {
+                    recognize(image) { pictureData ->
+                        if (!executor.isShutdown) {
+                            executor.shutdown()
+                            cameraProvider.unbindAll()
+                            mainHandler.post {
+                                continuation.resume(pictureData)
+                            }
+                        }
+                    }
+                } catch (e: Exception) {
+                    if (!executor.isShutdown) {
+                        executor.shutdown()
+                        mainHandler.post {
+                            continuation.resumeWithException(e)
+                        }
+                    }
+                }
+            }
+
+            // Unbind use cases before rebinding
+            cameraProvider.unbindAll()
+
+            // Bind use cases to camera
+            cameraProvider.bindToLifecycle(
+                mActivity, CameraSelector.DEFAULT_BACK_CAMERA, previewUseCase, analysisUseCase
+            )
+        }
+    }
+
+    @androidx.annotation.OptIn(ExperimentalGetImage::class)
+    private fun recognize(imageProxy: ImageProxy, dataCb: (keyData: MrtdMrzData) -> Unit) {
+        val mediaImage = imageProxy.image ?: return
+        val image = InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees)
+        val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
+        recognizer.process(image).addOnSuccessListener { visionText ->
+                imageProxy.close()
+                val newValue = extractMrtdMrzData(visionText.text)
+                if (newValue != null) {
+                    dataCb(newValue)
+                }
+            }.addOnFailureListener { err ->
+                Log.e(TAG, "Error scanning: $err")
+                imageProxy.close()
+            }
+    }
+}

mrtd-reader/src/main/java/com/android/identity_credential/mrtd/MrtdNfcData.kt

@@ -0,0 +1,32 @@
+package com.android.identity_credential.mrtd
+
+/**
+ * Raw data read from the passport or ID card.
+ *
+ * This data is typically produced by [MrtdNfcReader] or [MrtdNfcScanner]. It is in its raw
+ * encoded form as it comes from the card and it is not validated. [MrtdNfcDataDecoder] can
+ * validate and decode it.
+ */
+data class MrtdNfcData(
+    val dg1: ByteArray,
+    val dg2: ByteArray,
+    val sod: ByteArray
+) {
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as MrtdNfcData
+
+        if (!dg1.contentEquals(other.dg1)) return false
+        if (!dg2.contentEquals(other.dg2)) return false
+        return sod.contentEquals(other.sod)
+    }
+
+    override fun hashCode(): Int {
+        var result = dg1.contentHashCode()
+        result = 31 * result + dg2.contentHashCode()
+        result = 31 * result + sod.contentHashCode()
+        return result
+    }
+}