From 97e3c856dfc8a24785500935ea3b275765c188b1 Mon Sep 17 00:00:00 2001 From: jonnyandrew Date: Fri, 17 Nov 2023 09:58:21 +0000 Subject: [PATCH] [Android] Take a screenshot when instrumentation tests fail (#871) --- .github/workflows/android.yml | 7 ++- platforms/android/gradle/libs.versions.toml | 1 + platforms/android/library/build.gradle | 1 + .../wysiwyg/EditorEditTextInputTests.kt | 9 ++++ .../test/utils/ScreenshotFailureHandler.kt | 54 +++++++++++++++++++ platforms/android/scripts/ci_test.sh | 23 ++++++++ 6 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 platforms/android/library/src/androidTest/java/io/element/android/wysiwyg/test/utils/ScreenshotFailureHandler.kt create mode 100755 platforms/android/scripts/ci_test.sh diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 7bbc8fc19..b02433afb 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -92,10 +92,8 @@ jobs: disable-animations: true enable-hw-keyboard: true script: | - ./gradlew unitTestsWithCoverage ${{ env.CI_GRADLE_ARG_PROPERTIES }} - ./gradlew generateUnitTestCoverageReport ${{ env.CI_GRADLE_ARG_PROPERTIES }} - ./gradlew instrumentationTestsWithCoverage ${{ env.CI_GRADLE_ARG_PROPERTIES }} - ./gradlew generateInstrumentationTestCoverageReport ${{ env.CI_GRADLE_ARG_PROPERTIES }} + chmod +x scripts/ci_test.sh + scripts/ci_test.sh - name : Upload test results if : ${{ always() }} @@ -105,6 +103,7 @@ jobs: path : | ./**/build/reports/tests/** ./**/build/reports/androidTests/connected/** + ./**/build/reports/screenshots/** - name: Upload unit test coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/platforms/android/gradle/libs.versions.toml b/platforms/android/gradle/libs.versions.toml index e1e7a6150..9ed0adf90 100644 --- a/platforms/android/gradle/libs.versions.toml +++ b/platforms/android/gradle/libs.versions.toml @@ -71,4 +71,5 @@ test-turbine = { module="app.cash.turbine:turbine", version="1.0.0" } test-androidx-junit = { module = "androidx.test.ext:junit", version.ref = "androidx-junit" } test-androidx-espresso = { module="androidx.test.espresso:espresso-core", version.ref="espresso" } test-androidx-espresso-accessibility = { module="androidx.test.espresso:espresso-accessibility", version.ref="espresso" } +test-androidx-uiautomator = "androidx.test.uiautomator:uiautomator:2.2.0" test-mockk-android = { module="io.mockk:mockk-android", version.ref="mockk" } diff --git a/platforms/android/library/build.gradle b/platforms/android/library/build.gradle index 37de1956b..b8d3e15f9 100644 --- a/platforms/android/library/build.gradle +++ b/platforms/android/library/build.gradle @@ -111,6 +111,7 @@ dependencies { androidTestImplementation libs.test.androidx.espresso androidTestImplementation libs.test.androidx.espresso.accessibility androidTestImplementation libs.test.mockk.android + androidTestImplementation libs.test.androidx.uiautomator } android.libraryVariants.all { variant -> diff --git a/platforms/android/library/src/androidTest/java/io/element/android/wysiwyg/EditorEditTextInputTests.kt b/platforms/android/library/src/androidTest/java/io/element/android/wysiwyg/EditorEditTextInputTests.kt index eef95f52b..1510de3b0 100644 --- a/platforms/android/library/src/androidTest/java/io/element/android/wysiwyg/EditorEditTextInputTests.kt +++ b/platforms/android/library/src/androidTest/java/io/element/android/wysiwyg/EditorEditTextInputTests.kt @@ -14,7 +14,9 @@ import android.view.View import android.widget.EditText import android.widget.TextView import androidx.core.text.getSpans +import androidx.test.core.app.ApplicationProvider import androidx.test.espresso.Espresso.onView +import androidx.test.espresso.Espresso.setFailureHandler import androidx.test.espresso.accessibility.AccessibilityChecks import androidx.test.espresso.action.ViewActions.pressKey import androidx.test.espresso.action.ViewActions.replaceText @@ -58,6 +60,13 @@ class EditorEditTextInputTests { AccessibilityChecks.enable() } + @Before + fun setUp() { + setFailureHandler( + ScreenshotFailureHandler(ApplicationProvider.getApplicationContext()) + ) + } + @After fun cleanUp() { // Finish composing just in case, to prevent clashes between test cases diff --git a/platforms/android/library/src/androidTest/java/io/element/android/wysiwyg/test/utils/ScreenshotFailureHandler.kt b/platforms/android/library/src/androidTest/java/io/element/android/wysiwyg/test/utils/ScreenshotFailureHandler.kt new file mode 100644 index 000000000..98d1effc3 --- /dev/null +++ b/platforms/android/library/src/androidTest/java/io/element/android/wysiwyg/test/utils/ScreenshotFailureHandler.kt @@ -0,0 +1,54 @@ +package io.element.android.wysiwyg.test.utils + +import android.content.ContentValues +import android.content.Context +import android.graphics.Bitmap +import android.os.Environment +import android.provider.MediaStore +import android.view.View +import androidx.test.espresso.FailureHandler +import androidx.test.espresso.base.DefaultFailureHandler +import androidx.test.platform.app.InstrumentationRegistry.getInstrumentation +import org.hamcrest.Matcher +import java.io.IOException +import java.text.SimpleDateFormat +import java.util.Calendar + +class ScreenshotFailureHandler(appContext: Context) : FailureHandler { + private val defaultHandler: FailureHandler = DefaultFailureHandler(appContext) + + override fun handle(error: Throwable, viewMatcher: Matcher) { + getInstrumentation() + .uiAutomation + .takeScreenshot() + .save() + + defaultHandler.handle(error, viewMatcher) + } +} + +private fun Bitmap.save() { + val timestamp = timestamp() + val contentResolver = getInstrumentation().targetContext.applicationContext.contentResolver + try { + val contentValues = ContentValues().apply { + put(MediaStore.MediaColumns.DISPLAY_NAME, "$timestamp.jpeg") + put(MediaStore.Images.Media.RELATIVE_PATH, "${Environment.DIRECTORY_PICTURES}/UiTest") + } + + val uri = contentResolver + .insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, contentValues) + ?: return + + contentResolver.openOutputStream(uri)?.use { outputStream -> + compress(Bitmap.CompressFormat.PNG, 20, outputStream) + } + + contentResolver.update(uri, contentValues, null, null) + } catch (e: IOException) { + e.printStackTrace() + } +} + +private fun timestamp(): String = + SimpleDateFormat("yyyyMMdd_HHmmss").format(Calendar.getInstance().time) diff --git a/platforms/android/scripts/ci_test.sh b/platforms/android/scripts/ci_test.sh new file mode 100755 index 000000000..a40cbd066 --- /dev/null +++ b/platforms/android/scripts/ci_test.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +./gradlew unitTestsWithCoverage $CI_GRADLE_ARG_PROPERTIES +./gradlew generateUnitTestCoverageReport $CI_GRADLE_ARG_PROPERTIES + +# Don't exit immediately from UI test failure to collect screenshots +set +e + +./gradlew instrumentationTestsWithCoverage $CI_GRADLE_ARG_PROPERTIES + +UI_TEST_EXIT_CODE=$? +if [ $UI_TEST_EXIT_CODE -ne 0 ]; then + echo "UI tests failed." + echo "Pulling screenshots from device..." + adb shell ls /sdcard/Pictures/UiTest/ + mkdir build/reports/screenshots + adb pull /sdcard/Pictures/UiTest/ build/reports/screenshots/ + exit $UI_TEST_EXIT_CODE +fi +set -e + +./gradlew generateInstrumentationTestCoverageReport $CI_GRADLE_ARG_PROPERTIES +