diff --git a/docs/integration_tests.md b/docs/integration_tests.md new file mode 100644 index 0000000000..d3497ecacf --- /dev/null +++ b/docs/integration_tests.md @@ -0,0 +1,62 @@ +# Integration Tests + +Integration tests in Flutter allow self-driving end-to-end +testing of app code running with the full GUI. + +This document is about using integration tests to capture +performance metrics on physical devices. For more +information on that topic see +[Flutter cookbook on integration profiling][profiling-cookbook]. + +For more background on integration testing in general +see [Flutter docs on integration testing][flutter-docs]. + +[profiling-cookbook]: https://docs.flutter.dev/cookbook/testing/integration/profiling +[flutter-docs]: https://docs.flutter.dev/testing/integration-tests + + +## Capturing performance metrics + +Capturing performance metrics involves two parts: an +integration test that runs on a device and driver code that +runs on the host. + +Integration test code is written in a similar style as +widget test code, using a `testWidgets` function as well as +a `WidgetTester` instance to arrange widgets and run +interactions. A difference is the usage of +`IntegrationTestWidgetsFlutterBinding` which provides a +`traceAction` method used to record Dart VM timelines. + +Driver code runs on the host and is useful to configure +output of captured timeline data. There is a baseline driver +at `integration_test/perf_driver.dart` that additionally +configures output of a timeline summary containing widget +build times and frame rendering performance. + + +## Obtaining performance metrics + +First, obtain a device ID using `flutter devices`. + +The command to run an integration test on a device: + +``` +$ flutter drive \ + --driver=integration_test/perf_driver.dart \ + --target=integration_test/unreadmarker_test.dart \ + --profile \ + --no-dds \ + -d +``` + +A data file with raw event timings will be produced in +`build/trace_output.timeline.json`. + +A more readily consumable file will also be produced in +`build/trace_output.timeline_summary.json`. This file +contains widget build and render timing data in a JSON +structure. See the fields `frame_build_times` and +`frame_rasterizer_times` as well as the provided percentile +scores of those. These values are useful for objective +comparison between different runs. diff --git a/integration_test/perf_driver.dart b/integration_test/perf_driver.dart new file mode 100644 index 0000000000..49bf21e255 --- /dev/null +++ b/integration_test/perf_driver.dart @@ -0,0 +1,21 @@ +// This integration driver configures output of timeline data +// and a summary thereof from integration tests. See +// docs/integration_tests.md for background. + +import 'package:flutter_driver/flutter_driver.dart' as driver; +import 'package:integration_test/integration_test_driver.dart'; + +Future main() { + // See cookbook recipe for this sort of driver: + // https://docs.flutter.dev/cookbook/testing/integration/profiling#3-save-the-results-to-disk + return integrationDriver( + responseDataCallback: (data) async { + if (data == null) return; + final timeline = driver.Timeline.fromJson(data['timeline']); + final summary = driver.TimelineSummary.summarize(timeline); + await summary.writeTimelineToFile( + 'trace_output', + pretty: true, + includeSummary: true); + }); +} diff --git a/integration_test/unreadmarker_test.dart b/integration_test/unreadmarker_test.dart new file mode 100644 index 0000000000..2db4222dc2 --- /dev/null +++ b/integration_test/unreadmarker_test.dart @@ -0,0 +1,64 @@ + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:zulip/api/model/events.dart'; +import 'package:zulip/api/model/model.dart'; +import 'package:zulip/model/narrow.dart'; +import 'package:zulip/model/store.dart'; +import 'package:zulip/widgets/message_list.dart'; +import 'package:zulip/widgets/store.dart'; + +import '../test/api/fake_api.dart'; +import '../test/example_data.dart' as eg; +import '../test/model/binding.dart'; +import '../test/model/message_list_test.dart'; + +void main() { + final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + TestZulipBinding.ensureInitialized(); + + late PerAccountStore store; + late FakeApiConnection connection; + + Future> setupMessageListPage(WidgetTester tester, int messageCount) async { + addTearDown(testBinding.reset); + await testBinding.globalStore.add(eg.selfAccount, eg.initialSnapshot()); + store = await testBinding.globalStore.perAccount(eg.selfAccount.id); + connection = store.connection as FakeApiConnection; + + // prepare message list data + final messages = List.generate(messageCount, + (i) => eg.streamMessage(flags: [MessageFlag.read])); + connection.prepare(json: + newestResult(foundOldest: true, messages: messages).toJson()); + + await tester.pumpWidget( + MaterialApp( + home: GlobalStoreWidget( + child: PerAccountStoreWidget( + accountId: eg.selfAccount.id, + child: const MessageListPage(narrow: AllMessagesNarrow()))))); + await tester.pumpAndSettle(); + return messages; + } + + testWidgets('_UnreadMarker animation performance test', (tester) async { + // This integration test is meant for measuring performance. + // See docs/integration_test.md for how to use it. + + final messages = await setupMessageListPage(tester, 500); + await binding.traceAction(() async { + store.handleEvent(eg.updateMessageFlagsRemoveEvent( + MessageFlag.read, + messages)); + await tester.pumpAndSettle(); + store.handleEvent(UpdateMessageFlagsAddEvent( + id: 1, + flag: MessageFlag.read, + messages: messages.map((e) => e.id).toList(), + all: false)); + await tester.pumpAndSettle(); + }); + }); +}