From ba4785e609a14114671523bb55f394eec4b1d7f6 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Wed, 11 Sep 2024 12:35:58 -0700 Subject: [PATCH 01/14] Unskip the wasm benchmark --- packages/devtools_app/benchmark/devtools_benchmarks_test.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart index a6c64226d6f..58a0b59733e 100644 --- a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -40,10 +40,6 @@ void main() { await _runBenchmarks(useWasm: useWasm); }, timeout: const Timeout(Duration(minutes: 10)), - // TODO(https://github.com/dart-lang/sdk/issues/56664): unskip the wasm - // benchmarks once this issue is resolved and we can bump DevTools to a - // version of Flutter that includes the fix. - skip: useWasm, ); } From d8c88b695bc51d662d347a8aa33de9013b1ac992 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Wed, 11 Sep 2024 13:54:01 -0700 Subject: [PATCH 02/14] bump benchmarks dep --- packages/devtools_app/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 55328e7dd66..83071f7fc51 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -76,7 +76,7 @@ dev_dependencies: mockito: ^5.4.1 stager: ^1.0.1 test: ^1.21.0 - web_benchmarks: ^2.0.0 + web_benchmarks: ^2.0.2 webkit_inspection_protocol: ">=0.5.0 <2.0.0" flutter: From 337285afa1f33f9758b5466bc4eb9073dc658082 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Wed, 11 Sep 2024 16:17:23 -0700 Subject: [PATCH 03/14] Include query params --- .../benchmark/devtools_benchmarks_test.dart | 4 +++- .../test_infra/client/client_js.dart | 17 +++++++++++++++ .../client_shared.dart} | 15 +++---------- .../test_infra/client/client_wasm.dart | 21 +++++++++++++++++++ .../benchmark/test_infra/common.dart | 2 ++ packages/devtools_app/pubspec.yaml | 2 +- 6 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 packages/devtools_app/benchmark/test_infra/client/client_js.dart rename packages/devtools_app/benchmark/test_infra/{client.dart => client/client_shared.dart} (57%) create mode 100644 packages/devtools_app/benchmark/test_infra/client/client_wasm.dart diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart index 58a0b59733e..3ed950eaf2b 100644 --- a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -50,12 +50,14 @@ Future _runBenchmarks({bool useWasm = false}) async { stdout.writeln('Starting web benchmark tests ...'); final taskResult = await serveWebBenchmark( benchmarkAppDirectory: projectRootDirectory(), - entryPoint: 'benchmark/test_infra/client.dart', + entryPoint: + 'benchmark/test_infra/client/client_${useWasm ? 'wasm' : 'js'}.dart', compilationOptions: useWasm ? const CompilationOptions.wasm() : const CompilationOptions.js(), treeShakeIcons: false, initialPage: benchmarkInitialPage, + queryParameters: useWasm ? wasmQueryParameters : null, ); stdout.writeln('Web benchmark tests finished.'); diff --git a/packages/devtools_app/benchmark/test_infra/client/client_js.dart b/packages/devtools_app/benchmark/test_infra/client/client_js.dart new file mode 100644 index 00000000000..394c44cd861 --- /dev/null +++ b/packages/devtools_app/benchmark/test_infra/client/client_js.dart @@ -0,0 +1,17 @@ +// Copyright 2023 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:web_benchmarks/client.dart'; + +import '../common.dart'; +import 'client_shared.dart'; + +/// Runs the client of the DevTools web benchmarks. +/// +/// When the DevTools web benchmarks are run, the server builds an app with this +/// file as the entry point (see `run_benchmarks.dart`). The app automates +/// the DevTools web app, records some performance data, and reports them. +Future main() async { + await runBenchmarks(benchmarks, initialPage: benchmarkInitialPage); +} diff --git a/packages/devtools_app/benchmark/test_infra/client.dart b/packages/devtools_app/benchmark/test_infra/client/client_shared.dart similarity index 57% rename from packages/devtools_app/benchmark/test_infra/client.dart rename to packages/devtools_app/benchmark/test_infra/client/client_shared.dart index cc59063bfd2..66e86a76d0f 100644 --- a/packages/devtools_app/benchmark/test_infra/client.dart +++ b/packages/devtools_app/benchmark/test_infra/client/client_shared.dart @@ -1,11 +1,11 @@ -// Copyright 2023 The Chromium Authors. All rights reserved. +// Copyright 2024 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'package:web_benchmarks/client.dart'; -import 'common.dart'; -import 'devtools_recorder.dart'; +import '../common.dart'; +import '../devtools_recorder.dart'; typedef RecorderFactory = Recorder Function(); @@ -20,12 +20,3 @@ final benchmarks = { benchmark: DevToolsBenchmark.offlinePerformanceScreen, ), }; - -/// Runs the client of the DevTools web benchmarks. -/// -/// When the DevTools web benchmarks are run, the server builds an app with this -/// file as the entry point (see `run_benchmarks.dart`). The app automates -/// the DevTools web app, records some performance data, and reports them. -Future main() async { - await runBenchmarks(benchmarks, initialPage: benchmarkInitialPage); -} diff --git a/packages/devtools_app/benchmark/test_infra/client/client_wasm.dart b/packages/devtools_app/benchmark/test_infra/client/client_wasm.dart new file mode 100644 index 00000000000..53d53d407c7 --- /dev/null +++ b/packages/devtools_app/benchmark/test_infra/client/client_wasm.dart @@ -0,0 +1,21 @@ +// Copyright 2024 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:web_benchmarks/client.dart'; + +import '../common.dart'; +import 'client_shared.dart'; + +/// Runs the client of the DevTools web benchmarks. +/// +/// When the DevTools web benchmarks are run, the server builds an app with this +/// file as the entry point (see `run_benchmarks.dart`). The app automates +/// the DevTools web app, records some performance data, and reports them. +Future main() async { + await runBenchmarks( + benchmarks, + initialPage: benchmarkInitialPage, + queryParameters: wasmQueryParameters, + ); +} diff --git a/packages/devtools_app/benchmark/test_infra/common.dart b/packages/devtools_app/benchmark/test_infra/common.dart index c3f5b462d52..737af05c3a4 100644 --- a/packages/devtools_app/benchmark/test_infra/common.dart +++ b/packages/devtools_app/benchmark/test_infra/common.dart @@ -10,6 +10,8 @@ // found" in DevTools. const benchmarkInitialPage = ''; +const wasmQueryParameters = {'wasm': 'true'}; + const devtoolsBenchmarkPrefix = 'devtools'; enum DevToolsBenchmark { diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index 83071f7fc51..4e6f2599cee 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -76,7 +76,7 @@ dev_dependencies: mockito: ^5.4.1 stager: ^1.0.1 test: ^1.21.0 - web_benchmarks: ^2.0.2 + web_benchmarks: ^2.1.0 webkit_inspection_protocol: ">=0.5.0 <2.0.0" flutter: From 5d091fc8cdf726a38b21b27b4f8c151262573324 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Thu, 12 Sep 2024 16:37:47 -0700 Subject: [PATCH 04/14] Benchmarks 3.0.0 --- .../devtools_app/benchmark/devtools_benchmarks_test.dart | 3 +-- .../devtools_app/benchmark/scripts/run_benchmarks.dart | 2 +- .../benchmark/test_infra/client/client_js.dart | 2 +- .../benchmark/test_infra/client/client_wasm.dart | 6 +----- packages/devtools_app/benchmark/test_infra/common.dart | 9 +++++++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart index 3ed950eaf2b..1337cd205a8 100644 --- a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -56,8 +56,7 @@ Future _runBenchmarks({bool useWasm = false}) async { ? const CompilationOptions.wasm() : const CompilationOptions.js(), treeShakeIcons: false, - initialPage: benchmarkInitialPage, - queryParameters: useWasm ? wasmQueryParameters : null, + benchmarkPath: benchmarkPath(useWasm: useWasm), ); stdout.writeln('Web benchmark tests finished.'); diff --git a/packages/devtools_app/benchmark/scripts/run_benchmarks.dart b/packages/devtools_app/benchmark/scripts/run_benchmarks.dart index ee711fc40e8..3566945b0a0 100644 --- a/packages/devtools_app/benchmark/scripts/run_benchmarks.dart +++ b/packages/devtools_app/benchmark/scripts/run_benchmarks.dart @@ -60,7 +60,7 @@ Future runBenchmarks({ ? const CompilationOptions.wasm() : const CompilationOptions.js(), treeShakeIcons: false, - initialPage: benchmarkInitialPage, + benchmarkPath: benchmarkPath(useWasm: useWasm), headless: !useBrowser, ), ); diff --git a/packages/devtools_app/benchmark/test_infra/client/client_js.dart b/packages/devtools_app/benchmark/test_infra/client/client_js.dart index 394c44cd861..108b8c09445 100644 --- a/packages/devtools_app/benchmark/test_infra/client/client_js.dart +++ b/packages/devtools_app/benchmark/test_infra/client/client_js.dart @@ -13,5 +13,5 @@ import 'client_shared.dart'; /// file as the entry point (see `run_benchmarks.dart`). The app automates /// the DevTools web app, records some performance data, and reports them. Future main() async { - await runBenchmarks(benchmarks, initialPage: benchmarkInitialPage); + await runBenchmarks(benchmarks, benchmarkPath: benchmarkPath(useWasm: false)); } diff --git a/packages/devtools_app/benchmark/test_infra/client/client_wasm.dart b/packages/devtools_app/benchmark/test_infra/client/client_wasm.dart index 53d53d407c7..bc133d33c45 100644 --- a/packages/devtools_app/benchmark/test_infra/client/client_wasm.dart +++ b/packages/devtools_app/benchmark/test_infra/client/client_wasm.dart @@ -13,9 +13,5 @@ import 'client_shared.dart'; /// file as the entry point (see `run_benchmarks.dart`). The app automates /// the DevTools web app, records some performance data, and reports them. Future main() async { - await runBenchmarks( - benchmarks, - initialPage: benchmarkInitialPage, - queryParameters: wasmQueryParameters, - ); + await runBenchmarks(benchmarks, benchmarkPath: benchmarkPath(useWasm: true)); } diff --git a/packages/devtools_app/benchmark/test_infra/common.dart b/packages/devtools_app/benchmark/test_infra/common.dart index 737af05c3a4..67ec67b9134 100644 --- a/packages/devtools_app/benchmark/test_infra/common.dart +++ b/packages/devtools_app/benchmark/test_infra/common.dart @@ -8,9 +8,14 @@ // We use an empty initial page so that the benchmark server does not attempt // to load the default page 'index.html', which will show up as "page not // found" in DevTools. -const benchmarkInitialPage = ''; +const _benchmarkInitialPage = ''; -const wasmQueryParameters = {'wasm': 'true'}; +const _wasmQueryParameters = {'wasm': 'true'}; + +String benchmarkPath({required bool useWasm}) => Uri( + path: _benchmarkInitialPage, + queryParameters: useWasm ? _wasmQueryParameters : null, + ).toString(); const devtoolsBenchmarkPrefix = 'devtools'; From 3cd7a729e1e39bf11a827f5c7d7cf14cb82c1668 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Mon, 30 Sep 2024 14:15:12 -0700 Subject: [PATCH 05/14] fix test --- flutter-candidate.txt | 2 +- .../benchmark/devtools_benchmarks_test.dart | 39 +++++++++++++------ .../automators/devtools_automator.dart | 12 ++++++ .../benchmark/test_infra/common.dart | 8 ++-- .../test_infra/devtools_recorder.dart | 1 + 5 files changed, 45 insertions(+), 17 deletions(-) diff --git a/flutter-candidate.txt b/flutter-candidate.txt index 9c9b22f0851..d1156e34586 100644 --- a/flutter-candidate.txt +++ b/flutter-candidate.txt @@ -1 +1 @@ -8925e1ffdfe880b06a8bc5877f017083d6652f5b +e3828ec9402204bb81fcdb9c5fb7154367b4a6dd diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart index 1337cd205a8..be563e7f280 100644 --- a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -8,13 +8,14 @@ import 'dart:convert' show JsonEncoder; import 'dart:io'; +import 'package:collection/collection.dart'; import 'package:test/test.dart'; import 'package:web_benchmarks/server.dart'; import 'test_infra/common.dart'; import 'test_infra/project_root_directory.dart'; -List metricList({required bool useWasm}) => [ +List _metricList({required bool useWasm}) => [ // The skwasm renderer doesn't have preroll or apply frame steps in its // rendering. if (!useWasm) ...[ @@ -24,13 +25,22 @@ List metricList({required bool useWasm}) => [ 'drawFrameDuration', ]; -final valueList = [ +final _valueList = [ 'average', 'outlierAverage', 'outlierRatio', 'noise', ]; +const _totalUiFrameAverageScore = 'totalUiFrame.average'; + +const _isWasmScore = 'isWasm'; + +const _extraScores = [ + _totalUiFrameAverageScore, + _isWasmScore, +]; + /// Tests that the DevTools web benchmarks are run and reported correctly. void main() { for (final useWasm in [true, false]) { @@ -66,29 +76,34 @@ Future _runBenchmarks({bool useWasm = false}) async { ); for (final benchmarkName in DevToolsBenchmark.values.map((e) => e.id)) { - final expectedMetrics = metricList(useWasm: useWasm); + final expectedMetrics = _metricList(useWasm: useWasm); + final scores = taskResult.scores[benchmarkName] ?? []; expect( - taskResult.scores[benchmarkName], - hasLength(expectedMetrics.length * valueList.length + 1), + scores, + hasLength( + expectedMetrics.length * _valueList.length + _extraScores.length, + ), ); for (final metricName in expectedMetrics) { - for (final valueName in valueList) { + for (final valueName in _valueList) { expect( - taskResult.scores[benchmarkName]?.where( - (score) => score.metric == '$metricName.$valueName', - ), + scores.where((score) => score.metric == '$metricName.$valueName'), hasLength(1), ); } } expect( - taskResult.scores[benchmarkName]?.where( - (score) => score.metric == 'totalUiFrame.average', - ), + scores.where((score) => score.metric == _totalUiFrameAverageScore), hasLength(1), ); + + final isWasmScore = scores.firstWhereOrNull( + (BenchmarkScore score) => score.metric == _isWasmScore, + ); + expect(isWasmScore, isNotNull); + expect(isWasmScore!.value, useWasm ? 1 : 0); } expect( diff --git a/packages/devtools_app/benchmark/test_infra/automators/devtools_automator.dart b/packages/devtools_app/benchmark/test_infra/automators/devtools_automator.dart index 48613274d1b..034f380f809 100644 --- a/packages/devtools_app/benchmark/test_infra/automators/devtools_automator.dart +++ b/packages/devtools_app/benchmark/test_infra/automators/devtools_automator.dart @@ -9,8 +9,10 @@ import 'dart:async'; import 'package:devtools_app/devtools_app.dart'; import 'package:devtools_test/helpers.dart'; import 'package:devtools_test/test_data.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:web_benchmarks/client.dart'; import '../common.dart'; import '_cpu_profiler_automator.dart'; @@ -21,6 +23,7 @@ class DevToolsAutomater { DevToolsAutomater({ required this.benchmark, required this.stopWarmingUpCallback, + required this.profile, }); /// The current benchmark. @@ -32,6 +35,9 @@ class DevToolsAutomater { /// as over. final void Function() stopWarmingUpCallback; + /// The profile collected for the running benchmark + final Profile profile; + /// Whether the automation has ended. bool finished = false; @@ -65,6 +71,12 @@ class DevToolsAutomater { await PerformanceScreenAutomator(controller).run(); } + // Record whether we are in wasm mode or not. Ideally, we'd have a more + // first-class way to add metadata like this, but this will work for us to + // pass information about the environment back to the server for the + // purposes of our own tests. + profile.extraData['isWasm'] = kIsWasm ? 1 : 0; + // At the end of the test, mark as finished. finished = true; } diff --git a/packages/devtools_app/benchmark/test_infra/common.dart b/packages/devtools_app/benchmark/test_infra/common.dart index 67ec67b9134..ba34517d0e3 100644 --- a/packages/devtools_app/benchmark/test_infra/common.dart +++ b/packages/devtools_app/benchmark/test_infra/common.dart @@ -4,10 +4,10 @@ /// The initial page to load upon opening the DevTools benchmark app or /// reloading it in Chrome. -// -// We use an empty initial page so that the benchmark server does not attempt -// to load the default page 'index.html', which will show up as "page not -// found" in DevTools. +/// +/// We use an empty initial page so that the benchmark server does not attempt +/// to load the default page 'index.html', which will show up as "page not +/// found" in DevTools. const _benchmarkInitialPage = ''; const _wasmQueryParameters = {'wasm': 'true'}; diff --git a/packages/devtools_app/benchmark/test_infra/devtools_recorder.dart b/packages/devtools_app/benchmark/test_infra/devtools_recorder.dart index 2353fd7b75c..7a5d178ff8b 100644 --- a/packages/devtools_app/benchmark/test_infra/devtools_recorder.dart +++ b/packages/devtools_app/benchmark/test_infra/devtools_recorder.dart @@ -34,6 +34,7 @@ class DevToolsRecorder extends WidgetRecorder { _devToolsAutomator = DevToolsAutomater( benchmark: benchmark, stopWarmingUpCallback: profile.stopWarmingUp, + profile: profile, ); return _devToolsAutomator!.createWidget(); } From 0038a17e519048cd8085a16521d59c24e5f817d0 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Mon, 30 Sep 2024 14:16:24 -0700 Subject: [PATCH 06/14] pubspec.yaml --- packages/devtools_app/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index e864478a9bf..f646cd2dd5c 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -76,7 +76,7 @@ dev_dependencies: mockito: ^5.4.1 stager: ^1.0.1 test: ^1.21.0 - web_benchmarks: ^2.1.0 + web_benchmarks: ^3.0.0 webkit_inspection_protocol: ">=0.5.0 <2.0.0" flutter: From 0a57beedfb249e573b3fb977c058fbfe0a3fa86c Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Mon, 30 Sep 2024 14:17:44 -0700 Subject: [PATCH 07/14] year --- .../devtools_app/benchmark/test_infra/client/client_js.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/devtools_app/benchmark/test_infra/client/client_js.dart b/packages/devtools_app/benchmark/test_infra/client/client_js.dart index 108b8c09445..92ccf72d7ba 100644 --- a/packages/devtools_app/benchmark/test_infra/client/client_js.dart +++ b/packages/devtools_app/benchmark/test_infra/client/client_js.dart @@ -1,4 +1,4 @@ -// Copyright 2023 The Chromium Authors. All rights reserved. +// Copyright 2024 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. From f676c05fcb482d27e81f572dab77b6dd6aa1a0b5 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 10:23:13 -0700 Subject: [PATCH 08/14] flutter version --- flutter-candidate.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter-candidate.txt b/flutter-candidate.txt index d1156e34586..0892039e5de 100644 --- a/flutter-candidate.txt +++ b/flutter-candidate.txt @@ -1 +1 @@ -e3828ec9402204bb81fcdb9c5fb7154367b4a6dd +b05246d305ac94798ca5429754917c0afa92fd9b From 723dc783ebb02ef12467a19f1645f7e8e8226391 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 11:48:51 -0700 Subject: [PATCH 09/14] re-use --- packages/devtools_app/benchmark/devtools_benchmarks_test.dart | 3 +-- packages/devtools_app/benchmark/scripts/run_benchmarks.dart | 2 +- packages/devtools_app/benchmark/test_infra/common.dart | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart index be563e7f280..ebc129d2ef8 100644 --- a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -60,8 +60,7 @@ Future _runBenchmarks({bool useWasm = false}) async { stdout.writeln('Starting web benchmark tests ...'); final taskResult = await serveWebBenchmark( benchmarkAppDirectory: projectRootDirectory(), - entryPoint: - 'benchmark/test_infra/client/client_${useWasm ? 'wasm' : 'js'}.dart', + entryPoint: generateBenchmarkEntryPoint(useWasm: useWasm), compilationOptions: useWasm ? const CompilationOptions.wasm() : const CompilationOptions.js(), diff --git a/packages/devtools_app/benchmark/scripts/run_benchmarks.dart b/packages/devtools_app/benchmark/scripts/run_benchmarks.dart index 3566945b0a0..57ce768dcc0 100644 --- a/packages/devtools_app/benchmark/scripts/run_benchmarks.dart +++ b/packages/devtools_app/benchmark/scripts/run_benchmarks.dart @@ -55,7 +55,7 @@ Future runBenchmarks({ benchmarkResults.add( await serveWebBenchmark( benchmarkAppDirectory: projectRootDirectory(), - entryPoint: 'benchmark/test_infra/client.dart', + entryPoint: generateBenchmarkEntryPoint(useWasm: useWasm), compilationOptions: useWasm ? const CompilationOptions.wasm() : const CompilationOptions.js(), diff --git a/packages/devtools_app/benchmark/test_infra/common.dart b/packages/devtools_app/benchmark/test_infra/common.dart index ba34517d0e3..939d881efaa 100644 --- a/packages/devtools_app/benchmark/test_infra/common.dart +++ b/packages/devtools_app/benchmark/test_infra/common.dart @@ -17,6 +17,10 @@ String benchmarkPath({required bool useWasm}) => Uri( queryParameters: useWasm ? _wasmQueryParameters : null, ).toString(); +String generateBenchmarkEntryPoint({required bool useWasm}) { + return 'benchmark/test_infra/client/client_${useWasm ? 'wasm' : 'js'}.dart'; +} + const devtoolsBenchmarkPrefix = 'devtools'; enum DevToolsBenchmark { From dbdaa57db8d154a1c8092abe03f1172011b38d40 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Wed, 2 Oct 2024 11:01:01 -0700 Subject: [PATCH 10/14] upgrade to latest web_benchmarks changes --- .../benchmark/devtools_benchmarks_test.dart | 35 ++++++------------- .../scripts/dart2wasm_performance_diff.dart | 6 ++-- packages/devtools_app/pubspec.yaml | 2 +- 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart index ebc129d2ef8..30f55a4502e 100644 --- a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -10,34 +10,16 @@ import 'dart:io'; import 'package:collection/collection.dart'; import 'package:test/test.dart'; +import 'package:web_benchmarks/metrics.dart'; import 'package:web_benchmarks/server.dart'; import 'test_infra/common.dart'; import 'test_infra/project_root_directory.dart'; -List _metricList({required bool useWasm}) => [ - // The skwasm renderer doesn't have preroll or apply frame steps in its - // rendering. - if (!useWasm) ...[ - 'preroll_frame', - 'apply_frame', - ], - 'drawFrameDuration', - ]; - -final _valueList = [ - 'average', - 'outlierAverage', - 'outlierRatio', - 'noise', -]; - -const _totalUiFrameAverageScore = 'totalUiFrame.average'; - const _isWasmScore = 'isWasm'; const _extraScores = [ - _totalUiFrameAverageScore, + totalUiFrameAverage, _isWasmScore, ]; @@ -75,26 +57,29 @@ Future _runBenchmarks({bool useWasm = false}) async { ); for (final benchmarkName in DevToolsBenchmark.values.map((e) => e.id)) { - final expectedMetrics = _metricList(useWasm: useWasm); + final expectedMetrics = expectedBenchmarkMetrics(useWasm: useWasm) + .map((BenchmarkMetric metric) => metric.label) + .toList(); + const expectedComputations = BenchmarkMetricComputation.values; final scores = taskResult.scores[benchmarkName] ?? []; expect( scores, hasLength( - expectedMetrics.length * _valueList.length + _extraScores.length, + expectedMetrics.length * expectedComputations.length + _extraScores.length, ), ); for (final metricName in expectedMetrics) { - for (final valueName in _valueList) { + for (final computation in expectedComputations) { expect( - scores.where((score) => score.metric == '$metricName.$valueName'), + scores.where((score) => score.metric == '$metricName.${computation.name}'), hasLength(1), ); } } expect( - scores.where((score) => score.metric == _totalUiFrameAverageScore), + scores.where((score) => score.metric == totalUiFrameAverage), hasLength(1), ); diff --git a/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart b/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart index da2075110af..3aa2f3cf727 100644 --- a/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart +++ b/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart @@ -42,13 +42,13 @@ import 'utils.dart'; /// using the '--baseline' or '--test' arguments. /// /// Example usage: -/// * dart run benchmark/scripts/run_benchmarks.dart +/// * dart run benchmark/scripts/dart2wasm_performance_diff.dart /// /// Example usage that averages benchmark results over 5 runs: -/// * dart run benchmark/scripts/run_benchmarks.dart --average-of=5 +/// * dart run benchmark/scripts/dart2wasm_performance_diff.dart --average-of=5 /// /// Example usage that diffs against an existing basline: -/// * dart run benchmark/scripts/run_benchmarks.dart --baseline=/Users/me/Downloads/baseline_run.json +/// * dart run benchmark/scripts/dart2wasm_performance_diff.dart --baseline=/Users/me/Downloads/baseline_run.json void main(List args) async { if (!Directory.current.path.contains('devtools_app')) { stderr.writeln( diff --git a/packages/devtools_app/pubspec.yaml b/packages/devtools_app/pubspec.yaml index d4155aca748..67c5dc59725 100644 --- a/packages/devtools_app/pubspec.yaml +++ b/packages/devtools_app/pubspec.yaml @@ -74,7 +74,7 @@ dev_dependencies: mockito: ^5.4.1 stager: ^1.0.1 test: ^1.21.0 - web_benchmarks: ^3.0.0 + web_benchmarks: ^3.1.0 webkit_inspection_protocol: ">=0.5.0 <2.0.0" flutter: From 0d1293fd669b34a82e1a00fa5b814a9c0bf59961 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Wed, 2 Oct 2024 16:02:47 -0700 Subject: [PATCH 11/14] formatting --- .../devtools_app/benchmark/devtools_benchmarks_test.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart index 30f55a4502e..897ef775c96 100644 --- a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -65,14 +65,16 @@ Future _runBenchmarks({bool useWasm = false}) async { expect( scores, hasLength( - expectedMetrics.length * expectedComputations.length + _extraScores.length, + expectedMetrics.length * expectedComputations.length + + _extraScores.length, ), ); for (final metricName in expectedMetrics) { for (final computation in expectedComputations) { expect( - scores.where((score) => score.metric == '$metricName.${computation.name}'), + scores.where( + (score) => score.metric == '$metricName.${computation.name}'), hasLength(1), ); } From 34addc33b501d92af49f9570a65f3f5e54760817 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Wed, 2 Oct 2024 16:03:23 -0700 Subject: [PATCH 12/14] formatting --- packages/devtools_app/benchmark/devtools_benchmarks_test.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart index 897ef775c96..1f901430235 100644 --- a/packages/devtools_app/benchmark/devtools_benchmarks_test.dart +++ b/packages/devtools_app/benchmark/devtools_benchmarks_test.dart @@ -74,7 +74,8 @@ Future _runBenchmarks({bool useWasm = false}) async { for (final computation in expectedComputations) { expect( scores.where( - (score) => score.metric == '$metricName.${computation.name}'), + (score) => score.metric == '$metricName.${computation.name}', + ), hasLength(1), ); } From 41b5a05ea5ae6b071831d3e50b90622801b5e4f5 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Wed, 2 Oct 2024 16:08:52 -0700 Subject: [PATCH 13/14] try using setup-chrome action --- .github/workflows/build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f3b0e9179a6..723dad915b3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -271,6 +271,9 @@ jobs: path: | ./tool/flutter-sdk key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} + - uses: browser-actions/setup-chrome@v1 + with: + install-chromedriver: true - name: tool/ci/benchmark_performance.sh run: ./tool/ci/benchmark_performance.sh From 1e71217aaf23be6988646248151aff0d6b285f2f Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Wed, 2 Oct 2024 16:11:30 -0700 Subject: [PATCH 14/14] undo yaml changes --- .github/workflows/build.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 723dad915b3..f3b0e9179a6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -271,9 +271,6 @@ jobs: path: | ./tool/flutter-sdk key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} - - uses: browser-actions/setup-chrome@v1 - with: - install-chromedriver: true - name: tool/ci/benchmark_performance.sh run: ./tool/ci/benchmark_performance.sh