Unskip the wasm benchmark

flutter-candidate.txt

@@ -1 +1 @@
-8925e1ffdfe880b06a8bc5877f017083d6652f5b
+e3828ec9402204bb81fcdb9c5fb7154367b4a6dd

packages/devtools_app/benchmark/devtools_benchmarks_test.dart

@@ -8,13 +8,14 @@
 import 'dart:convert' show JsonEncoder;
 import 'dart:io';
 
+import 'package:collection/collection.dart';
 import 'package:test/test.dart';
 import 'package:web_benchmarks/server.dart';
 
 import 'test_infra/common.dart';
 import 'test_infra/project_root_directory.dart';
 
-List<String> metricList({required bool useWasm}) => [
+List<String> _metricList({required bool useWasm}) => [
       // The skwasm renderer doesn't have preroll or apply frame steps in its
       // rendering.
       if (!useWasm) ...[
@@ -24,13 +25,22 @@ List<String> metricList({required bool useWasm}) => [
       'drawFrameDuration',
     ];
 
-final valueList = <String>[
+final _valueList = <String>[
   'average',
   'outlierAverage',
   'outlierRatio',
   'noise',
 ];
 
+const _totalUiFrameAverageScore = 'totalUiFrame.average';
+
+const _isWasmScore = 'isWasm';
+
+const _extraScores = [
+  _totalUiFrameAverageScore,
+  _isWasmScore,
+];
+
 /// Tests that the DevTools web benchmarks are run and reported correctly.
 void main() {
   for (final useWasm in [true, false]) {
@@ -40,10 +50,6 @@ void main() {
         await _runBenchmarks(useWasm: useWasm);
       },
       timeout: const Timeout(Duration(minutes: 10)),
-      // TODO(https://github.com/dart-lang/sdk/issues/56664): unskip the wasm
-      // benchmarks once this issue is resolved and we can bump DevTools to a
-      // version of Flutter that includes the fix.
-      skip: useWasm,
     );
   }
 
@@ -54,12 +60,13 @@ Future<void> _runBenchmarks({bool useWasm = false}) async {
   stdout.writeln('Starting web benchmark tests ...');
   final taskResult = await serveWebBenchmark(
     benchmarkAppDirectory: projectRootDirectory(),
-    entryPoint: 'benchmark/test_infra/client.dart',
+    entryPoint:
+        'benchmark/test_infra/client/client_${useWasm ? 'wasm' : 'js'}.dart',
     compilationOptions: useWasm
         ? const CompilationOptions.wasm()
         : const CompilationOptions.js(),
     treeShakeIcons: false,
-    initialPage: benchmarkInitialPage,
+    benchmarkPath: benchmarkPath(useWasm: useWasm),
   );
   stdout.writeln('Web benchmark tests finished.');
 
@@ -69,29 +76,34 @@ Future<void> _runBenchmarks({bool useWasm = false}) async {
   );
 
   for (final benchmarkName in DevToolsBenchmark.values.map((e) => e.id)) {
-    final expectedMetrics = metricList(useWasm: useWasm);
+    final expectedMetrics = _metricList(useWasm: useWasm);
+    final scores = taskResult.scores[benchmarkName] ?? [];
     expect(
-      taskResult.scores[benchmarkName],
-      hasLength(expectedMetrics.length * valueList.length + 1),
+      scores,
+      hasLength(
+        expectedMetrics.length * _valueList.length + _extraScores.length,
+      ),
     );
 
     for (final metricName in expectedMetrics) {
-      for (final valueName in valueList) {
+      for (final valueName in _valueList) {
         expect(
-          taskResult.scores[benchmarkName]?.where(
-            (score) => score.metric == '$metricName.$valueName',
-          ),
+          scores.where((score) => score.metric == '$metricName.$valueName'),
           hasLength(1),
         );
       }
     }
 
     expect(
-      taskResult.scores[benchmarkName]?.where(
-        (score) => score.metric == 'totalUiFrame.average',
-      ),
+      scores.where((score) => score.metric == _totalUiFrameAverageScore),
       hasLength(1),
     );
+
+    final isWasmScore = scores.firstWhereOrNull(
+      (BenchmarkScore score) => score.metric == _isWasmScore,
+    );
+    expect(isWasmScore, isNotNull);
+    expect(isWasmScore!.value, useWasm ? 1 : 0);
   }
 
   expect(

packages/devtools_app/benchmark/scripts/run_benchmarks.dart

@@ -60,7 +60,7 @@ Future<BenchmarkResults> runBenchmarks({
             ? const CompilationOptions.wasm()
             : const CompilationOptions.js(),
         treeShakeIcons: false,
-        initialPage: benchmarkInitialPage,
+        benchmarkPath: benchmarkPath(useWasm: useWasm),
         headless: !useBrowser,
       ),
     );

packages/devtools_app/benchmark/test_infra/automators/devtools_automator.dart

@@ -9,8 +9,10 @@ import 'dart:async';
 import 'package:devtools_app/devtools_app.dart';
 import 'package:devtools_test/helpers.dart';
 import 'package:devtools_test/test_data.dart';
+import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
+import 'package:web_benchmarks/client.dart';
 
 import '../common.dart';
 import '_cpu_profiler_automator.dart';
@@ -21,6 +23,7 @@ class DevToolsAutomater {
   DevToolsAutomater({
     required this.benchmark,
     required this.stopWarmingUpCallback,
+    required this.profile,
   });
 
   /// The current benchmark.
@@ -32,6 +35,9 @@ class DevToolsAutomater {
   /// as over.
   final void Function() stopWarmingUpCallback;
 
+  /// The profile collected for the running benchmark
+  final Profile profile;
+
   /// Whether the automation has ended.
   bool finished = false;
 
@@ -65,6 +71,12 @@ class DevToolsAutomater {
         await PerformanceScreenAutomator(controller).run();
     }
 
+    // Record whether we are in wasm mode or not. Ideally, we'd have a more
+    // first-class way to add metadata like this, but this will work for us to
+    // pass information about the environment back to the server for the
+    // purposes of our own tests.
+    profile.extraData['isWasm'] = kIsWasm ? 1 : 0;
+
     // At the end of the test, mark as finished.
     finished = true;
   }

packages/devtools_app/benchmark/test_infra/client/client_js.dart

@@ -0,0 +1,17 @@
+// Copyright 2024 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:web_benchmarks/client.dart';
+
+import '../common.dart';
+import 'client_shared.dart';
+
+/// Runs the client of the DevTools web benchmarks.
+///
+/// When the DevTools web benchmarks are run, the server builds an app with this
+/// file as the entry point (see `run_benchmarks.dart`). The app automates
+/// the DevTools web app, records some performance data, and reports them.
+Future<void> main() async {
+  await runBenchmarks(benchmarks, benchmarkPath: benchmarkPath(useWasm: false));
+}

packages/devtools_app/benchmark/test_infra/client.dart → packages/devtools_app/benchmark/test_infra/client/client_shared.dart

@@ -1,11 +1,11 @@
-// Copyright 2023 The Chromium Authors. All rights reserved.
+// Copyright 2024 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 import 'package:web_benchmarks/client.dart';
 
-import 'common.dart';
-import 'devtools_recorder.dart';
+import '../common.dart';
+import '../devtools_recorder.dart';
 
 typedef RecorderFactory = Recorder Function();
 
@@ -20,12 +20,3 @@ final benchmarks = <String, RecorderFactory>{
         benchmark: DevToolsBenchmark.offlinePerformanceScreen,
       ),
 };
-
-/// Runs the client of the DevTools web benchmarks.
-///
-/// When the DevTools web benchmarks are run, the server builds an app with this
-/// file as the entry point (see `run_benchmarks.dart`). The app automates
-/// the DevTools web app, records some performance data, and reports them.
-Future<void> main() async {
-  await runBenchmarks(benchmarks, initialPage: benchmarkInitialPage);
-}

packages/devtools_app/benchmark/test_infra/client/client_wasm.dart

@@ -0,0 +1,17 @@
+// Copyright 2024 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:web_benchmarks/client.dart';
+
+import '../common.dart';
+import 'client_shared.dart';
+
+/// Runs the client of the DevTools web benchmarks.
+///
+/// When the DevTools web benchmarks are run, the server builds an app with this
+/// file as the entry point (see `run_benchmarks.dart`). The app automates
+/// the DevTools web app, records some performance data, and reports them.
+Future<void> main() async {
+  await runBenchmarks(benchmarks, benchmarkPath: benchmarkPath(useWasm: true));
+}

packages/devtools_app/benchmark/test_infra/common.dart

@@ -4,11 +4,18 @@
 
 /// The initial page to load upon opening the DevTools benchmark app or
 /// reloading it in Chrome.
-//
-// We use an empty initial page so that the benchmark server does not attempt
-// to load the default page 'index.html', which will show up as "page not
-// found" in DevTools.
-const benchmarkInitialPage = '';
+///
+/// We use an empty initial page so that the benchmark server does not attempt
+/// to load the default page 'index.html', which will show up as "page not
+/// found" in DevTools.
+const _benchmarkInitialPage = '';
+
+const _wasmQueryParameters = {'wasm': 'true'};
+
+String benchmarkPath({required bool useWasm}) => Uri(
+      path: _benchmarkInitialPage,
+      queryParameters: useWasm ? _wasmQueryParameters : null,
+    ).toString();
 
 const devtoolsBenchmarkPrefix = 'devtools';
 

packages/devtools_app/benchmark/test_infra/devtools_recorder.dart

@@ -34,6 +34,7 @@ class DevToolsRecorder extends WidgetRecorder {
     _devToolsAutomator = DevToolsAutomater(
       benchmark: benchmark,
       stopWarmingUpCallback: profile.stopWarmingUp,
+      profile: profile,
     );
     return _devToolsAutomator!.createWidget();
   }

packages/devtools_app/pubspec.yaml

@@ -76,7 +76,7 @@ dev_dependencies:
   mockito: ^5.4.1
   stager: ^1.0.1
   test: ^1.21.0
-  web_benchmarks: ^2.0.0
+  web_benchmarks: ^3.0.0
   webkit_inspection_protocol: ">=0.5.0 <2.0.0"
 
 flutter: