WPT: Autogenerate a test case for each test file

build/wpt_test.bzl

@@ -9,50 +9,77 @@
 
 load("//:build/wd_test.bzl", "wd_test")
 
-def wpt_test(name, wpt_directory, test_js):
-    test_gen_rule = "{}@_wpt_test_gen".format(name)
-    _wpt_test_gen(
-        name = test_gen_rule,
+def wpt_test(name, wpt_directory, test_config):
+    js_gen_rule = "{}@_wpt_js_test_gen".format(name)
+    _wpt_js_test_gen(
+        name = js_gen_rule,
         test_name = name,
         wpt_directory = wpt_directory,
-        test_js = test_js,
+        test_config = test_config,
+    )
+
+    wd_gen_rule = "{}@_wpt_wd_test_gen".format(name)
+    _wpt_wd_test_gen(
+        name = wd_gen_rule,
+        test_name = name,
+        wpt_directory = wpt_directory,
+        test_config = test_config,
+        test_js = js_gen_rule,
     )
 
     wd_test(
         name = "{}".format(name),
-        src = test_gen_rule,
+        src = wd_gen_rule,
         args = ["--experimental"],
         data = [
             "//src/wpt:wpt-test-harness",
-            test_js,
+            test_config,
+            js_gen_rule,
             wpt_directory,
             "//src/workerd/io:trimmed-supported-compatibility-date.txt",
         ],
     )
 
-def _wpt_test_gen_impl(ctx):
-    src = ctx.actions.declare_file("{}.wd-test".format(ctx.attr.test_name))
+def _wpt_wd_test_gen_impl(ctx):
+    wd_src = ctx.actions.declare_file("{}.wd-test".format(ctx.attr.test_name))
+
     ctx.actions.write(
-        output = src,
-        content = WPT_TEST_TEMPLATE.format(
+        output = wd_src,
+        content = WD_TEST_TEMPLATE.format(
             test_name = ctx.attr.test_name,
             test_js = wd_relative_path(ctx.file.test_js),
+            test_config = wd_relative_path(ctx.file.test_config),
             modules = generate_external_modules(ctx.attr.wpt_directory.files),
         ),
     )
 
     return DefaultInfo(
-        files = depset([src]),
+        files = depset([wd_src]),
+    )
+
+def _wpt_js_test_gen_impl(ctx):
+    test_src = ctx.actions.declare_file("{}-test.js".format(ctx.attr.test_name))
+
+    ctx.actions.write(
+        output = test_src,
+        content = JS_TEST_TEMPLATE.format(
+            cases = generate_external_cases(ctx.attr.wpt_directory.files),
+        ),
+    )
+
+    return DefaultInfo(
+        files = depset([test_src]),
     )
 
-WPT_TEST_TEMPLATE = """
+WD_TEST_TEMPLATE = """
 using Workerd = import "/workerd/workerd.capnp";
 const unitTests :Workerd.Config = (
   services = [
     ( name = "{test_name}",
       worker = (
         modules = [
           (name = "worker", esModule = embed "{test_js}"),
+          (name = "config", esModule = embed "{test_config}"),
           (name = "harness", esModule = embed "../../../../../workerd/src/wpt/harness.js"),
           {modules}
         ],
@@ -70,6 +97,15 @@ const unitTests :Workerd.Config = (
   ],
 );"""
 
+JS_TEST_TEMPLATE = """
+import {{ runner }} from 'harness';
+import {{ config }} from 'config';
+
+const run = runner(config);
+
+{cases}
+"""
+
 def wd_relative_path(file):
     """
     Returns a relative path which can be referenced in the .wd-test file.
@@ -102,14 +138,43 @@ def generate_external_modules(files):
 
     return ",\n".join(result)
 
-_wpt_test_gen = rule(
-    implementation = _wpt_test_gen_impl,
+def generate_external_cases(files):
+    result = []
+    for file in files.to_list():
+        file_path = wd_relative_path(file)
+        if file.extension == "js":
+            entry = """export const {} = run("{}");""".format(file_to_identifier(file.basename), file.basename)
+            result.append(entry)
+
+    return "\n".join(result)
+
+def file_to_identifier(file):
+    stem = file.replace(".js", "").replace(".any", "")
+    stem_title = "".join([ch for ch in stem.title().elems() if ch.isalnum()])
+    return stem_title[0].lower() + stem_title[1:]
+
+_wpt_wd_test_gen = rule(
+    implementation = _wpt_wd_test_gen_impl,
     attrs = {
         # A string to use as the test name. Used in the wd-test filename and the worker's name
         "test_name": attr.string(),
         # A file group representing a directory of wpt tests. All files in the group will be embedded.
         "wpt_directory": attr.label(),
-        # A JS file containing the actual test logic.
+        # A JS file containing the config for the test cases
+        "test_config": attr.label(allow_single_file = True),
+        # The generated JS file invoking each test case
         "test_js": attr.label(allow_single_file = True),
     },
 )
+
+_wpt_js_test_gen = rule(
+    implementation = _wpt_js_test_gen_impl,
+    attrs = {
+        # A string to use as the test name. Used in the wd-test filename and the worker's name
+        "test_name": attr.string(),
+        # A file group representing a directory of wpt tests. All files in the group will be embedded.
+        "wpt_directory": attr.label(),
+        # A JS file containing the config for the test cases
+        "test_config": attr.label(allow_single_file = True),
+    },
+)

src/workerd/api/wpt/BUILD.bazel

@@ -4,8 +4,10 @@
 
 load("//:build/wpt_test.bzl", "wpt_test")
 
+TEST_CONFIG_SUFFIX = "-test-config.js"
+
 [wpt_test(
-    name = file.replace("-test.js", ""),
-    test_js = file,
-    wpt_directory = "@wpt//:{}".format(file.replace("-test.js", "")),
-) for file in glob(["*-test.js"])]
+    name = file.replace(TEST_CONFIG_SUFFIX, ""),
+    test_config = file,
+    wpt_directory = "@wpt//:{}".format(file.replace(TEST_CONFIG_SUFFIX, "")),
+) for file in glob(["*" + TEST_CONFIG_SUFFIX])]

src/workerd/api/wpt/README.md

@@ -0,0 +1,45 @@
+## How to add WPT test suites to workerd
+
+[WPT groups its test suites into directories](https://github.com/web-platform-tests/wpt). To add a new test suite to workerd, simply create a file named `suite-test-config.js`. For example, `url/` tests are added in the `url-test-config.js` file.
+
+A test config file is expected to export an object named `config`. This object can be empty to run all tests and subtests.
+
+```js
+export const config = {};
+```
+
+A test export is generated for each JS file in the directory. WPT test files are sometimes located in subdirectories, but the test importer flattens the hierarchy within the test suite.
+
+Each test file can contain several subtests, created by invoking `test` or `promise_test` within the test file. They are named according to the message argument provided.
+
+## Options
+
+Each entry in the `config` object specifies the test file name, and the options to apply to those tests. For example, `urlsearchparams-sort.any.js` can be configured using:
+
+```js
+export const config = {
+    'urlsearchparams-sort.any.js': {
+        expectedFailures: ['Parse and sort: ffi&🌈', 'URL parse and sort: ffi&🌈'],
+    },
+};
+```
+
+
+The following options are currently supported:
+
+* `ignore: bool`: Don't import a test file that is irrelevant for workerd tests
+* `skippedTests: string[]`: A list of subtests that should not be executed. This should only be used for subtests that would crash workerd.
+* `expectedFailures: string[]`: A list of subtests that are expected to fail, either due to a bug in workerd or an intentional choice not to support a feature.
+
+## Implementation
+
+Once a test config file is detected, the `wpt_test` macro is invoked for the suite. A JS test file is created that invokes the test harness on each JS file within the suite.
+
+A WD test file is generated which links all of the necessary components:
+
+* Generated JS test file (e.g. `url-test.js`)
+* The test config file (e.g. `url-test-config.js`)
+* Test harness (`harness.js`)
+* Test files provided by WPT  (e.g. `url/urlsearchparams-sort.any.js`)
+* JSON resources provided by WPT (e.g. `url/resources/urltestdata.json`)
+

src/workerd/api/wpt/url-test-config.js

@@ -0,0 +1,110 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+export const config = {
+  'a-element.js': { ignore: true },
+  'a-element-origin.js': { ignore: true },
+  'idlharness.any.js': { ignore: true },
+  'idlharness-shadowrealm.window.js': { ignore: true },
+  'javascript-urls.window.js': { ignore: true },
+  'toascii.window.js': { ignore: true },
+  'url-setters-a-area.window.js': { ignore: true },
+  'percent-encoding.window.js': { ignore: true },
+  'historical.any.js': {
+    expectedFailures: ["Setting URL's href attribute and base URLs"],
+  },
+  'url-constructor.any.js': {
+    expectedFailures: [
+      'Parsing: <http://example.com/\uD800\uD801\uDFFE\uDFFF\uFDD0\uFDCF\uFDEF\uFDF0\uFFFE\uFFFF?\uD800\uD801\uDFFE\uDFFF\uFDD0\uFDCF\uFDEF\uFDF0\uFFFE\uFFFF> without base',
+    ],
+  },
+  'urlencoded-parser.any.js': {
+    expectedFailures: [
+      'request.formData() with input: test',
+      'response.formData() with input: test',
+      'request.formData() with input: \uFEFFtest=\uFEFF',
+      'response.formData() with input: \uFEFFtest=\uFEFF',
+      'request.formData() with input: %EF%BB%BFtest=%EF%BB%BF',
+      'response.formData() with input: %EF%BB%BFtest=%EF%BB%BF',
+      'request.formData() with input: %EF%BF%BF=%EF%BF%BF',
+      'response.formData() with input: %EF%BF%BF=%EF%BF%BF',
+      'request.formData() with input: %FE%FF',
+      'response.formData() with input: %FE%FF',
+      'request.formData() with input: %FF%FE',
+      'response.formData() with input: %FF%FE',
+      'request.formData() with input: †&†=x',
+      'response.formData() with input: †&†=x',
+      'request.formData() with input: %C2',
+      'response.formData() with input: %C2',
+      'request.formData() with input: %C2x',
+      'response.formData() with input: %C2x',
+      'request.formData() with input: _charset_=windows-1252&test=%C2x',
+      'response.formData() with input: _charset_=windows-1252&test=%C2x',
+      'request.formData() with input: ',
+      'response.formData() with input: ',
+      'request.formData() with input: a',
+      'response.formData() with input: a',
+      'request.formData() with input: a=b',
+      'response.formData() with input: a=b',
+      'request.formData() with input: a=',
+      'response.formData() with input: a=',
+      'request.formData() with input: =b',
+      'response.formData() with input: =b',
+      'request.formData() with input: &',
+      'response.formData() with input: &',
+      'request.formData() with input: &a',
+      'response.formData() with input: &a',
+      'request.formData() with input: a&',
+      'response.formData() with input: a&',
+      'request.formData() with input: a&a',
+      'response.formData() with input: a&a',
+      'request.formData() with input: a&b&c',
+      'response.formData() with input: a&b&c',
+      'request.formData() with input: a=b&c=d',
+      'response.formData() with input: a=b&c=d',
+      'request.formData() with input: a=b&c=d&',
+      'response.formData() with input: a=b&c=d&',
+      'request.formData() with input: &&&a=b&&&&c=d&',
+      'response.formData() with input: &&&a=b&&&&c=d&',
+      'request.formData() with input: a=a&a=b&a=c',
+      'response.formData() with input: a=a&a=b&a=c',
+      'request.formData() with input: a==a',
+      'response.formData() with input: a==a',
+      'request.formData() with input: a=a+b+c+d',
+      'response.formData() with input: a=a+b+c+d',
+      'request.formData() with input: %=a',
+      'response.formData() with input: %=a',
+      'request.formData() with input: %a=a',
+      'response.formData() with input: %a=a',
+      'request.formData() with input: %a_=a',
+      'response.formData() with input: %a_=a',
+      'request.formData() with input: %61=a',
+      'response.formData() with input: %61=a',
+      'request.formData() with input: %61+%4d%4D=',
+      'response.formData() with input: %61+%4d%4D=',
+      'request.formData() with input: id=0&value=%',
+      'response.formData() with input: id=0&value=%',
+      'request.formData() with input: b=%2sf%2a',
+      'response.formData() with input: b=%2sf%2a',
+      'request.formData() with input: b=%2%2af%2a',
+      'response.formData() with input: b=%2%2af%2a',
+      'request.formData() with input: b=%%2a',
+      'response.formData() with input: b=%%2a',
+    ],
+  },
+  'urlsearchparams-constructor.any.js': {
+    expectedFailures: [
+      'URLSearchParams constructor, DOMException as argument',
+      'Construct with 2 unpaired surrogates (no trailing)',
+      'Construct with 3 unpaired surrogates (no leading)',
+      'Construct with object with NULL, non-ASCII, and surrogate keys',
+    ],
+  },
+  'urlsearchparams-foreach.any.js': {
+    skippedTests: ['For-of Check'],
+  },
+  'urlsearchparams-sort.any.js': {
+    expectedFailures: ['Parse and sort: ffi&🌈', 'URL parse and sort: ffi&🌈'],
+  },
+};