From 01c8e7b3aaf773591f20ce48aa310953b9c949ae Mon Sep 17 00:00:00 2001 From: Trey Turner Date: Mon, 23 Sep 2024 10:15:29 -0500 Subject: [PATCH] feat: support default test timeout in bunfig.toml --- docs/runtime/bunfig.md | 9 ++ src/bunfig.zig | 5 ++ src/cli.zig | 14 +-- test/cli/test/bun-test-bunfig.test.ts | 118 ++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 7 deletions(-) create mode 100644 test/cli/test/bun-test-bunfig.test.ts diff --git a/docs/runtime/bunfig.md b/docs/runtime/bunfig.md index 4af518744556a2..7e22c6d492fb8a 100644 --- a/docs/runtime/bunfig.md +++ b/docs/runtime/bunfig.md @@ -144,6 +144,15 @@ Same as the top-level `smol` field, but only applies to `bun test`. smol = true ``` +### `test.timeout` + +A default timeout for each test in milliseconds. Default `5000`. Use `--timeout` to override. + +```toml +[test] +timeout = 15000 +``` + ### `test.coverage` Enables coverage reporting. Default `false`. Use `--coverage` to override. diff --git a/src/bunfig.zig b/src/bunfig.zig index ede8389cde3c3d..ca5b7b4a5b115b 100644 --- a/src/bunfig.zig +++ b/src/bunfig.zig @@ -253,6 +253,11 @@ pub const Bunfig = struct { this.ctx.runtime_options.smol = expr.data.e_boolean.value; } + if (test_.get("timeout")) |expr| { + try this.expect(expr, .e_number); + this.ctx.test_options.default_timeout_ms = expr.data.e_number.toU32(); + } + if (test_.get("coverage")) |expr| { try this.expect(expr, .e_boolean); this.ctx.test_options.coverage.enabled = expr.data.e_boolean.value; diff --git a/src/cli.zig b/src/cli.zig index ad59352479eb1c..b7f32e8e585a59 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -453,6 +453,13 @@ pub const Arguments = struct { ctx.filters = args.options("--filter"); } + ctx.args.absolute_working_dir = cwd; + ctx.positionals = args.positionals(); + + if (comptime Command.Tag.loads_config.get(cmd)) { + try loadConfigWithCmdArgs(cmd, allocator, args, ctx); + } + if (cmd == .TestCommand) { if (args.option("--timeout")) |timeout_ms| { if (timeout_ms.len > 0) { @@ -527,13 +534,6 @@ pub const Arguments = struct { ctx.test_options.only = args.flag("--only"); } - ctx.args.absolute_working_dir = cwd; - ctx.positionals = args.positionals(); - - if (comptime Command.Tag.loads_config.get(cmd)) { - try loadConfigWithCmdArgs(cmd, allocator, args, ctx); - } - var opts: Api.TransformOptions = ctx.args; const defines_tuple = try DefineColonList.resolve(allocator, args.options("--define")); diff --git a/test/cli/test/bun-test-bunfig.test.ts b/test/cli/test/bun-test-bunfig.test.ts new file mode 100644 index 00000000000000..2d3d159d2a7855 --- /dev/null +++ b/test/cli/test/bun-test-bunfig.test.ts @@ -0,0 +1,118 @@ +import { spawnSync } from "bun"; +import { afterEach, describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, tempDirWithFiles } from "harness"; +import { rmSync } from "node:fs"; + +describe("bunfig test options", () => { + describe("timeout", () => { + let cwd: string; + + const getBunfigWithTimeout = (ms: number) => `[test]\ntimeout = ${ms}\n`; + + const getTestWithDuration = (ms: number) => { + return ` + import { test, expect } from "bun:test"; + test("takes ${ms} milliseconds", async () => { + const p = new Promise(r => setTimeout(r, ${ms}, true)); + expect(await p).toBeTrue(); + }); + `; + }; + + const errorPtn = /timed out after (\d+)ms/; + const durationPtn = /\(fail\) .* \[(\d+)(?:\.\d+)?ms\]/; + + afterEach(() => { + if (cwd) rmSync(cwd, { recursive: true }); + }); + + test("bunfig timeout overrides default", async () => { + const bunfigTimeout = 500; + cwd = tempDirWithFiles("test.bunfig.timeout", { + "bunfig.toml": getBunfigWithTimeout(bunfigTimeout), + "bun-test-bunfig-timeout.test.ts": getTestWithDuration(2000), + }); + + const result = spawnSync({ + cmd: [bunExe(), "-c=bunfig.toml", "test"], + env: bunEnv, + stderr: "pipe", + cwd, + }); + const stderr = result.stderr.toString().trim(); + + const errorMatch = stderr.match(errorPtn); + expect(errorMatch, "test didn't report timeout error to stderr").not.toBeNull(); + const errorTimeout = parseInt(errorMatch!.at(1)!); + expect(errorTimeout, "test timeout error doesn't reflect bunfig value").toEqual(bunfigTimeout); + + const durationMatch = stderr.match(durationPtn); + expect(durationMatch, "test didn't output failing result with actual duration to stderr").not.toBeNull(); + const duration = parseInt(durationMatch!.at(1)!); + expect(duration, "test timed out before bunfig timeout value").toBeGreaterThanOrEqual(bunfigTimeout); + expect(duration, "test didn't honor bunfig timeout value").toBeLessThanOrEqual(5000); + }); + + test("cli timeout overrides bunfig", async () => { + const cliTimeout = 500; + const bunfigTimeout = 1000; + cwd = tempDirWithFiles("test.cli.timeout.wins", { + "bunfig.toml": getBunfigWithTimeout(bunfigTimeout), + "bun-test-cli-timeout-wins.test.ts": getTestWithDuration(2000), + }); + + const result = spawnSync({ + cmd: [bunExe(), "-c=bunfig.toml", "test", "--timeout", `${cliTimeout}`], + env: bunEnv, + stderr: "pipe", + cwd, + }); + const stderr = result.stderr.toString().trim(); + + const errorMatch = stderr.match(errorPtn); + expect(errorMatch, "test didn't report timeout error to stderr").not.toBeNull(); + const errorTimeout = parseInt(errorMatch!.at(1)!); + expect(errorTimeout, "test timeout error doesn't reflect cli value").toEqual(cliTimeout); + + const durationMatch = stderr.match(durationPtn); + expect(durationMatch, "test didn't output failing result with actual duration to stderr").not.toBeNull(); + const duration = parseInt(durationMatch!.at(1)!); + expect(duration, "test timed out before cli value").toBeGreaterThanOrEqual(cliTimeout); + expect(duration, "test honored bunfig timeout instead of cli").toBeLessThan(bunfigTimeout); + }); + + test( + "default timeout specified via cli overrides bunfig", + async () => { + // This addresses a corner case identified by a weak initial implementation. + // By necessity it requires more than the default test timeout to prove (5s) + // so it should probably be removed after getting PR approval. + const cliTimeout = 5000; + const bunfigTimeout = 500; + cwd = tempDirWithFiles("test.cli.timeout.wins.default", { + "bunfig.toml": getBunfigWithTimeout(bunfigTimeout), + "bun-test-cli-timeout-wins-default.test.ts": getTestWithDuration(7000), + }); + + const result = spawnSync({ + cmd: [bunExe(), "-c=bunfig.toml", "test", "--timeout", `${cliTimeout}`], + env: bunEnv, + stderr: "pipe", + cwd, + }); + const stderr = result.stderr.toString().trim(); + + const errorMatch = stderr.match(errorPtn); + expect(errorMatch, "test didn't report timeout error to stderr").not.toBeNull(); + const errorTimeout = parseInt(errorMatch!.at(1)!); + expect(errorTimeout, "test timeout error doesn't reflect cli value").toEqual(cliTimeout); + + const durationMatch = stderr.match(durationPtn); + expect(durationMatch, "test didn't output failing result with actual duration to stderr").not.toBeNull(); + const duration = parseInt(durationMatch!.at(1)!); + expect(duration, "test timed out before cli value").toBeGreaterThanOrEqual(cliTimeout); + }, + { timeout: 7000 }, + ); + }); +});