From 4f57b1ebcf7e361b411743675577214a7a742867 Mon Sep 17 00:00:00 2001 From: Kevin Schaaf Date: Fri, 9 Oct 2020 03:10:58 -0700 Subject: [PATCH 01/12] Add option to collate results tables by measurements --- config.schema.json | 4 ++++ package.json | 1 + src/config.ts | 5 +++++ src/configfile.ts | 7 +++++++ src/defaults.ts | 1 + src/flags.ts | 8 ++++++++ src/format.ts | 26 ++++++++++++++++++++++++++ src/runner.ts | 10 ++++++++-- 8 files changed, 60 insertions(+), 2 deletions(-) diff --git a/config.schema.json b/config.schema.json index dca9e375..f68f4eeb 100644 --- a/config.schema.json +++ b/config.schema.json @@ -428,6 +428,10 @@ "minItems": 1, "type": "array" }, + "collate": { + "description": "Whether to collate result table by measurement (when multiple measurements\nare in use)", + "type": "boolean" + }, "horizons": { "description": "The degrees of difference to try and resolve when auto-sampling\n(e.g. 0ms, +1ms, -1ms, 0%, +1%, -1%, default 0%).", "items": { diff --git a/package.json b/package.json index 9d336107..212fd9bd 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "scripts": { "prepare": "if [ -f './tsconfig.json' ]; then npm run build; fi;", "build": "rimraf lib/ client/lib/ && mkdir lib && npm run generate-json-schema && tsc && tsc -p client/ && npm run lint", + "build:watch": "tsc --watch", "generate-json-schema": "typescript-json-schema tsconfig.json ConfigFile --include src/configfile.ts --required --noExtraProps > config.schema.json", "lint": "tslint --project . --format stylish", "format": "clang-format --style=file -i \"--glob=./{client,}/src/**/*.ts\"", diff --git a/src/config.ts b/src/config.ts index 57f226bc..8a7a9cf5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -41,6 +41,7 @@ export interface Config { forceCleanNpmInstall: boolean; csvFileStats: string; csvFileRaw: string; + collate?: boolean; } export async function makeConfig(opts: Opts): Promise { @@ -86,6 +87,9 @@ export async function makeConfig(opts: Opts): Promise { if (opts['window-size'] !== undefined) { throw new Error('--window-size cannot be specified when using --config'); } + if (opts['collate'] !== undefined) { + throw new Error('--collate cannot be specified when using --config'); + } const rawConfigObj = await fsExtra.readJson(opts.config); const validatedConfigObj = await parseConfigFile(rawConfigObj); @@ -164,6 +168,7 @@ export function applyDefaults(partial: Partial): Config { defaults.resolveBareModules, root: partial.root !== undefined ? partial.root : defaults.root, timeout: partial.timeout !== undefined ? partial.timeout : defaults.timeout, + collate: partial.collate !== undefined ? partial.collate : defaults.collate, }; } diff --git a/src/configfile.ts b/src/configfile.ts index aaea2a5b..950db31c 100644 --- a/src/configfile.ts +++ b/src/configfile.ts @@ -47,6 +47,12 @@ export interface ConfigFile { */ horizons?: string[]; + /** + * Whether to collate result table by measurement (when multiple measurements + * are in use) + */ + collate?: boolean; + /** * Benchmarks to run. * @TJS-minItems 1 @@ -300,6 +306,7 @@ export async function parseConfigFile(parsedJson: unknown): undefined, benchmarks, resolveBareModules: validated.resolveBareModules, + collate: validated.collate, }; } diff --git a/src/defaults.ts b/src/defaults.ts index 3b7c2580..3d439629 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -24,6 +24,7 @@ export const mode = 'automatic'; export const resolveBareModules = true; export const forceCleanNpmInstall = false; export const measurementExpression = 'window.tachometerResult'; +export const collate = false; export function measurement(url: LocalUrl|RemoteUrl): Measurement { if (url.kind === 'remote') { diff --git a/src/flags.ts b/src/flags.ts index 24fab9ea..e85d0db6 100644 --- a/src/flags.ts +++ b/src/flags.ts @@ -201,6 +201,13 @@ export const optDefs: commandLineUsage.OptionDefinition[] = [ ` (default "${defaults.windowWidth},${defaults.windowHeight}").`, type: String, }, + { + name: 'collate', + description: + `Whether to collate result table by measurement (when multiple ` + + `measurements are in use)`, + type: Boolean, + }, ]; export interface Opts { @@ -228,6 +235,7 @@ export interface Opts { 'csv-file': string; 'csv-file-raw': string; 'json-file': string; + collate: boolean; // Extra arguments not associated with a flag are put here. These are our // benchmark names/URLs. diff --git a/src/format.ts b/src/format.ts index e0d266f4..4fd5cc88 100644 --- a/src/format.ts +++ b/src/format.ts @@ -104,6 +104,32 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults { return {fixed: fixedTable, unfixed: unfixedTable}; } +const nameAndMeas = (resultName: string) => { + const match = /(.*) \[(.*)\]/.exec(resultName); + if (match) { + return {name: match[1], meas: match[2]}; + } else { + return {name: resultName, meas: ''}; + } +}; + +export function collatedResultTables(results: ResultStatsWithDifferences[]) { + const collated: {[index: string]: ResultStatsWithDifferences[]} = {}; + results.forEach((result) => { + const {meas} = nameAndMeas(result.result.name); + (collated[meas] || (collated[meas] = [])).push({ + ...result, + differences: result.differences.filter( + (_, i) => nameAndMeas(results[i].result.name).meas === meas) + }); + }); + const tables: AutomaticResults[] = []; + for (const results of Object.values(collated)) { + tables.push(automaticResultTable(results)); + } + return tables; +} + /** * Format a terminal text result table where each result is a row: * diff --git a/src/runner.ts b/src/runner.ts index a1ea6445..61ac3724 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -21,7 +21,7 @@ import {measure, measurementName} from './measure'; import {BenchmarkResult, BenchmarkSpec} from './types'; import {formatCsvStats, formatCsvRaw} from './csv'; import {ResultStatsWithDifferences, horizonsResolved, summaryStats, computeDifferences} from './stats'; -import {verticalTermResultTable, horizontalTermResultTable, verticalHtmlResultTable, horizontalHtmlResultTable, automaticResultTable, spinner, benchmarkOneLiner} from './format'; +import {verticalTermResultTable, horizontalTermResultTable, verticalHtmlResultTable, horizontalHtmlResultTable, automaticResultTable, spinner, benchmarkOneLiner, collatedResultTables} from './format'; import {Config} from './config'; import * as github from './github'; import {Server, Session} from './server'; @@ -343,7 +343,13 @@ export class Runner { console.log(); const {fixed, unfixed} = automaticResultTable(withDifferences); console.log(horizontalTermResultTable(fixed)); - console.log(verticalTermResultTable(unfixed)); + if (config.collate) { + for (const {unfixed} of collatedResultTables(withDifferences)) { + console.log(verticalTermResultTable(unfixed)); + } + } else { + console.log(verticalTermResultTable(unfixed)); + } if (hitTimeout === true) { console.log(ansi.format( From f9f2f91236c4e2afab28f4064fa2f7797e4489e1 Mon Sep 17 00:00:00 2001 From: Kevin Schaaf Date: Fri, 9 Oct 2020 03:12:15 -0700 Subject: [PATCH 02/12] Add CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 486073ac..83534382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ project adheres to [Semantic Versioning](http://semver.org/). +- Add `collate` option for collating results tables by measurement (when + multiple measurements are in use) + ## [0.5.5] 2020-09-21 - A warning is now displayed when there are multiple performance marks or From 49bdd44db98f43329253ce2b3b9aa6f962082d8f Mon Sep 17 00:00:00 2001 From: Kevin Schaaf Date: Fri, 9 Oct 2020 12:22:50 -0700 Subject: [PATCH 03/12] Add `collate` flag to existing tests --- CHANGELOG.md | 2 +- src/test/config_test.ts | 4 ++++ src/test/configfile_test.ts | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83534382..c664287a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - Add `collate` option for collating results tables by measurement (when - multiple measurements are in use) + multiple measurements are in use). ## [0.5.5] 2020-09-21 diff --git a/src/test/config_test.ts b/src/test/config_test.ts index e91e419a..d539cd10 100644 --- a/src/test/config_test.ts +++ b/src/test/config_test.ts @@ -50,6 +50,7 @@ suite('makeConfig', function() { csvFileStats: '', csvFileRaw: '', githubCheck: undefined, + collate: false, benchmarks: [ { browser: { @@ -93,6 +94,7 @@ suite('makeConfig', function() { csvFileRaw: '', // TODO(aomarks) Be consistent about undefined vs unset. githubCheck: undefined, + collate: false, benchmarks: [ { browser: { @@ -136,6 +138,7 @@ suite('makeConfig', function() { csvFileStats: '', csvFileRaw: '', githubCheck: undefined, + collate: false, benchmarks: [ { browser: { @@ -186,6 +189,7 @@ suite('makeConfig', function() { remoteAccessibleHost: '', // TODO(aomarks) Be consistent about undefined vs unset. githubCheck: undefined, + collate: false, benchmarks: [ { browser: { diff --git a/src/test/configfile_test.ts b/src/test/configfile_test.ts index 47d8380a..2281f8a9 100644 --- a/src/test/configfile_test.ts +++ b/src/test/configfile_test.ts @@ -50,6 +50,7 @@ suite('config', () => { timeout: 7, horizons: ['0ms', '1ms', '2%', '+3%'], resolveBareModules: false, + collate: true, benchmarks: [ { name: 'remote', @@ -103,6 +104,7 @@ suite('config', () => { relative: [-0.02, 0.02, 0.03], }, resolveBareModules: false, + collate: true, benchmarks: [ { name: 'remote', @@ -186,6 +188,7 @@ suite('config', () => { timeout: undefined, horizons: undefined, resolveBareModules: undefined, + collate: undefined, benchmarks: [ { name: 'http://example.com?foo=bar', @@ -252,6 +255,7 @@ suite('config', () => { timeout: undefined, horizons: undefined, resolveBareModules: undefined, + collate: undefined, benchmarks: [ { name: '/mybench/index.html?foo=a', @@ -331,6 +335,7 @@ suite('config', () => { timeout: undefined, horizons: undefined, resolveBareModules: undefined, + collate: undefined, benchmarks: [ { name: 'http://example.com', @@ -406,6 +411,7 @@ suite('config', () => { timeout: undefined, horizons: undefined, resolveBareModules: undefined, + collate: undefined, benchmarks: [ { name: 'http://example.com?foo=bar', From cd898388934e2a76585354ac0b5eb625d2675f2b Mon Sep 17 00:00:00 2001 From: Kevin Schaaf Date: Fri, 9 Oct 2020 12:54:41 -0700 Subject: [PATCH 04/12] Add collated table formatting tests --- src/test/format_test.ts | 133 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) diff --git a/src/test/format_test.ts b/src/test/format_test.ts index de7d255f..792a25bc 100644 --- a/src/test/format_test.ts +++ b/src/test/format_test.ts @@ -15,7 +15,7 @@ import * as path from 'path'; import stripAnsi = require('strip-ansi'); import {ConfigFile} from '../configfile'; -import {automaticResultTable, verticalTermResultTable} from '../format'; +import {automaticResultTable, verticalTermResultTable, collatedResultTables} from '../format'; import {fakeResults, testData} from './test_helpers'; /** @@ -29,6 +29,18 @@ async function fakeResultTable(configFile: ConfigFile): Promise { return stripAnsi(verticalTermResultTable(resultTable)); } +/** + * Given a config file object, generates fake measurement results, and returns + * the collated terminal formatted result table that would be printed (minus + * color etc. formatting). + */ +async function fakeCollatedResultTable(configFile: ConfigFile): Promise { + const results = await fakeResults(configFile); + return collatedResultTables(results).map((result) => { + return stripAnsi(verticalTermResultTable(result.unfixed)); + }).join('\n'); +} + suite('format', () => { let prevCwd: string; suiteSetup(() => { @@ -169,4 +181,123 @@ suite('format', () => { `; assert.equal(actual, expected.trim() + '\n'); }); + + test('multiple measurements, collate: false', async () => { + const config: ConfigFile = { + benchmarks: [ + { + name: 'foo', + url: 'http://foo.com', + measurement: [{ + name: 'render', + mode: 'performance', + entryName: 'render' + }, { + name: 'update', + mode: 'performance', + entryName: 'update' + }], + }, + { + name: 'bar', + url: 'http://bar.com', + measurement: [{ + name: 'render', + mode: 'performance', + entryName: 'render' + }, { + name: 'update', + mode: 'performance', + entryName: 'update' + }], + }, + ], + }; + + const actual = await fakeResultTable(config); + const expected = ` +┌──────────────┬──────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┐ +│ Benchmark │ Bytes │ Avg time │ vs foo [render] │ vs foo [update] │ vs bar [render] │ vs bar [update] │ +├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ +│ foo [render] │ 1.00 KiB │ 8.56ms - 11.44ms │ │ unsure │ faster │ faster │ +│ │ │ │ - │ -20% - +20% │ 42% - 58% │ 42% - 58% │ +│ │ │ │ │ -2.03ms - +2.03ms │ 7.97ms - 12.03ms │ 7.97ms - 12.03ms │ +├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ +│ foo [update] │ 1.00 KiB │ 8.56ms - 11.44ms │ unsure │ │ faster │ faster │ +│ │ │ │ -20% - +20% │ - │ 42% - 58% │ 42% - 58% │ +│ │ │ │ -2.03ms - +2.03ms │ │ 7.97ms - 12.03ms │ 7.97ms - 12.03ms │ +├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ +│ bar [render] │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ slower │ │ unsure │ +│ │ │ │ 68% - 132% │ 68% - 132% │ - │ -10% - +10% │ +│ │ │ │ 7.97ms - 12.03ms │ 7.97ms - 12.03ms │ │ -2.03ms - +2.03ms │ +├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ +│ bar [update] │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ slower │ unsure │ │ +│ │ │ │ 68% - 132% │ 68% - 132% │ -10% - +10% │ - │ +│ │ │ │ 7.97ms - 12.03ms │ 7.97ms - 12.03ms │ -2.03ms - +2.03ms │ │ +└──────────────┴──────────┴───────────────────┴───────────────────┴───────────────────┴───────────────────┴───────────────────┘ + `; + assert.equal(actual, expected.trim() + '\n'); + }); + + test('multiple measurements, collate: false', async () => { + const config: ConfigFile = { + benchmarks: [ + { + name: 'foo', + url: 'http://foo.com', + measurement: [{ + name: 'render', + mode: 'performance', + entryName: 'render' + }, { + name: 'update', + mode: 'performance', + entryName: 'update' + }], + }, + { + name: 'bar', + url: 'http://bar.com', + measurement: [{ + name: 'render', + mode: 'performance', + entryName: 'render' + }, { + name: 'update', + mode: 'performance', + entryName: 'update' + }], + }, + ], + }; + + const actual = await fakeCollatedResultTable(config); + const expected = ` +┌──────────────┬──────────┬───────────────────┬──────────────────┬──────────────────┐ +│ Benchmark │ Bytes │ Avg time │ vs foo [render] │ vs bar [render] │ +├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ foo [render] │ 1.00 KiB │ 8.56ms - 11.44ms │ │ faster │ +│ │ │ │ - │ 42% - 58% │ +│ │ │ │ │ 7.97ms - 12.03ms │ +├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ bar [render] │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ │ +│ │ │ │ 68% - 132% │ - │ +│ │ │ │ 7.97ms - 12.03ms │ │ +└──────────────┴──────────┴───────────────────┴──────────────────┴──────────────────┘ + +┌──────────────┬──────────┬───────────────────┬──────────────────┬──────────────────┐ +│ Benchmark │ Bytes │ Avg time │ vs foo [update] │ vs bar [update] │ +├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ foo [update] │ 1.00 KiB │ 8.56ms - 11.44ms │ │ faster │ +│ │ │ │ - │ 42% - 58% │ +│ │ │ │ │ 7.97ms - 12.03ms │ +├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ bar [update] │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ │ +│ │ │ │ 68% - 132% │ - │ +│ │ │ │ 7.97ms - 12.03ms │ │ +└──────────────┴──────────┴───────────────────┴──────────────────┴──────────────────┘ + `; + assert.equal(actual, expected.trim() + '\n'); + }); + }); From 8c909ea1714dff14c3e44b0aa2ad171c53446d1f Mon Sep 17 00:00:00 2001 From: Kevin Schaaf Date: Fri, 9 Oct 2020 14:32:31 -0700 Subject: [PATCH 05/12] Simplify --- src/format.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/format.ts b/src/format.ts index 4fd5cc88..8c7162b1 100644 --- a/src/format.ts +++ b/src/format.ts @@ -104,23 +104,19 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults { return {fixed: fixedTable, unfixed: unfixedTable}; } -const nameAndMeas = (resultName: string) => { - const match = /(.*) \[(.*)\]/.exec(resultName); - if (match) { - return {name: match[1], meas: match[2]}; - } else { - return {name: resultName, meas: ''}; - } +const measurementForName = (resultName: string) => { + const match = /.* \[(.*)\]/.exec(resultName); + return match ? match[1] : ''; }; export function collatedResultTables(results: ResultStatsWithDifferences[]) { const collated: {[index: string]: ResultStatsWithDifferences[]} = {}; results.forEach((result) => { - const {meas} = nameAndMeas(result.result.name); + const meas = measurementForName(result.result.name); (collated[meas] || (collated[meas] = [])).push({ ...result, differences: result.differences.filter( - (_, i) => nameAndMeas(results[i].result.name).meas === meas) + (_, i) => measurementForName(results[i].result.name) === meas) }); }); const tables: AutomaticResults[] = []; From a7e12e799097da7acc11563b3fa989343359170d Mon Sep 17 00:00:00 2001 From: Kevin Schaaf Date: Fri, 9 Oct 2020 14:36:52 -0700 Subject: [PATCH 06/12] Use measurementName helper --- src/format.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/format.ts b/src/format.ts index 8c7162b1..8eb0116c 100644 --- a/src/format.ts +++ b/src/format.ts @@ -17,6 +17,7 @@ import ansi = require('ansi-escape-sequences'); import {Difference, ConfidenceInterval, ResultStats, ResultStatsWithDifferences} from './stats'; import {BenchmarkSpec, BenchmarkResult} from './types'; +import { measurementName } from './measure'; export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'].map( (frame) => ansi.format(`[blue]{${frame}}`)); @@ -104,19 +105,14 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults { return {fixed: fixedTable, unfixed: unfixedTable}; } -const measurementForName = (resultName: string) => { - const match = /.* \[(.*)\]/.exec(resultName); - return match ? match[1] : ''; -}; - export function collatedResultTables(results: ResultStatsWithDifferences[]) { const collated: {[index: string]: ResultStatsWithDifferences[]} = {}; results.forEach((result) => { - const meas = measurementForName(result.result.name); + const meas = measurementName(result.result.measurement); (collated[meas] || (collated[meas] = [])).push({ ...result, differences: result.differences.filter( - (_, i) => measurementForName(results[i].result.name) === meas) + (_, i) => measurementName(results[i].result.measurement) === meas) }); }); const tables: AutomaticResults[] = []; From c71f0511e4b8c6dec93040cdb4af12279674367c Mon Sep 17 00:00:00 2001 From: Kevin Schaaf Date: Fri, 9 Oct 2020 14:37:49 -0700 Subject: [PATCH 07/12] Format --- src/format.ts | 2 +- src/test/format_test.ts | 64 +++++++++++++++-------------------------- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/src/format.ts b/src/format.ts index 8eb0116c..e7abb21b 100644 --- a/src/format.ts +++ b/src/format.ts @@ -17,7 +17,7 @@ import ansi = require('ansi-escape-sequences'); import {Difference, ConfidenceInterval, ResultStats, ResultStatsWithDifferences} from './stats'; import {BenchmarkSpec, BenchmarkResult} from './types'; -import { measurementName } from './measure'; +import {measurementName} from './measure'; export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'].map( (frame) => ansi.format(`[blue]{${frame}}`)); diff --git a/src/test/format_test.ts b/src/test/format_test.ts index 792a25bc..3e25b449 100644 --- a/src/test/format_test.ts +++ b/src/test/format_test.ts @@ -34,11 +34,14 @@ async function fakeResultTable(configFile: ConfigFile): Promise { * the collated terminal formatted result table that would be printed (minus * color etc. formatting). */ -async function fakeCollatedResultTable(configFile: ConfigFile): Promise { +async function fakeCollatedResultTable(configFile: ConfigFile): + Promise { const results = await fakeResults(configFile); - return collatedResultTables(results).map((result) => { - return stripAnsi(verticalTermResultTable(result.unfixed)); - }).join('\n'); + return collatedResultTables(results) + .map((result) => { + return stripAnsi(verticalTermResultTable(result.unfixed)); + }) + .join('\n'); } suite('format', () => { @@ -188,28 +191,18 @@ suite('format', () => { { name: 'foo', url: 'http://foo.com', - measurement: [{ - name: 'render', - mode: 'performance', - entryName: 'render' - }, { - name: 'update', - mode: 'performance', - entryName: 'update' - }], + measurement: [ + {name: 'render', mode: 'performance', entryName: 'render'}, + {name: 'update', mode: 'performance', entryName: 'update'} + ], }, { name: 'bar', url: 'http://bar.com', - measurement: [{ - name: 'render', - mode: 'performance', - entryName: 'render' - }, { - name: 'update', - mode: 'performance', - entryName: 'update' - }], + measurement: [ + {name: 'render', mode: 'performance', entryName: 'render'}, + {name: 'update', mode: 'performance', entryName: 'update'} + ], }, ], }; @@ -245,28 +238,18 @@ suite('format', () => { { name: 'foo', url: 'http://foo.com', - measurement: [{ - name: 'render', - mode: 'performance', - entryName: 'render' - }, { - name: 'update', - mode: 'performance', - entryName: 'update' - }], + measurement: [ + {name: 'render', mode: 'performance', entryName: 'render'}, + {name: 'update', mode: 'performance', entryName: 'update'} + ], }, { name: 'bar', url: 'http://bar.com', - measurement: [{ - name: 'render', - mode: 'performance', - entryName: 'render' - }, { - name: 'update', - mode: 'performance', - entryName: 'update' - }], + measurement: [ + {name: 'render', mode: 'performance', entryName: 'render'}, + {name: 'update', mode: 'performance', entryName: 'update'} + ], }, ], }; @@ -299,5 +282,4 @@ suite('format', () => { `; assert.equal(actual, expected.trim() + '\n'); }); - }); From 8a5d818b3cd7a0582dc435e4092a5edb6774ca4d Mon Sep 17 00:00:00 2001 From: Andrew Jakubowicz Date: Mon, 27 Nov 2023 14:04:02 -0800 Subject: [PATCH 08/12] fix build error --- src/format.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/format.ts b/src/format.ts index 76c1f5ac..d56e47b6 100644 --- a/src/format.ts +++ b/src/format.ts @@ -16,7 +16,7 @@ import { ResultStatsWithDifferences, } from './stats.js'; import {BenchmarkSpec, BenchmarkResult} from './types.js'; -import {measurementName} from './measure'; +import {measurementName} from './measure.js'; export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'].map( (frame) => ansi.format(`[blue]{${frame}}`) From 97ae7fd31f507626ff6b4b07456c9cd1c80aa2ec Mon Sep 17 00:00:00 2001 From: Andrew Jakubowicz Date: Mon, 27 Nov 2023 14:36:03 -0800 Subject: [PATCH 09/12] update tests so they all pass --- src/test/config_test.ts | 1 + src/test/configfile_test.ts | 3 +++ src/test/format_test.ts | 11 ++++++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/test/config_test.ts b/src/test/config_test.ts index cc1cac61..ae5e91cc 100644 --- a/src/test/config_test.ts +++ b/src/test/config_test.ts @@ -224,6 +224,7 @@ suite('makeConfig', function () { const expected: Config = { mode: 'automatic', csvFileStats: '', + collate: false, csvFileRaw: '', jsonFile: '', legacyJsonFile: '', diff --git a/src/test/configfile_test.ts b/src/test/configfile_test.ts index 97684ede..63a0118b 100644 --- a/src/test/configfile_test.ts +++ b/src/test/configfile_test.ts @@ -236,6 +236,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, + collate: undefined, resolveBareModules: undefined, benchmarks: [ { @@ -495,6 +496,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, + collate: undefined, resolveBareModules: undefined, benchmarks: [ { @@ -581,6 +583,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, + collate: undefined, resolveBareModules: undefined, benchmarks: [ { diff --git a/src/test/format_test.ts b/src/test/format_test.ts index 7031487d..105548d2 100644 --- a/src/test/format_test.ts +++ b/src/test/format_test.ts @@ -251,6 +251,15 @@ suite('format', () => { const actual = await fakeResultTable(config); const expected = ` +┌─────────────┬───────────────┐ +│ Version │ │ +├─────────────┼───────────────┤ +│ Browser │ chrome │ +│ │ 75.0.3770.100 │ +├─────────────┼───────────────┤ +│ Sample size │ 50 │ +└─────────────┴───────────────┘ + ┌──────────────┬──────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┐ │ Benchmark │ Bytes │ Avg time │ vs foo [render] │ vs foo [update] │ vs bar [render] │ vs bar [update] │ ├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ @@ -274,7 +283,7 @@ suite('format', () => { assert.equal(actual, expected.trim() + '\n'); }); - test('multiple measurements, collate: false', async () => { + test('multiple measurements, collate: true', async () => { const config: ConfigFile = { benchmarks: [ { From 9d779ecb0298c2824183c9262453165fdfcdb734 Mon Sep 17 00:00:00 2001 From: Andrew Jakubowicz Date: Mon, 27 Nov 2023 20:39:45 -0800 Subject: [PATCH 10/12] Replace collate option to use partition: "measurement" --- CHANGELOG.md | 4 ++-- README.md | 3 ++- config.schema.json | 8 ++++---- src/config.ts | 8 ++++---- src/configfile.ts | 16 ++++++++++++---- src/defaults.ts | 2 +- src/flags.ts | 11 ++++++----- src/format.ts | 7 +++++-- src/runner.ts | 8 +++++--- src/test/config_test.ts | 10 +++++----- src/test/configfile_test.ts | 18 +++++++++--------- src/test/format_test.ts | 36 ++++++++++++++++++------------------ 12 files changed, 73 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eba60f11..ec73474e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,8 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased -- Add `collate` option for collating results tables by measurement (when - multiple measurements are in use). +- Add `partition: "measurement"` option for collating results tables by + measurement (when multiple measurements are in use). ## [0.7.0] 2022-07-15 diff --git a/README.md b/README.md index 4a53845f..a79da492 100644 --- a/README.md +++ b/README.md @@ -857,7 +857,7 @@ tach http://example.com | `--package-version` / `-p` | _(none)_ | Specify an NPM package version to swap in ([details](#swap-npm-dependencies)) | | `--browser` / `-b` | `chrome` | Which browsers to launch in automatic mode, comma-delimited (chrome, firefox, safari, edge, ie) ([details](#browsers)) | | `--window-size` | `1024,768` | "width,height" in pixels of the browser windows that will be created | -| `--sample-size` / `-n` | `50` | Minimum number of times to run each benchmark ([details](#minimum-sample-size)) | +| `--sample-size` / `-n` | `50` | Minimum number of times to run each benchmark ([details](#minimum-sample-size)) | | `--auto-sample-conditions` | `0%` | The degrees of difference to try and resolve when auto-sampling ("N%" or "Nms", comma-delimited) ([details](#auto-sample-conditions)) | | `--timeout` | `3` | The maximum number of minutes to spend auto-sampling ([details](#auto-sample)) | | `--measure` | `callback` | Which time interval to measure (`callback`, `global`, `fcp`) ([details](#measurement-modes)) | @@ -872,3 +872,4 @@ tach http://example.com | `--trace` | `false` | Enable performance tracing ([details](#performance-traces)) | | `--trace-log-dir` | `${cwd}/logs` | The directory to put tracing log files. Defaults to `${cwd}/logs`. | | `--trace-cat` | [default categories](./src/defaults.ts) | The tracing categories to record. Should be a string of comma-separated category names | +| `--partition` | _(none)_ | Use `"measurement"` to partition a single large results table into multiple tables for each [measurement](#measurement-modes). | diff --git a/config.schema.json b/config.schema.json index c07d512c..972f50ae 100644 --- a/config.schema.json +++ b/config.schema.json @@ -476,10 +476,6 @@ "minItems": 1, "type": "array" }, - "collate": { - "description": "Whether to collate result table by measurement (when multiple measurements\nare in use)", - "type": "boolean" - }, "horizons": { "description": "Deprecated alias for autoSampleConditions.", "items": { @@ -487,6 +483,10 @@ }, "type": "array" }, + "partition": { + "description": "What to partition the results table by. Use \"measurement\" when multiple\nmeasurements are in use and you want multiple smaller results tables.", + "type": "string" + }, "resolveBareModules": { "description": "Whether to automatically convert ES module imports with bare module\nspecifiers to paths.", "type": "boolean" diff --git a/src/config.ts b/src/config.ts index 62ccd6d0..7e8d3a8d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,7 +36,7 @@ export interface Config { forceCleanNpmInstall: boolean; csvFileStats: string; csvFileRaw: string; - collate?: boolean; + partition?: 'measurement'; } export async function makeConfig(opts: Opts): Promise { @@ -86,8 +86,8 @@ export async function makeConfig(opts: Opts): Promise { if (opts['window-size'] !== undefined) { throw new Error('--window-size cannot be specified when using --config'); } - if (opts['collate'] !== undefined) { - throw new Error('--collate cannot be specified when using --config'); + if (opts['partition'] !== undefined) { + throw new Error('--partition cannot be specified when using --config'); } const rawConfigObj = await fsExtra.readJson(opts.config); const validatedConfigObj = await parseConfigFile(rawConfigObj, opts.config); @@ -176,7 +176,7 @@ export function applyDefaults(partial: Partial): Config { : defaults.resolveBareModules, root: partial.root !== undefined ? partial.root : defaults.root, timeout: partial.timeout !== undefined ? partial.timeout : defaults.timeout, - collate: partial.collate !== undefined ? partial.collate : defaults.collate, + partition: partial.partition, }; } diff --git a/src/configfile.ts b/src/configfile.ts index 344207e3..49705625 100644 --- a/src/configfile.ts +++ b/src/configfile.ts @@ -63,10 +63,10 @@ export interface ConfigFile { horizons?: string[]; /** - * Whether to collate result table by measurement (when multiple measurements - * are in use) + * What to partition the results table by. Use "measurement" when multiple + * measurements are in use and you want multiple smaller results tables. */ - collate?: boolean; + partition?: string; /** * Benchmarks to run. @@ -386,6 +386,14 @@ export async function parseConfigFile( validated.autoSampleConditions = validated.horizons; } + if (validated.partition !== undefined) { + if (validated.partition !== 'measurement') { + throw new Error( + `The "partition" setting only accepts "measurement" as an option.` + ); + } + } + return { root, sampleSize: validated.sampleSize, @@ -396,7 +404,7 @@ export async function parseConfigFile( : undefined, benchmarks, resolveBareModules: validated.resolveBareModules, - collate: validated.collate, + partition: validated.partition, }; } diff --git a/src/defaults.ts b/src/defaults.ts index 25ed40f2..b9a3eddc 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -20,7 +20,7 @@ export const mode = 'automatic'; export const resolveBareModules = true; export const forceCleanNpmInstall = false; export const measurementExpression = 'window.tachometerResult'; -export const collate = false; +export const partition = ''; export const traceLogDir = path.join(process.cwd(), 'logs'); export const traceCategories = [ 'blink', diff --git a/src/flags.ts b/src/flags.ts index 8a4e83bc..71eda35a 100644 --- a/src/flags.ts +++ b/src/flags.ts @@ -230,11 +230,12 @@ export const optDefs: commandLineUsage.OptionDefinition[] = [ defaultValue: defaults.traceCategories.join(','), }, { - name: 'collate', + name: 'partition', description: - `Whether to collate result table by measurement (when multiple ` + - `measurements are in use)`, - type: Boolean, + `What to partition the results table by. Use "measurement" ` + + `when multiple measurements are in use and you want multiple ` + + `smaller results tables.`, + type: String, }, ]; @@ -266,7 +267,7 @@ export interface Opts { trace: boolean; 'trace-log-dir': string; 'trace-cat': string; - collate: boolean; + partition: string; // Extra arguments not associated with a flag are put here. These are our // benchmark names/URLs. diff --git a/src/format.ts b/src/format.ts index d56e47b6..c112fcdb 100644 --- a/src/format.ts +++ b/src/format.ts @@ -103,14 +103,17 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults { return {fixed: fixedTable, unfixed: unfixedTable}; } -export function collatedResultTables(results: ResultStatsWithDifferences[]) { +export function partitionResultTableByMeasurement( + results: ResultStatsWithDifferences[] +) { const collated: {[index: string]: ResultStatsWithDifferences[]} = {}; results.forEach((result) => { const meas = measurementName(result.result.measurement); (collated[meas] || (collated[meas] = [])).push({ ...result, differences: result.differences.filter( - (_, i) => measurementName(results[i].result.measurement) === meas) + (_, i) => measurementName(results[i].result.measurement) === meas + ), }); }); const tables: AutomaticResults[] = []; diff --git a/src/runner.ts b/src/runner.ts index 618eca67..db9993f4 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -33,7 +33,7 @@ import { automaticResultTable, spinner, benchmarkOneLiner, - collatedResultTables, + partitionResultTableByMeasurement, } from './format.js'; import {Config} from './config.js'; import * as github from './github.js'; @@ -421,8 +421,10 @@ export class Runner { console.log(); const {fixed, unfixed} = automaticResultTable(withDifferences); console.log(horizontalTermResultTable(fixed)); - if (config.collate) { - for (const {unfixed} of collatedResultTables(withDifferences)) { + if (config.partition === 'measurement') { + for (const {unfixed} of partitionResultTableByMeasurement( + withDifferences + )) { console.log(verticalTermResultTable(unfixed)); } } else { diff --git a/src/test/config_test.ts b/src/test/config_test.ts index ae5e91cc..ab081f45 100644 --- a/src/test/config_test.ts +++ b/src/test/config_test.ts @@ -45,7 +45,7 @@ suite('makeConfig', function () { csvFileStats: '', csvFileRaw: '', githubCheck: undefined, - collate: false, + partition: undefined, benchmarks: [ { browser: { @@ -91,7 +91,7 @@ suite('makeConfig', function () { csvFileRaw: '', // TODO(aomarks) Be consistent about undefined vs unset. githubCheck: undefined, - collate: false, + partition: undefined, benchmarks: [ { browser: { @@ -137,7 +137,7 @@ suite('makeConfig', function () { csvFileStats: '', csvFileRaw: '', githubCheck: undefined, - collate: false, + partition: undefined, benchmarks: [ { browser: { @@ -190,7 +190,7 @@ suite('makeConfig', function () { remoteAccessibleHost: '', // TODO(aomarks) Be consistent about undefined vs unset. githubCheck: undefined, - collate: false, + partition: undefined, benchmarks: [ { browser: { @@ -224,7 +224,7 @@ suite('makeConfig', function () { const expected: Config = { mode: 'automatic', csvFileStats: '', - collate: false, + partition: undefined, csvFileRaw: '', jsonFile: '', legacyJsonFile: '', diff --git a/src/test/configfile_test.ts b/src/test/configfile_test.ts index 63a0118b..10b996d9 100644 --- a/src/test/configfile_test.ts +++ b/src/test/configfile_test.ts @@ -48,7 +48,7 @@ suite('config', () => { timeout: 7, autoSampleConditions: ['0ms', '1ms', '2%', '+3%'], resolveBareModules: false, - collate: true, + partition: 'measurement', benchmarks: [ { name: 'remote', @@ -102,7 +102,7 @@ suite('config', () => { relative: [-0.02, 0.02, 0.03], }, resolveBareModules: false, - collate: true, + partition: 'measurement', benchmarks: [ { name: 'remote', @@ -186,7 +186,7 @@ suite('config', () => { timeout: undefined, autoSampleConditions: undefined, resolveBareModules: undefined, - collate: undefined, + partition: undefined, benchmarks: [ { name: 'http://example.com?foo=bar', @@ -236,7 +236,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, - collate: undefined, + partition: undefined, resolveBareModules: undefined, benchmarks: [ { @@ -290,7 +290,7 @@ suite('config', () => { timeout: undefined, autoSampleConditions: undefined, resolveBareModules: undefined, - collate: undefined, + partition: undefined, benchmarks: [ { name: '/mybench/index.html?foo=a', @@ -370,7 +370,7 @@ suite('config', () => { timeout: undefined, autoSampleConditions: undefined, resolveBareModules: undefined, - collate: undefined, + partition: undefined, benchmarks: [ { name: 'http://example.com', @@ -446,7 +446,7 @@ suite('config', () => { timeout: undefined, autoSampleConditions: undefined, resolveBareModules: undefined, - collate: undefined, + partition: undefined, benchmarks: [ { name: 'http://example.com?foo=bar', @@ -496,7 +496,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, - collate: undefined, + partition: undefined, resolveBareModules: undefined, benchmarks: [ { @@ -583,7 +583,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, - collate: undefined, + partition: undefined, resolveBareModules: undefined, benchmarks: [ { diff --git a/src/test/format_test.ts b/src/test/format_test.ts index 105548d2..6c4632b6 100644 --- a/src/test/format_test.ts +++ b/src/test/format_test.ts @@ -14,7 +14,7 @@ import { automaticResultTable, horizontalTermResultTable, verticalTermResultTable, - collatedResultTables, + partitionResultTableByMeasurement, } from '../format.js'; import {fakeResults, testData} from './test_helpers.js'; @@ -33,20 +33,20 @@ async function fakeResultTable(configFile: ConfigFile): Promise { /** * Given a config file object, generates fake measurement results, and returns - * the collated terminal formatted result table that would be printed (minus - * color etc. formatting). + * the partitioned formatted result tables that would be printed (minus color + * etc. formatting). */ -async function fakeCollatedResultTable(configFile: ConfigFile): - Promise { +async function fakePartitionedResultTablesByMeasurement( + configFile: ConfigFile +): Promise { const results = await fakeResults(configFile); - return collatedResultTables(results) - .map((result) => { - return stripAnsi(verticalTermResultTable(result.unfixed)); - }) - .join('\n'); + return partitionResultTableByMeasurement(results) + .map((result) => { + return stripAnsi(verticalTermResultTable(result.unfixed)); + }) + .join('\n'); } - suite('format', () => { let prevCwd: string; suiteSetup(() => { @@ -227,7 +227,7 @@ suite('format', () => { assert.equal(actual, expected.trim() + '\n'); }); - test('multiple measurements, collate: false', async () => { + test('multiple measurements, partition: undefined', async () => { const config: ConfigFile = { benchmarks: [ { @@ -235,7 +235,7 @@ suite('format', () => { url: 'http://foo.com', measurement: [ {name: 'render', mode: 'performance', entryName: 'render'}, - {name: 'update', mode: 'performance', entryName: 'update'} + {name: 'update', mode: 'performance', entryName: 'update'}, ], }, { @@ -243,7 +243,7 @@ suite('format', () => { url: 'http://bar.com', measurement: [ {name: 'render', mode: 'performance', entryName: 'render'}, - {name: 'update', mode: 'performance', entryName: 'update'} + {name: 'update', mode: 'performance', entryName: 'update'}, ], }, ], @@ -283,7 +283,7 @@ suite('format', () => { assert.equal(actual, expected.trim() + '\n'); }); - test('multiple measurements, collate: true', async () => { + test('multiple measurements, partition: "measurement"', async () => { const config: ConfigFile = { benchmarks: [ { @@ -291,7 +291,7 @@ suite('format', () => { url: 'http://foo.com', measurement: [ {name: 'render', mode: 'performance', entryName: 'render'}, - {name: 'update', mode: 'performance', entryName: 'update'} + {name: 'update', mode: 'performance', entryName: 'update'}, ], }, { @@ -299,13 +299,13 @@ suite('format', () => { url: 'http://bar.com', measurement: [ {name: 'render', mode: 'performance', entryName: 'render'}, - {name: 'update', mode: 'performance', entryName: 'update'} + {name: 'update', mode: 'performance', entryName: 'update'}, ], }, ], }; - const actual = await fakeCollatedResultTable(config); + const actual = await fakePartitionedResultTablesByMeasurement(config); const expected = ` ┌──────────────┬──────────┬───────────────────┬──────────────────┬──────────────────┐ │ Benchmark │ Bytes │ Avg time │ vs foo [render] │ vs bar [render] │ From 2bb8bcf32610caff9d14dd78b1fdee7a1660107a Mon Sep 17 00:00:00 2001 From: Andrew Jakubowicz Date: Mon, 27 Nov 2023 20:45:07 -0800 Subject: [PATCH 11/12] fix typo in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec73474e..674df606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased -- Add `partition: "measurement"` option for collating results tables by +- Add `partition: "measurement"` option for partitioning results table by measurement (when multiple measurements are in use). ## [0.7.0] 2022-07-15 From 7d707cd23e94e3464016a5da35e376e7ff9d2858 Mon Sep 17 00:00:00 2001 From: Andrew Jakubowicz Date: Tue, 28 Nov 2023 11:44:14 -0800 Subject: [PATCH 12/12] add "none" as the default value. --- README.md | 2 +- src/config.ts | 5 +++-- src/configfile.ts | 7 +++++-- src/defaults.ts | 2 +- src/test/config_test.ts | 10 +++++----- src/test/format_test.ts | 2 +- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a79da492..9e2cbef5 100644 --- a/README.md +++ b/README.md @@ -872,4 +872,4 @@ tach http://example.com | `--trace` | `false` | Enable performance tracing ([details](#performance-traces)) | | `--trace-log-dir` | `${cwd}/logs` | The directory to put tracing log files. Defaults to `${cwd}/logs`. | | `--trace-cat` | [default categories](./src/defaults.ts) | The tracing categories to record. Should be a string of comma-separated category names | -| `--partition` | _(none)_ | Use `"measurement"` to partition a single large results table into multiple tables for each [measurement](#measurement-modes). | +| `--partition` | `"none"` | Use `"measurement"` to partition a single large results table into multiple tables for each [measurement](#measurement-modes). | diff --git a/src/config.ts b/src/config.ts index 7e8d3a8d..90e42bc6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,7 +36,7 @@ export interface Config { forceCleanNpmInstall: boolean; csvFileStats: string; csvFileRaw: string; - partition?: 'measurement'; + partition: 'none' | 'measurement'; } export async function makeConfig(opts: Opts): Promise { @@ -176,7 +176,8 @@ export function applyDefaults(partial: Partial): Config { : defaults.resolveBareModules, root: partial.root !== undefined ? partial.root : defaults.root, timeout: partial.timeout !== undefined ? partial.timeout : defaults.timeout, - partition: partial.partition, + partition: + partial.partition !== undefined ? partial.partition : defaults.partition, }; } diff --git a/src/configfile.ts b/src/configfile.ts index 49705625..d6a321ad 100644 --- a/src/configfile.ts +++ b/src/configfile.ts @@ -387,9 +387,12 @@ export async function parseConfigFile( } if (validated.partition !== undefined) { - if (validated.partition !== 'measurement') { + if ( + validated.partition !== 'measurement' && + validated.partition !== 'none' + ) { throw new Error( - `The "partition" setting only accepts "measurement" as an option.` + `The "partition" setting only accepts "measurement" or "none" as an option.` ); } } diff --git a/src/defaults.ts b/src/defaults.ts index b9a3eddc..557268b4 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -20,7 +20,7 @@ export const mode = 'automatic'; export const resolveBareModules = true; export const forceCleanNpmInstall = false; export const measurementExpression = 'window.tachometerResult'; -export const partition = ''; +export const partition = 'none'; export const traceLogDir = path.join(process.cwd(), 'logs'); export const traceCategories = [ 'blink', diff --git a/src/test/config_test.ts b/src/test/config_test.ts index ab081f45..1d185bd1 100644 --- a/src/test/config_test.ts +++ b/src/test/config_test.ts @@ -45,7 +45,7 @@ suite('makeConfig', function () { csvFileStats: '', csvFileRaw: '', githubCheck: undefined, - partition: undefined, + partition: 'none', benchmarks: [ { browser: { @@ -91,7 +91,7 @@ suite('makeConfig', function () { csvFileRaw: '', // TODO(aomarks) Be consistent about undefined vs unset. githubCheck: undefined, - partition: undefined, + partition: 'none', benchmarks: [ { browser: { @@ -137,7 +137,7 @@ suite('makeConfig', function () { csvFileStats: '', csvFileRaw: '', githubCheck: undefined, - partition: undefined, + partition: 'none', benchmarks: [ { browser: { @@ -190,7 +190,7 @@ suite('makeConfig', function () { remoteAccessibleHost: '', // TODO(aomarks) Be consistent about undefined vs unset. githubCheck: undefined, - partition: undefined, + partition: 'none', benchmarks: [ { browser: { @@ -224,7 +224,7 @@ suite('makeConfig', function () { const expected: Config = { mode: 'automatic', csvFileStats: '', - partition: undefined, + partition: 'none', csvFileRaw: '', jsonFile: '', legacyJsonFile: '', diff --git a/src/test/format_test.ts b/src/test/format_test.ts index 6c4632b6..511ad78e 100644 --- a/src/test/format_test.ts +++ b/src/test/format_test.ts @@ -227,7 +227,7 @@ suite('format', () => { assert.equal(actual, expected.trim() + '\n'); }); - test('multiple measurements, partition: undefined', async () => { + test('multiple measurements, partition: "none"', async () => { const config: ConfigFile = { benchmarks: [ {