Add option to partition results table by measurements

CHANGELOG.md

@@ -5,7 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/) and this
 project adheres to [Semantic Versioning](http://semver.org/).
 
-<!-- ## Unreleased -->
+## Unreleased
+
+- Add `partition: "measurement"` option for partitioning results table by
+  measurement (when multiple measurements are in use).
 
 ## [0.7.0] 2022-07-15
 

README.md

@@ -857,7 +857,7 @@ tach http://example.com
 | `--package-version` / `-p`  | _(none)_                                | Specify an NPM package version to swap in ([details](#swap-npm-dependencies))                                                                                      |
 | `--browser` / `-b`          | `chrome`                                | Which browsers to launch in automatic mode, comma-delimited (chrome, firefox, safari, edge, ie) ([details](#browsers))                                             |
 | `--window-size`             | `1024,768`                              | "width,height" in pixels of the browser windows that will be created                                                                                               |
-| `--sample-size` / `-n`      | `50`                                    | Minimum number of times to run each benchmark ([details](#minimum-sample-size))                                                                                            |
+| `--sample-size` / `-n`      | `50`                                    | Minimum number of times to run each benchmark ([details](#minimum-sample-size))                                                                                    |
 | `--auto-sample-conditions`  | `0%`                                    | The degrees of difference to try and resolve when auto-sampling ("N%" or "Nms", comma-delimited) ([details](#auto-sample-conditions))                              |
 | `--timeout`                 | `3`                                     | The maximum number of minutes to spend auto-sampling ([details](#auto-sample))                                                                                     |
 | `--measure`                 | `callback`                              | Which time interval to measure (`callback`, `global`, `fcp`) ([details](#measurement-modes))                                                                       |
@@ -872,3 +872,4 @@ tach http://example.com
 | `--trace`                   | `false`                                 | Enable performance tracing ([details](#performance-traces))                                                                                                        |
 | `--trace-log-dir`           | `${cwd}/logs`                           | The directory to put tracing log files. Defaults to `${cwd}/logs`.                                                                                                 |
 | `--trace-cat`               | [default categories](./src/defaults.ts) | The tracing categories to record. Should be a string of comma-separated category names                                                                             |
+| `--partition`               | `"none"`                                | Use `"measurement"` to partition a single large results table into multiple tables for each [measurement](#measurement-modes).                                     |

config.schema.json

@@ -483,6 +483,10 @@
             },
             "type": "array"
         },
+        "partition": {
+            "description": "What to partition the results table by. Use \"measurement\" when multiple\nmeasurements are in use and you want multiple smaller results tables.",
+            "type": "string"
+        },
         "resolveBareModules": {
             "description": "Whether to automatically convert ES module imports with bare module\nspecifiers to paths.",
             "type": "boolean"

package.json

@@ -14,6 +14,7 @@
   "scripts": {
     "prepare": "if [ -f './tsconfig.json' ]; then npm run build; fi;",
     "build": "rimraf lib/ client/lib/ && mkdir lib && npm run generate-json-schema && tsc && tsc -p client/ && npm run lint",
+    "build:watch": "tsc --watch",
     "generate-json-schema": "typescript-json-schema tsconfig.json ConfigFile --include src/configfile.ts --required --noExtraProps > config.schema.json",
     "lint": "eslint .",
     "format": "prettier --write .",

src/config.ts

@@ -36,6 +36,7 @@ export interface Config {
   forceCleanNpmInstall: boolean;
   csvFileStats: string;
   csvFileRaw: string;
+  partition: 'none' | 'measurement';
 }
 
 export async function makeConfig(opts: Opts): Promise<Config> {
@@ -85,6 +86,9 @@ export async function makeConfig(opts: Opts): Promise<Config> {
     if (opts['window-size'] !== undefined) {
       throw new Error('--window-size cannot be specified when using --config');
     }
+    if (opts['partition'] !== undefined) {
+      throw new Error('--partition cannot be specified when using --config');
+    }
     const rawConfigObj = await fsExtra.readJson(opts.config);
     const validatedConfigObj = await parseConfigFile(rawConfigObj, opts.config);
 
@@ -172,6 +176,8 @@ export function applyDefaults(partial: Partial<Config>): Config {
         : defaults.resolveBareModules,
     root: partial.root !== undefined ? partial.root : defaults.root,
     timeout: partial.timeout !== undefined ? partial.timeout : defaults.timeout,
+    partition:
+      partial.partition !== undefined ? partial.partition : defaults.partition,
   };
 }
 

src/configfile.ts

@@ -62,6 +62,12 @@ export interface ConfigFile {
    */
   horizons?: string[];
 
+  /**
+   * What to partition the results table by. Use "measurement" when multiple
+   * measurements are in use and you want multiple smaller results tables.
+   */
+  partition?: string;
+
   /**
    * Benchmarks to run.
    * @TJS-minItems 1
@@ -380,6 +386,17 @@ export async function parseConfigFile(
     validated.autoSampleConditions = validated.horizons;
   }
 
+  if (validated.partition !== undefined) {
+    if (
+      validated.partition !== 'measurement' &&
+      validated.partition !== 'none'
+    ) {
+      throw new Error(
+        `The "partition" setting only accepts "measurement" or "none" as an option.`
+      );
+    }
+  }
+
   return {
     root,
     sampleSize: validated.sampleSize,
@@ -390,6 +407,7 @@ export async function parseConfigFile(
         : undefined,
     benchmarks,
     resolveBareModules: validated.resolveBareModules,
+    partition: validated.partition,
   };
 }
 

src/defaults.ts

@@ -20,6 +20,7 @@ export const mode = 'automatic';
 export const resolveBareModules = true;
 export const forceCleanNpmInstall = false;
 export const measurementExpression = 'window.tachometerResult';
+export const partition = 'none';
 export const traceLogDir = path.join(process.cwd(), 'logs');
 export const traceCategories = [
   'blink',

src/flags.ts

@@ -229,6 +229,14 @@ export const optDefs: commandLineUsage.OptionDefinition[] = [
     type: String,
     defaultValue: defaults.traceCategories.join(','),
   },
+  {
+    name: 'partition',
+    description:
+      `What to partition the results table by. Use "measurement" ` +
+      `when multiple measurements are in use and you want multiple ` +
+      `smaller results tables.`,
+    type: String,
+  },
 ];
 
 export interface Opts {
@@ -259,6 +267,7 @@ export interface Opts {
   trace: boolean;
   'trace-log-dir': string;
   'trace-cat': string;
+  partition: string;
 
   // Extra arguments not associated with a flag are put here. These are our
   // benchmark names/URLs.

src/format.ts

@@ -16,6 +16,7 @@ import {
   ResultStatsWithDifferences,
 } from './stats.js';
 import {BenchmarkSpec, BenchmarkResult} from './types.js';
+import {measurementName} from './measure.js';
 
 export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'].map(
   (frame) => ansi.format(`[blue]{${frame}}`)
@@ -102,6 +103,26 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults {
   return {fixed: fixedTable, unfixed: unfixedTable};
 }
 
+export function partitionResultTableByMeasurement(
+  results: ResultStatsWithDifferences[]
+) {
+  const collated: {[index: string]: ResultStatsWithDifferences[]} = {};
+  results.forEach((result) => {
+    const meas = measurementName(result.result.measurement);
+    (collated[meas] || (collated[meas] = [])).push({
+      ...result,
+      differences: result.differences.filter(
+        (_, i) => measurementName(results[i].result.measurement) === meas
+      ),
+    });
+  });
+  const tables: AutomaticResults[] = [];
+  for (const results of Object.values(collated)) {
+    tables.push(automaticResultTable(results));
+  }
+  return tables;
+}
+
 /**
  * Format a terminal text result table where each result is a row:
  *

src/runner.ts

@@ -33,6 +33,7 @@ import {
   automaticResultTable,
   spinner,
   benchmarkOneLiner,
+  partitionResultTableByMeasurement,
 } from './format.js';
 import {Config} from './config.js';
 import * as github from './github.js';
@@ -420,7 +421,15 @@ export class Runner {
     console.log();
     const {fixed, unfixed} = automaticResultTable(withDifferences);
     console.log(horizontalTermResultTable(fixed));
-    console.log(verticalTermResultTable(unfixed));
+    if (config.partition === 'measurement') {
+      for (const {unfixed} of partitionResultTableByMeasurement(
+        withDifferences
+      )) {
+        console.log(verticalTermResultTable(unfixed));
+      }
+    } else {
+      console.log(verticalTermResultTable(unfixed));
+    }
 
     if (hitTimeout === true) {
       console.log(

src/test/config_test.ts

@@ -45,6 +45,7 @@ suite('makeConfig', function () {
       csvFileStats: '',
       csvFileRaw: '',
       githubCheck: undefined,
+      partition: 'none',
       benchmarks: [
         {
           browser: {
@@ -90,6 +91,7 @@ suite('makeConfig', function () {
       csvFileRaw: '',
       // TODO(aomarks) Be consistent about undefined vs unset.
       githubCheck: undefined,
+      partition: 'none',
       benchmarks: [
         {
           browser: {
@@ -135,6 +137,7 @@ suite('makeConfig', function () {
       csvFileStats: '',
       csvFileRaw: '',
       githubCheck: undefined,
+      partition: 'none',
       benchmarks: [
         {
           browser: {
@@ -187,6 +190,7 @@ suite('makeConfig', function () {
       remoteAccessibleHost: '',
       // TODO(aomarks) Be consistent about undefined vs unset.
       githubCheck: undefined,
+      partition: 'none',
       benchmarks: [
         {
           browser: {
@@ -220,6 +224,7 @@ suite('makeConfig', function () {
     const expected: Config = {
       mode: 'automatic',
       csvFileStats: '',
+      partition: 'none',
       csvFileRaw: '',
       jsonFile: '',
       legacyJsonFile: '',

src/test/configfile_test.ts

@@ -48,6 +48,7 @@ suite('config', () => {
         timeout: 7,
         autoSampleConditions: ['0ms', '1ms', '2%', '+3%'],
         resolveBareModules: false,
+        partition: 'measurement',
         benchmarks: [
           {
             name: 'remote',
@@ -101,6 +102,7 @@ suite('config', () => {
           relative: [-0.02, 0.02, 0.03],
         },
         resolveBareModules: false,
+        partition: 'measurement',
         benchmarks: [
           {
             name: 'remote',
@@ -184,6 +186,7 @@ suite('config', () => {
         timeout: undefined,
         autoSampleConditions: undefined,
         resolveBareModules: undefined,
+        partition: undefined,
         benchmarks: [
           {
             name: 'http://example.com?foo=bar',
@@ -233,6 +236,7 @@ suite('config', () => {
         sampleSize: undefined,
         timeout: undefined,
         autoSampleConditions: undefined,
+        partition: undefined,
         resolveBareModules: undefined,
         benchmarks: [
           {
@@ -286,6 +290,7 @@ suite('config', () => {
         timeout: undefined,
         autoSampleConditions: undefined,
         resolveBareModules: undefined,
+        partition: undefined,
         benchmarks: [
           {
             name: '/mybench/index.html?foo=a',
@@ -365,6 +370,7 @@ suite('config', () => {
         timeout: undefined,
         autoSampleConditions: undefined,
         resolveBareModules: undefined,
+        partition: undefined,
         benchmarks: [
           {
             name: 'http://example.com',
@@ -440,6 +446,7 @@ suite('config', () => {
         timeout: undefined,
         autoSampleConditions: undefined,
         resolveBareModules: undefined,
+        partition: undefined,
         benchmarks: [
           {
             name: 'http://example.com?foo=bar',
@@ -489,6 +496,7 @@ suite('config', () => {
         sampleSize: undefined,
         timeout: undefined,
         autoSampleConditions: undefined,
+        partition: undefined,
         resolveBareModules: undefined,
         benchmarks: [
           {
@@ -575,6 +583,7 @@ suite('config', () => {
         sampleSize: undefined,
         timeout: undefined,
         autoSampleConditions: undefined,
+        partition: undefined,
         resolveBareModules: undefined,
         benchmarks: [
           {