Fix overview labelling and base table columns on available data (#162)

src/backend/compare/html/stats-row-across-exes.html

@@ -18,52 +18,38 @@
   %}{%= e.criteria.total.samples %}
 {% }
 %}</td>
+{% for (const i of it.criteriaOrder) {
+  const cssClassForTotal = i === 'total' ? ' stats-total' : '';
+  let medianFmtFn;
+  if (i === 'total') {
+    medianFmtFn = f.r2;
+  } else {
+    switch (it.criteria[i].unit) {
+      case 'bytes':
+        medianFmtFn = f.asHumanMem;
+        break;
+      default:
+        medianFmtFn = f.r0;
+        break;
+    }
+  }
+
+  const changeFmtFn = f.per;
+%}
 <td><span class="stats-median" title="median">
 {%
   itOut = new h.PerIterationOutput('', '<br>');
   for (const e of it.exes) {
   %}{%- itOut.next()
-  %}{%= f.r2(e.criteria.total.median) %}
+  %}{%= medianFmtFn(e.criteria[i].median) %}
 {% }
 %}</span></td>
 <td>
 {%
   itOut = new h.PerIterationOutput('', '<br>');
   for (const e of it.exes) {
   %}{%- itOut.next()
-  %}<span class="stats-change" title="change over median run time">{%= f.per(e.criteria.total.change_m) %}</span>
+  %}<span class="stats-change{%= cssClassForTotal %}" title="diff %">{%= changeFmtFn(e.criteria[i].change_m) %}</span>
 {% }
-%}</td>
-{% if (it.exes[0].criteria.gcTime) {
-%}<td><span class="stats-median" title="median">
-{%
-  itOut = new h.PerIterationOutput('', '<br>');
-  for (const e of it.exes) {
-  %}{%- itOut.next()
-  %}{%= f.r0(e.criteria.gcTime.median) %}
-{% }
-%}</span></td>
-<td>
-{%
-  itOut = new h.PerIterationOutput('', '<br>');
-  for (const e of it.exes) {
-  %}{%- itOut.next()
-  %}<span class="stats-change" title="change over median GC time">{%= f.per(e.criteria.gcTime.change_m) %}</span>
-{% }
-%}</td>{% } %}
-{% if (it.exes[0].criteria.allocated) { %}<td><span class="stats-median" title="median">
-{%
-  itOut = new h.PerIterationOutput('', '<br>');
-  for (const e of it.exes) {
-  %}{%- itOut.next()
-  %}{%= f.asHumanMem(e.criteria.allocated.median) %}
-{% }
-%}</span></td>
-<td>
-{%
-  itOut = new h.PerIterationOutput('', '<br>');
-  for (const e of it.exes) {
-  %}{%- itOut.next()
-  %}<span class="stats-change" title="change over median allocated">{%= f.per(e.criteria.allocated.change_m) %}</span>
-{% }
-%}</td>{% } %}
+%}</td>{%
+  } %}

src/backend/compare/html/stats-row-across-versions.html

@@ -1,14 +1,26 @@
 {%
 const f = it.dataFormatters;
 const s = it.stats;
+
 %}<td class="stats-samples">{%= s.total.samples %}</td>
-<td><span class="stats-median" title="median">{%= f.r2(s.total.median) %}</span></td>
-<td><span class="stats-change" title="change over median run time">{%= f.per(s.total.change_m) %}</span></td>
-{% if (s.gcTime) {
-%}<td><span class="stats-median" title="median">{%= f.r0(s.gcTime.median) %}</span></td>
-<td><span class="stats-gc-change" title="change over median GC time">{%= f.per(s.gcTime.change_m) %}</span></td>
-{% }
-   if (s.allocated) {
-  %}<td><span class="stats-median" title="median">{%= f.asHumanMem(s.allocated.median) %}</span></td>
-<td><span class="stats-alloc-change" title="change over median allocated memory">{%= f.per(s.allocated.change_m) %}</span></td>{%
-} %}
+{% for (const i of it.criteriaOrder) {
+    const cssClassForTotal = i === 'total' ? ' stats-total' : '';
+    let median;
+    if (i === 'total') {
+      median = f.r2(s[i].median);
+    } else {
+      switch (it.criteria[i].unit) {
+        case 'bytes':
+          median = f.asHumanMem(s[i].median);
+          break;
+        default:
+          median = f.r0(s[i].median);
+          break;
+      }
+    }
+
+    const change = f.per(s[i].change_m);
+%}
+<td><span class="stats-median" title="median">{%= median %}</span></td>
+<td><span class="stats-change{%= cssClassForTotal %}" title="diff %">{%= change %}</span></td>{%
+} %}

src/backend/compare/html/stats-row.html

@@ -46,12 +46,16 @@
 %}{%-   include('stats-row-across-exes.html', {
           exes: stats.exeStats,
           dataFormatters: it.dataFormatters,
-          viewHelpers: it.viewHelpers}
+          viewHelpers: it.viewHelpers,
+          criteriaOrder: it.criteriaOrder,
+          criteria: it.criteria}
       ) %}
 {%   } else {
 %}{%-   include('stats-row-across-versions.html', {
         stats: stats.versionStats,
-        dataFormatters: it.dataFormatters}
+        dataFormatters: it.dataFormatters,
+        criteriaOrder: it.criteriaOrder,
+        criteria: it.criteria}
       ) %}
 {%   }
 %}<td>{%- include('stats-row-buttons-info.html', {

src/backend/compare/html/stats-summary.html

@@ -11,13 +11,13 @@ <h2 id="overview">Result Overview</h2>
 %}
 </div>
 <dl class="row">
-  <dt class="col-sm-5" style="white-space: nowrap;">Number of Run Configurations</dt>
-  <dd class="col-sm-5">{%= it.numRunConfigs %}</dd>
+  <dt class="col-sm-3" style="white-space: nowrap;">Number of Run Configurations</dt>
+  <dd class="col-sm-7">{%= it.numRunConfigs %}</dd>
 
 {% for (let name in s) {
       const data = s[name];
       const label = name === 'total' ? 'Run time' : name;
-%}  <dt class="col-sm-5">Median {%= label %} in {%= data.unit %}</dt>
-  <dd class="col-sm-5">{%= d.r2(data.median) %} (min. {%= d.r2(data.min) %}, max. {%= d.r2(data.max) %})</dd>
+%}  <dt class="col-sm-3">Change of {%= label %}</dt>
+  <dd class="col-sm-7">median {%= d.per(data.median) %}% (min. {%= d.per(data.min) %}%, max. {%= d.per(data.max) %}%)</dd>
 {% }
 %}</dl>

src/backend/compare/html/stats-tbl-header.html

@@ -3,15 +3,13 @@
 <th scope="col"></th>
 {% if (it.isAcrossExes) { %}<th scope="col" title="Executor, without common prefix with baseline">Exe</th>{% } %}
 <th scope="col" title="Number of Samples">#M</th>
-<th scope="col">median time<br>in {%= it.criteria.total.unit %}</th>
-<th scope="col">time diff %</th>
-{% if (it.criteria['GC time']) { %}
-<th scope="col">median GC<br>time in {%= it.criteria['GC time'].unit %}</th>
-<th scope="col">GC diff %</th>
-{% }
-   if (it.criteria.Allocated) { %}
-<th scope="col">median <br>allocated {%= it.criteria.Allocated.unit %}</th>
-<th scope="col">Alloc diff %</th>
+{% for (const i of it.criteriaOrder) {
+   const name = it.criteria[i].name;
+   const unit = it.criteria[i].unit;
+   const longLabel = it.dataFormatters.smartLower(name === 'total' ? 'time' : name);
+   const shortLabel = name === 'total' ? 'time' : it.dataFormatters.shortenCriteria(longLabel);
+%}<th scope="col" title="median {%= longLabel %}">median {%= shortLabel %}<br>in {%= unit %}</th>
+<th scope="col" title="difference in % of {%= longLabel %}">{%= shortLabel %} diff %</th>
 {% } %}
 <th scope="col"></th>
 </tr></thead>

src/backend/compare/html/stats-tbl.html

@@ -1,7 +1,11 @@
 <table class="table table-sm benchmark-details">
-{%- include('stats-tbl-header.html', {
+{% const criteria = Object.keys(it.criteria);
+    it.viewHelpers.sortTotalToFront(criteria);
+%}{%- include('stats-tbl-header.html', {
   criteria: it.criteria,
-  isAcrossExes: it.isAcrossExes
+  criteriaOrder: criteria,
+  isAcrossExes: it.isAcrossExes,
+  dataFormatters: it.dataFormatters,
 }) %}
 <tbody>
 {%  it.benchmarks.sort(it.viewHelpers.sortByNameAndArguments);
@@ -12,7 +16,9 @@
           environments: it.environments,
           dataFormatters: it.dataFormatters,
           viewHelpers: it.viewHelpers,
-          isAcrossExes: it.isAcrossExes
+          isAcrossExes: it.isAcrossExes,
+          criteriaOrder: criteria,
+          criteria: it.criteria,
         }) %}
 {%  }
 %}</tbody>

src/backend/compare/prep-data.ts

@@ -539,28 +539,26 @@ async function computeStatisticsAndInlinePlot(
 
     row.versionStats[change.criterion.name] = changeStats;
 
-    if (
-      outputFolder !== null &&
-      plotName != null &&
-      change.criterion.name === inlinePlotCriterion
-    ) {
-      lastPlotId += 1;
-      row.inlinePlot = await createInlinePlot(
-        base.commitId,
-        change.commitId,
-        sortedBase,
-        sortedChange,
-        outputFolder,
-        plotName,
-        lastPlotId
-      );
+    if (change.criterion.name === inlinePlotCriterion) {
+      numRunConfigs += 1;
+
+      if (outputFolder !== null && plotName != null) {
+        lastPlotId += 1;
+        row.inlinePlot = await createInlinePlot(
+          base.commitId,
+          change.commitId,
+          sortedBase,
+          sortedChange,
+          outputFolder,
+          plotName,
+          lastPlotId
+        );
+      }
     }
 
     if (perCriteria !== null) {
       recordPerCriteria(perCriteria, measurements, i, baseOffset, changeStats);
     }
-
-    numRunConfigs += 1;
   }
 
   return { lastPlotId, numRunConfigs };
@@ -1117,6 +1115,39 @@ export async function prepareCompareView(
   return data;
 }
 
+function filterOutZeroChange(criteria: Map<string, ComparisonStatsWithUnit>) {
+  const removed: string[] = [];
+  for (const key of criteria.keys()) {
+    const stats = criteria.get(key);
+    if (stats?.data.every((s) => s.change_m === 0 && s.median === 0)) {
+      criteria.delete(key);
+      removed.push(key);
+    }
+  }
+  return removed;
+}
+
+function removeCriteriaV(
+  comparisonData: ByExeSuiteComparison,
+  removed: string[]
+) {
+  for (const bySuite of comparisonData.values()) {
+    for (const byBench of bySuite.values()) {
+      for (const r of removed) {
+        delete byBench.criteria[r];
+      }
+    }
+  }
+}
+
+function removeCriteriaE(acrossExes: BySuiteComparison, removed: string[]) {
+  for (const byBench of acrossExes.values()) {
+    for (const r of removed) {
+      delete byBench.criteria[r];
+    }
+  }
+}
+
 export async function calculateAllStatisticsAndRenderPlots(
   byExeSuiteBench: ResultsByExeSuiteBenchmark,
   suitesWithMultipleExecutors: string[],
@@ -1161,6 +1192,11 @@ export async function calculateAllStatisticsAndRenderPlots(
     inlinePlotName + '-exe'
   );
 
+  const removedCriteriaV = filterOutZeroChange(criteriaAcrossVersions);
+  const removedCriteriaE = filterOutZeroChange(criteriaAcrossExes);
+  removeCriteriaV(comparisonData, removedCriteriaV);
+  removeCriteriaE(acrossExes, removedCriteriaE);
+
   const plotData = calculateDataForOverviewPlot(comparisonData, 'total');
 
   const files = await renderOverviewPlots(

src/frontend/compare.ts

@@ -9,10 +9,11 @@ function determineAndDisplaySignificance() {
 
 function displaySignificance(sig) {
   $('#significance-val').val(`${sig}%`);
-  $('.stats-change').each((i, e) => {
+  $('.stats-change.stats-total').each((i, e) => {
     const change = parseFloat(<string>e.textContent);
     const parent = $(e).parent();
-    const target = parent.find('.stats-change').length > 1 ? $(e) : parent;
+    const target =
+      parent.find('.stats-change.stats-total').length > 1 ? $(e) : parent;
     if (change < -sig) {
       target.css('background-color', '#e4ffc7');
     } else if (change > sig) {

src/shared/data-format.ts

@@ -130,3 +130,32 @@ export function dataSeriesIds(
     `${ids.change.commitId}/${changeTrialId}`
   );
 }
+
+/**
+ * Turn the text into lower case, but keep abbreviations in upper case.
+ */
+export function smartLower(text: string): string {
+  const words = text.split(' ');
+  const result: string[] = [];
+  for (const word of words) {
+    if (word.length > 1 && word === word.toUpperCase()) {
+      result.push(word);
+    } else {
+      result.push(word.toLowerCase());
+    }
+  }
+  return result.join(' ');
+}
+
+export function shortenCriteria(text: string): string {
+  const words = text.split(' ');
+  const result: string[] = [];
+  for (const word of words) {
+    // skip the word 'time'
+    if (word.toLowerCase() === 'time') {
+      continue;
+    }
+    result.push(word);
+  }
+  return result.join(' ');
+}

src/shared/helpers.ts

@@ -92,3 +92,12 @@ export function sortByNameAndArguments(
     numeric: true
   });
 }
+
+export function sortTotalToFront(criteria: string[]): string[] {
+  const i = criteria.indexOf('total');
+  if (i !== -1) {
+    criteria.splice(i, 1);
+    criteria.unshift('total');
+  }
+  return criteria;
+}

src/shared/view-types.ts

@@ -34,6 +34,9 @@ export type CompareStatsTableHeader = Record<string, CriterionData>;
 
 export interface CompareStatsTableHeaderPartial {
   criteria: CompareStatsTableHeader;
+  criteriaOrder: string[];
+  dataFormatters: DataFormat;
+  isAcrossExes: boolean;
 }
 
 export interface CompareStatsRowAcrossExes {
@@ -45,6 +48,8 @@ export interface CompareStatsRowAcrossExesPartial {
   exes: CompareStatsRowAcrossExes[];
   dataFormatters: DataFormat;
   viewHelpers: ViewHelpers;
+  criteriaOrder: string[];
+  criteria: CompareStatsTableHeader;
 }
 
 export type CompareStatsRowAcrossVersions = Record<
@@ -55,6 +60,8 @@ export type CompareStatsRowAcrossVersions = Record<
 export interface CompareStatsRowAcrossVersionsPartial {
   stats: CompareStatsRowAcrossVersions;
   dataFormatters: DataFormat;
+  criteriaOrder: string[];
+  criteria: CompareStatsTableHeader;
 }
 
 /**
@@ -128,6 +135,8 @@ export interface CompareStatsRowPartial {
   dataFormatters: DataFormat;
   viewHelpers: ViewHelpers;
   config: ReportConfig;
+  criteriaOrder: string[];
+  criteria: CompareStatsTableHeader;
 }
 
 export interface ButtonsAdditionalInfoPartial {
@@ -149,6 +158,7 @@ export interface CompareStatsTablePartial extends CompareStatsTable {
   dataFormatters: DataFormat;
   viewHelpers: ViewHelpers;
   config: ReportConfig;
+  isAcrossExes: boolean;
 }
 
 export interface CompareNavPartial {