Change Measurement table to store iteration values in an array instea…

…d of in rows (#100)

.github/workflows/ci.yml

@@ -83,7 +83,7 @@ jobs:
 
       - name: Install ReBench
         run: |
-          git clone --depth 1 --branch rebenchdb https://github.com/smarr/rebench.git
+          git clone --depth 1 https://github.com/smarr/rebench.git
           pushd rebench
           pip install .
           popd

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rebenchdb",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "A Web-Based Database for ReBench Results",
   "main": "index.js",
   "author": {

src/backend/common/api-v1.ts

@@ -0,0 +1,80 @@
+import type { BenchmarkData, DataPoint } from '../../shared/api.js';
+
+export interface MeasureV1 {
+  /** Criterion id. */
+  c: number;
+
+  /** Value */
+  v: number;
+}
+
+export interface DataPointV1 {
+  /** Invocation */
+  in: number;
+
+  /** Iteration */
+  it: number;
+
+  m: MeasureV1[];
+}
+
+function convertDataPointsToCurrentApi(oldDs: any): DataPoint[] {
+  const result: Map<number, DataPoint> = new Map();
+  const oldDataPoints = oldDs as DataPointV1[];
+
+  for (const oldD of oldDataPoints) {
+    if (!result.has(oldD.in)) {
+      result.set(oldD.in, {
+        in: oldD.in,
+        m: []
+      });
+    }
+
+    const newDP = result.get(oldD.in)!;
+    const iteration = oldD.it;
+    for (const measure of oldD.m) {
+      if (newDP.m[measure.c] === undefined || newDP.m[measure.c] === null) {
+        newDP.m[measure.c] = [];
+
+        // mark criteria we have not seen yet explicitly with null
+        for (let i = 0; i < measure.c; i += 1) {
+          if (newDP.m[i] === undefined) {
+            newDP.m[i] = null;
+          }
+        }
+      }
+
+      // iteration 1 is at index 0, etc
+      newDP.m[measure.c]![iteration - 1] = measure.v;
+    }
+  }
+
+  const newDataPoints = [...result.values()];
+
+  // turn undefined into null, to have a consistent absent value
+  for (const dp of newDataPoints) {
+    for (const criterionMs of dp.m) {
+      if (criterionMs !== null) {
+        const cMs = criterionMs as (number | null)[];
+        for (let i = 0; i < cMs.length; i += 1) {
+          if (cMs[i] === undefined) {
+            cMs[i] = null;
+          }
+        }
+      }
+    }
+  }
+
+  return newDataPoints;
+}
+
+export function convertToCurrentApi(data: BenchmarkData): BenchmarkData {
+  for (const run of data.data) {
+    if (!run.d) {
+      continue;
+    }
+
+    run.d = convertDataPointsToCurrentApi(run.d);
+  }
+  return data;
+}

src/backend/compare/compare.ts

@@ -1,7 +1,10 @@
 import { ParameterizedContext } from 'koa';
 
 import { Database } from '../db/db.js';
-import { completeRequest, startRequest } from '../perf-tracker.js';
+import {
+  completeRequestAndHandlePromise,
+  startRequest
+} from '../perf-tracker.js';
 import type {
   ProfileRow,
   WarmupDataForTrial,
@@ -10,7 +13,7 @@ import type {
 import { respondProjectNotFound } from '../common/standard-responses.js';
 import { refreshSecret } from '../util.js';
 import { deleteReport, renderCompare } from './report.js';
-import { TimelineRequest } from '../../shared/api.js';
+import type { TimelineRequest } from '../../shared/api.js';
 
 export async function getProfileAsJson(
   ctx: ParameterizedContext,
@@ -28,7 +31,7 @@ export async function getProfileAsJson(
     ctx.body = {};
   }
   ctx.type = 'application/json';
-  completeRequest(start, db, 'get-profiles');
+  completeRequestAndHandlePromise(start, db, 'get-profiles');
 }
 
 async function getProfile(
@@ -78,7 +81,7 @@ export async function getMeasurementsAsJson(
   );
 
   ctx.type = 'application/json';
-  completeRequest(start, db, 'get-measurements');
+  completeRequestAndHandlePromise(start, db, 'get-measurements');
 }
 
 async function getMeasurements(
@@ -210,7 +213,7 @@ export async function renderComparePage(
     ctx.set('Cache-Control', 'no-cache');
   }
 
-  completeRequest(start, db, 'change');
+  completeRequestAndHandlePromise(start, db, 'change');
 }
 
 export async function deleteCachedReport(

src/backend/compare/db-data.ts

@@ -31,10 +31,8 @@ export function collateMeasurements(
   const runSettings = new Map<string, RunSettings>();
   const criteria = new Map<string, CriterionData>();
 
-  let lastInvocation = 0;
   let lastTrialId = -1;
   let lastMeasurements: Measurements | null = null;
-  let lastValues: number[] = [];
 
   // the data here needs to be sorted by
   // runId, expId, trialId, criterion, invocation, iteration
@@ -83,7 +81,7 @@ export function collateMeasurements(
 
     if (
       lastMeasurements === null ||
-      !isSameInvocation(row, lastMeasurements, lastInvocation, lastTrialId)
+      !isSameTrial(row, lastMeasurements, lastTrialId)
     ) {
       const benchResult = findOrConstructProcessedResult(forSuiteByBench, row);
 
@@ -95,21 +93,18 @@ export function collateMeasurements(
         forSuiteByBench
       );
 
-      // We don't store the invocation number anymore
-      // I think this is fine, we don't really need it.
-      // We just need to distinguish iterations, but their ordering
-      // doesn't have any particular meaning.
-      // If we should need it for statistical analysis of inter-invocation
-      // effects, we may need to re-introduce it.
-      lastValues = [];
-      m.values.push(lastValues);
-      lastInvocation = row.invocation;
       lastMeasurements = m;
       lastTrialId = row.trialid;
     }
 
-    // adjusted to be zero-based
-    lastValues[row.iteration - 1] = row.value;
+    // We don't store the invocation number anymore, since we combine
+    // data from different experiments and trials.
+    // I think this is fine, we don't really need it.
+    // We just need to distinguish iterations, but their ordering
+    // doesn't have any particular meaning.
+    // If we should need it for statistical analysis of inter-invocation
+    // effects, we may need to re-introduce it.
+    lastMeasurements.values.push(row.values);
   }
 
   return sortResultsAlphabetically(byExeSuiteBench);
@@ -217,15 +212,6 @@ function isSameMeasurements(row: MeasurementData, m: Measurements) {
   );
 }
 
-function isSameInvocation(
-  row: MeasurementData,
-  m: Measurements,
-  invocation: number,
-  trialId: number
-) {
-  return (
-    invocation == row.invocation &&
-    trialId == row.trialid &&
-    isSameMeasurements(row, m)
-  );
+function isSameTrial(row: MeasurementData, m: Measurements, trialId: number) {
+  return trialId == row.trialid && isSameMeasurements(row, m);
 }

src/backend/compare/prep-data.ts

@@ -43,6 +43,7 @@ import { collateMeasurements } from './db-data.js';
 import { HasProfile } from '../db/has-profile.js';
 import { assert } from '../../backend/logging.js';
 import { getBenchmarkArgumentsAsNamePart } from '../../shared/helpers.js';
+import type { ValuesPossiblyMissing } from '../../shared/api.js';
 
 export function compareStringOrNull(
   a: string | null,
@@ -456,17 +457,23 @@ export async function calculateChangeStatsForBenchmark(
   };
 }
 
+export function flattenedAndSortedValues(
+  values: ValuesPossiblyMissing[]
+): number[] {
+  const flattenedAndFiltered = <number[]>(
+    values.flat().filter((v) => v !== null)
+  );
+  flattenedAndFiltered.sort((a, b) => a - b);
+  return flattenedAndFiltered;
+}
+
 export function getMsFlattenedAndSorted(
   base: Measurements,
   change: Measurements
 ): { sortedBase: number[]; sortedChange: number[] } {
   // TODO: need to consider warmup setting there, before flattening
-  const sortedBase = base.values.flat();
-  sortedBase.sort((a, b) => a - b);
-
-  // TODO: need to consider warmup setting there, before flattening
-  const sortedChange = change.values.flat();
-  sortedChange.sort((a, b) => a - b);
+  const sortedBase = flattenedAndSortedValues(base.values);
+  const sortedChange = flattenedAndSortedValues(change.values);
   return { sortedBase, sortedChange };
 }
 
@@ -844,8 +851,9 @@ export async function calculateAcrossExesStatsForBenchmark(
         result.measurements[i + changeOffset] &&
         result.measurements[i + changeOffset].criterion.name === criterion
       ) {
-        const sorted = result.measurements[i + changeOffset].values.flat();
-        sorted.sort((a, b) => a - b);
+        const sorted = flattenedAndSortedValues(
+          result.measurements[i + changeOffset].values
+        );
         values.push(sorted);
       } else {
         values.push([0]);

src/backend/compare/report.ts

@@ -4,7 +4,10 @@ import { existsSync, unlinkSync, rmSync } from 'node:fs';
 
 import { Database } from '../db/db.js';
 import { assert, log } from '../logging.js';
-import { completeRequest, startRequest } from '../perf-tracker.js';
+import {
+  completeRequestAndHandlePromise,
+  startRequest
+} from '../perf-tracker.js';
 import { robustPath } from '../util.js';
 import type { CompareGenView, CompareView } from '../../shared/view-types.js';
 import { prepareCompareView } from './prep-data.js';
@@ -141,15 +144,15 @@ export async function renderCompare(
       db
     );
     const pp = p
-      .then(async () => {
+      .then(() => {
         reportStatus.inProgress = false;
-        await completeRequest(start, db, 'generate-report');
+        completeRequestAndHandlePromise(start, db, 'generate-report');
       })
-      .catch(async (e) => {
+      .catch((e) => {
         log.error('Report generation error', e);
         reportStatus.e = e;
         reportStatus.inProgress = false;
-        await completeRequest(start, db, 'generate-report');
+        completeRequestAndHandlePromise(start, db, 'generate-report');
       });
     data.completionPromise = pp;
   } else if (prevGenerationDetails.e) {

src/backend/db/database-with-pool.ts

@@ -1,17 +1,22 @@
 import pg, { PoolConfig, QueryConfig, QueryResult, QueryResultRow } from 'pg';
 
 import { Database } from './db.js';
+import { BatchingTimelineUpdater } from '../timeline/timeline-calc.js';
 
 export class DatabaseWithPool extends Database {
   private pool: pg.Pool;
 
   constructor(
     config: PoolConfig,
-    numReplicates = 1000,
+    numBootstrapSamples = 1000,
     timelineEnabled = false,
     cacheInvalidationDelay = 0
   ) {
-    super(config, numReplicates, timelineEnabled, cacheInvalidationDelay);
+    super(
+      config,
+      timelineEnabled ? new BatchingTimelineUpdater(numBootstrapSamples) : null,
+      cacheInvalidationDelay
+    );
     this.pool = new pg.Pool(config);
   }
 

src/backend/db/db.sql

@@ -154,11 +154,10 @@ CREATE TABLE Measurement (
   trialId smallint,
   criterion smallint,
   invocation smallint,
-  iteration smallint,
 
-  value float4 NOT NULL,
+  values float4[] NOT NULL,
 
-  primary key (iteration, invocation, runId, trialId, criterion),
+  primary key (invocation, runId, trialId, criterion),
   foreign key (trialId) references Trial (id),
   foreign key (runId) references Run (id),
   foreign key (criterion) references Criterion (id)
@@ -200,3 +199,27 @@ CREATE TABLE Timeline (
   foreign key (runId) references Run (id),
   foreign key (criterion) references Criterion (id)
 );
+
+-- Used by ReBenchDB's perf-tracker, for self-performance tracking
+CREATE PROCEDURE recordAdditionalMeasurement(
+  aRunId smallint,
+  aTrialId smallint,
+  aCriterionId smallint,
+  aValue float4)
+LANGUAGE plpgsql
+AS $$
+  BEGIN
+    UPDATE Measurement m
+      SET values = array_append(values, aValue)
+      WHERE
+        m.runId = aRunId AND
+        m.trialId = aTrialId AND
+        m.criterion = aCriterionId AND
+        m.invocation = 1;
+
+    IF NOT FOUND THEN
+      INSERT INTO Measurement (runId, trialId, criterion, invocation, values)
+      VALUES (aRunId, aTrialId, aCriterionId, 1, ARRAY[aValue]);
+    END IF;
+  END;
+$$;