Benchmarks in CI (without prev results comparison).

.github/workflows/benchmarks.yml

@@ -0,0 +1,42 @@
+name: Benchmark o1js
+on:
+  push:
+    branches:
+      - main
+      - berkeley
+      - develop
+  pull_request:
+  workflow_dispatch: {}
+
+jobs:
+  benchmarks:
+    timeout-minutes: 30
+    strategy:
+      fail-fast: true
+      matrix:
+        node: [18, 20]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Node.JS ${{ matrix.node }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+      - name: Build o1js and execute benchmarks on ${{ matrix.os }} and Node.JS ${{ matrix.node }}
+        env:
+          GIT_HASH: ${{ github.sha }}
+          GIT_BRANCH: ${{ github.head_ref }}
+          INFLUXDB_URL: ${{ secrets.INFLUXDB_URL }}
+          INFLUXDB_ORG: ${{ secrets.INFLUXDB_ORG }}
+          INFLUXDB_BUCKET: ${{ secrets.INFLUXDB_BUCKET }}
+          INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}
+          METRICS_SOURCE_ENVIRONMENT: 'o1js GitHub Actions'
+        run: |
+          git submodule update --init --recursive
+          npm ci
+          npm run build
+          bash run-ci-benchmarks.sh
+          cat benchmarks.log >> $GITHUB_STEP_SUMMARY
+        shell: bash

benchmarks/benchmark.ts

@@ -1,8 +1,11 @@
 /**
  * Benchmark runner
  */
+
+import { InfluxDB, Point } from '@influxdata/influxdb-client';
 import jStat from 'jstat';
-export { BenchmarkResult, Benchmark, benchmark, printResult, pValue };
+import os from 'node:os';
+export { Benchmark, BenchmarkResult, benchmark, logResult, pValue };
 
 type BenchmarkResult = {
   label: string;
@@ -13,6 +16,8 @@ type BenchmarkResult = {
 
 type Benchmark = { run: () => Promise<BenchmarkResult[]> };
 
+const influxDbClient = setupInfluxDbClient();
+
 function benchmark(
   label: string,
   run:
@@ -100,14 +105,16 @@ function getStatistics(numbers: number[]) {
   return { mean, variance, size: n };
 }
 
-function printResult(
-  result: BenchmarkResult,
-  previousResult?: BenchmarkResult
-) {
+function logResult(result: BenchmarkResult, previousResult?: BenchmarkResult) {
   console.log(result.label + `\n`);
   console.log(`time: ${resultToString(result)}`);
 
-  if (previousResult === undefined) return;
+  writeResultToInfluxDb(result);
+
+  if (previousResult === undefined) {
+    console.log('\n');
+    return;
+  }
 
   let change = (result.mean - previousResult.mean) / previousResult.mean;
   let p = pValue(result, previousResult);
@@ -156,3 +163,61 @@ function pValue(sample1: BenchmarkResult, sample2: BenchmarkResult): number {
   const pValue = 2 * (1 - jStat.studentt.cdf(Math.abs(tStatistic), df));
   return pValue;
 }
+
+function getInfluxDbClientOptions() {
+  return {
+    url: process.env.INFLUXDB_URL,
+    token: process.env.INFLUXDB_TOKEN,
+    org: process.env.INFLUXDB_ORG,
+    bucket: process.env.INFLUXDB_BUCKET,
+  };
+}
+
+function getInfluxDbPointTags() {
+  return {
+    sourceEnvironment: process.env.METRICS_SOURCE_ENVIRONMENT ?? 'local',
+    operatingSystem: `${os.type()} ${os.release()} ${os.arch()} `,
+    hardware: `${os.cpus()[0].model} ${os.cpus().length} cores, ${(
+      os.totalmem() /
+      1024 /
+      1024 /
+      1024
+    ).toFixed(2)}Gb of RAM`,
+    gitHash: process.env.GIT_HASH ?? 'unknown',
+    gitBranch: process.env.GIT_BRANCH ?? 'unknown',
+  };
+}
+
+function setupInfluxDbClient(): InfluxDB | undefined {
+  const { url, token } = getInfluxDbClientOptions();
+  if (url === undefined || token === undefined) {
+    return undefined;
+  }
+  return new InfluxDB({ url, token });
+}
+
+function writeResultToInfluxDb(result: BenchmarkResult) {
+  const { org, bucket } = getInfluxDbClientOptions();
+  if (influxDbClient && org && bucket) {
+    console.log('Writing result to InfluxDB');
+    const influxDbWriteClient = influxDbClient.getWriteApi(org, bucket, 'ms');
+    try {
+      const point = new Point(`${result.label} - ${result.size} samples`)
+        .tag('benchmarkName', result.label)
+        .tag('sampledTimes', result.size.toString())
+        .floatField('mean', result.mean)
+        .floatField('variance', result.variance)
+        .intField('size', result.size);
+      for (const [key, value] of Object.entries(getInfluxDbPointTags())) {
+        point.tag(key, value);
+      }
+      influxDbWriteClient.writePoint(point);
+    } catch (e) {
+      console.error('Error writing to InfluxDB: ', e);
+    } finally {
+      influxDbWriteClient.close();
+    }
+  } else {
+    console.error('Skipping writing to InfluxDB: not configured');
+  }
+}

benchmarks/ecdsa.ts

@@ -1,19 +1,20 @@
 /**
- * Benchmark runner example
+ * ECDSA benchmark
  *
  * Run with
  * ```
  * ./run benchmarks/ecdsa.ts --bundle
  * ```
  */
+
 import { Provable } from 'o1js';
 import {
-  keccakAndEcdsa,
-  Secp256k1,
-  Ecdsa,
   Bytes32,
+  Ecdsa,
+  Secp256k1,
+  keccakAndEcdsa,
 } from '../src/examples/crypto/ecdsa/ecdsa.js';
-import { BenchmarkResult, benchmark, printResult } from './benchmark.js';
+import { BenchmarkResult, benchmark, logResult } from './benchmark.js';
 
 let privateKey = Secp256k1.Scalar.random();
 let publicKey = Secp256k1.generator.scale(privateKey);
@@ -28,20 +29,23 @@ const EcdsaBenchmark = benchmark(
     toc();
 
     tic('witness generation');
-    await Provable.runAndCheck(() => {
+    await Provable.runAndCheck(async () => {
       let message_ = Provable.witness(Bytes32.provable, () => message);
       let signature_ = Provable.witness(Ecdsa.provable, () => signature);
       let publicKey_ = Provable.witness(Secp256k1.provable, () => publicKey);
-      keccakAndEcdsa.rawMethods.verifyEcdsa(message_, signature_, publicKey_);
+      await keccakAndEcdsa.rawMethods.verifyEcdsa(
+        message_,
+        signature_,
+        publicKey_
+      );
     });
     toc();
   },
   // two warmups to ensure full caching
   { numberOfWarmups: 2, numberOfRuns: 5 }
 );
 
-// mock: load previous results
-
+// TODO: load previous results from InfluxDB
 let previousResults: BenchmarkResult[] = [
   {
     label: 'ecdsa - build constraint system',
@@ -58,14 +62,12 @@ let previousResults: BenchmarkResult[] = [
 ];
 
 // run benchmark
-
 let results = await EcdsaBenchmark.run();
 
 // example for how to log results
 // criterion-style comparison of result to previous one, check significant improvement
-
 for (let i = 0; i < results.length; i++) {
   let result = results[i];
   let previous = previousResults[i];
-  printResult(result, previous);
+  logResult(result, previous);
 }

package.json

@@ -94,6 +94,7 @@
     "typescript": "5.1"
   },
   "dependencies": {
+    "@influxdata/influxdb-client": "^1.33.2",
     "blakejs": "1.2.1",
     "cachedir": "^2.4.0",
     "isomorphic-fetch": "^3.0.0",

run-ci-benchmarks.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -e
+
+echo ""
+echo "Running o1js benchmarks."
+echo ""
+
+./run benchmarks/ecdsa.ts --bundle >>benchmarks.log 2>&1