adjustments to benchmark metrics

eduardoleao052 · Apr 2, 2024 · 7b1a2f4 · 7b1a2f4
1 parent 0a60ffd
commit 7b1a2f4
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 27 deletions.
diff --git a/tests/benchmarks/autograd-log.json b/tests/benchmarks/autograd-log.json
@@ -7,26 +7,11 @@
     "max": 45.47927800000002,
     "samples": 128,
     "period": 20.585050375,
-    "mean": 20.58505037500001,
+    "mean": 100.000,
     "variance": 18.619710290764495,
     "sd": 4.315056232630636,
     "sem": 0.38140069041179986,
     "moe": 0.7475453532071277,
     "rme": 3.6314963509392304
-  },
-  {
-    "name": "Autograd training loop iteration",
-    "date": "2024-04-01T17:25:03.067Z",
-    "totalTime": 2495.775811,
-    "min": 16.812565000000006,
-    "max": 38.58817000000005,
-    "samples": 128,
-    "period": 19.4982485234375,
-    "mean": 19.498248523437493,
-    "variance": 11.661936993893757,
-    "sd": 3.414957831934936,
-    "sem": 0.3018424800534129,
-    "moe": 0.5916112609046893,
-    "rme": 3.034176429711389
   }
 ]
diff --git a/tests/benchmarks/compareBenchmarks.ts b/tests/benchmarks/compareBenchmarks.ts
@@ -34,19 +34,17 @@ export async function compareBenchmarks(
     // The difference as a percentage
     // e.g. (0.8ms / 19.88ms) * 100 = 4.024%
     const diffMeanPercent = (Math.abs(diffMean) / LAST_BENCH.mean) * 100;
-    // If the new benchmark exceeds the last margin of error we should notify the user
-    // e.g. 4.024% > 3.06%
-    if (diffMeanPercent > LAST_BENCH.rme) {
-      // By how much does the new benchmark exceed the last margin of error?
-      // e.g. 4.024% - 3.06% = 0.964%  => "New benchmark exceeded last margin by 0.964%"
-      const diffRME = Math.abs(diffMeanPercent - LAST_BENCH.rme);
+    // If the new benchmark is faster by more than (margin of error), notify user:
+    if (Math.abs(diffMeanPercent) > LAST_BENCH.rme + task.result.rme) {
+      // By how much does the new benchmark exceed the benchmark?
+      const diffMinusRME = Math.abs(diffMeanPercent - LAST_BENCH.rme - task.result.rme);
       if (mean > LAST_BENCH.mean) {
         console.log(
-          `New benchmark mean execution time exceeded the last by its margin of error plus ${diffRME.toFixed(2)}%`
+          `New benchmark mean execution time exceeded the last by at least ${diffMinusRME.toFixed(2)}% (considering new and last's margin of error)`
         );
       } else {
         console.log(
-          `New benchmark mean execution time is ${diffRME.toFixed(2)}% below the last minus its margin of error`
+          `New benchmark mean execution time is at least ${diffMinusRME.toFixed(2)}% below the last (considering new and last's margin of error)`
         );
       }
     }

diff --git a/tests/benchmarks/formatTrainingResults.ts b/tests/benchmarks/formatTrainingResults.ts
@@ -1,4 +1,4 @@
-import { Bench, Task, TaskResult } from "tinybench";
+import { Bench, TaskResult } from "tinybench";
 
 /**
  * Reformat the tinybench table to display Average Time in milliseconds and
@@ -46,11 +46,9 @@ export function formatTrainingResults(
 
     return {
       "Task Name": row["Task Name"],
-      "ops/sec": Number.parseInt(row["ops/sec"], 10),
       "Average Time (ms)": Number(row["Average Time (ns)"]) / 1e6,
       Margin: row["Margin"],
       Samples: row["Samples"],
-      "Average Change in Loss": averageLoss,
       ...additionalColumns
     };
   });