Mark bars gray if validation fails (#38)

sourcemeta-research · Sep 12, 2024 · 6ddb9c3 · 6ddb9c3
1 parent e4155a1
commit 6ddb9c3
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 13 deletions.
diff --git a/.github/csv_min.py b/.github/csv_min.py
@@ -16,4 +16,6 @@
 for row in reader:
     if min_impl[row['name']] == row['implementation']:
         row['name'] += ' :white_check_mark:'
+    if row['exit_status'] != '0':
+        row['name'] += ' :x:'
     writer.writerow(row)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -21,7 +21,9 @@ jobs:
       - name: Install uv
         run: pipx install uv
 
-      - run: make
+      - name: Run benchmarks
+        continue-on-error: true
+        run: make dist/report.csv
       - run: cat dist/report.csv
       - name: Process CSV
         run: python .github/csv_min.py > dist/report-min.csv

diff --git a/Makefile b/Makefile
@@ -27,6 +27,13 @@ plots: $(ALL_PLOTS)
 .PHONY: all
 all: dist/report.csv ; cat $<
 
+define docker_run
+  $(eval $@_TOOL = $(1))
+  $(eval $@_INPUT = $(2))
+				$(shell docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/$($@_TOOL) $($@_INPUT) > $@)
+				@sed -i 's/$$/,$(.SHELLSTATUS)/' $@
+endef
+
 # JSON Toolkit
 
 implementations/jsontoolkit/.dockertimestamp: \
@@ -41,7 +48,7 @@ dist/results/jsontoolkit/%: \
 	schemas/%/schema.json \
 	schemas/%/instances.jsonl \
 	| dist/results/jsontoolkit
-	docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/jsontoolkit /workspace/$(dir $(word 2,$^)) > $@
+	@$(call docker_run,jsontoolkit,/workspace/$(dir $(word 2,$^)))
 
 # AJV
 
@@ -58,7 +65,7 @@ dist/results/ajv/%: \
 	schemas/%/schema.json \
 	schemas/%/instances.jsonl \
 	| dist/results/ajv
-	docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/ajv /workspace/$(word 2,$^) /workspace/$(word 3,$^) > $@
+	@$(call docker_run,ajv,/workspace/$(word 2,$^) /workspace/$(word 3,$^))
 
 # BOON
 
@@ -74,7 +81,7 @@ dist/results/boon/%: \
 	schemas/%/schema.json \
 	schemas/%/instances.jsonl \
 	| dist/results/boon
-	docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/boon /workspace/$(dir $(word 2,$^)) > $@
+	@$(call docker_run,boon,/workspace/$(dir $(word 2,$^)))
 
 # JSON_SCHEMER
 
@@ -91,7 +98,7 @@ dist/results/json_schemer/%: \
 	schemas/%/schema.json \
 	schemas/%/instances.jsonl \
 	| dist/results/json_schemer
-	docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/json_schemer /workspace/$(dir $(word 3,$^)) > $@
+	@$(call docker_run,json_schemer,/workspace/$(dir $(word 3,$^)))
 
 # PYTHON / JSONSCHEMA
 
@@ -108,7 +115,7 @@ dist/results/python-jsonschema/%: \
 	schemas/%/schema.json \
 	schemas/%/instances.jsonl \
 	| dist/results/python-jsonschema
-	docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/python-jsonschema /workspace/$(dir $(word 2,$^)) > $@
+	@$(call docker_run,python-jsonschema,/workspace/$(dir $(word 2,$^)))
 
 # GO / JSONSCHEMA
 
@@ -125,4 +132,4 @@ dist/results/go-jsonschema/%: \
 	schemas/%/schema.json \
 	schemas/%/instances.jsonl \
 	| dist/results/go-jsonschema
-	docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/go-jsonschema /workspace/$(dir $(word 2,$^)) > $@
+	@$(call docker_run,go-jsonschema,/workspace/$(dir $(word 2,$^)))
diff --git a/implementations/ajv/main.mjs b/implementations/ajv/main.mjs
@@ -38,18 +38,23 @@ async function validateSchema(schemaPath, instancePath) {
   for await (const instance of readJSONLines(instancePath)) {
     instances.push(instance);
   }
+  let failed = false;
   const startTime = performance.now();
   for (const instance of instances) {
     if (!validate(instance)) {
-      // XXX Temporarily allow failures
-      // process.exit(1);
+      failed = true;
     }
   }
 
   const endTime = performance.now();
 
   const durationNs = (endTime - startTime) * 1e6;
   console.log(durationNs.toFixed(0));
+
+  // Exit with non-zero status on validation failure
+  if (failed) {
+    process.exit(1);
+  }
 }
 
 if (process.argv.length !== 4) {

diff --git a/plot.py b/plot.py
@@ -18,7 +18,11 @@
     for (name, data) in examples.items():
         data = pd.DataFrame(data)
         plt.figure(figsize=(4, 6), dpi=96)
-        plot = sns.barplot(data, x="implementation", y="milliseconds", errorbar=None)
+
+        # Mark with gray if the command failed
+        colors = ['green' if s == '0' else 'gray' for s in data.exit_status]
+
+        plot = sns.barplot(data, x="implementation", y="milliseconds", palette=colors, errorbar=None)
         plot.set(xlabel=None)
         plot.tick_params(axis='x', rotation=30)
         plt.tight_layout()

diff --git a/report.sh b/report.sh
@@ -9,13 +9,13 @@ then
   exit 1
 fi
 
-echo "implementation,version,name,nanoseconds"
+echo "implementation,version,name,nanoseconds,exit_status"
 
 for argument in "$@"
 do
   IMPLEMENTATION="$(basename "$(dirname "$argument")")"
   EXAMPLE="$(basename "$argument")"
-  NANOSECONDS="$(tr -d '\n\r' < "$argument")"
+  OUTPUT="$(tr -d '\n\r' < "$argument")"
   VERSION="$("./implementations/$IMPLEMENTATION/version.sh")"
-  echo "$IMPLEMENTATION,$VERSION,$EXAMPLE,$NANOSECONDS"
+  echo "$IMPLEMENTATION,$VERSION,$EXAMPLE,$OUTPUT"
 done