Skip to content

Commit

Permalink
Continuous benchmarking in CI (#7100)
Browse files Browse the repository at this point in the history
* Syntax benchmark: cleanup

* Output benchmark results as JSON

* Output JSON lazily

* Increment opam cache key

* Build benchmarks only for OCaml >= 4.14.0

* Continuous benchmarking in CI

* Permissions

* Beautify

* Cleanup and fix benchmark

* More cleanup

* Do not attempt to run action for PRs created from other repos

* Fix caml_mach_absolute_time for Linux

* Add job summary
  • Loading branch information
cknitt authored Oct 13, 2024
1 parent 5f5917e commit e1b7fb7
Show file tree
Hide file tree
Showing 6 changed files with 116 additions and 88 deletions.
35 changes: 33 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ on:
pull_request:
branches: [master, 11.0_release]

permissions:
# allow posting comments to pull request
pull-requests: write

concurrency:
group: ci-${{ github.ref }}-1
# Cancel previous builds for pull requests only.
Expand Down Expand Up @@ -90,8 +94,9 @@ jobs:
ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static
upload_binaries: true
upload_libs: true
# Build the playground compiler on the fastest runner
# Build the playground compiler and run the benchmarks on the fastest runner
build_playground: true
benchmarks: true
- os: buildjet-2vcpu-ubuntu-2204-arm # ARM
ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static
upload_binaries: true
Expand Down Expand Up @@ -150,7 +155,7 @@ jobs:
# matrix.ocaml_compiler may contain commas
- name: Get OPAM cache key
shell: bash
run: echo "opam_cache_key=opam-env-v3-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV
run: echo "opam_cache_key=opam-env-v4-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV

- name: Restore OPAM environment
id: cache-opam-env
Expand Down Expand Up @@ -320,6 +325,32 @@ jobs:
if: runner.os != 'Windows'
run: make -C tests/gentype_tests/typescript-react-example clean test

- name: Run syntax benchmarks
if: matrix.benchmarks
run: ./_build/install/default/bin/syntax_benchmarks | tee tests/benchmark-output.json

- name: Download previous benchmark data
if: matrix.benchmarks
uses: actions/cache@v4
with:
path: ./tests/benchmark-cache
key: syntax-benchmark-v1

- name: Store benchmark result
# Do not run for PRs created from other repos as those won't be able to write to the pull request
if: ${{ matrix.benchmarks && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.event.repository.full_name) }}
uses: benchmark-action/github-action-benchmark@v1
with:
name: Syntax Benchmarks
tool: customSmallerIsBetter
output-file-path: tests/benchmark-output.json
external-data-json-path: ./tests/benchmark-cache/benchmark-data.json
github-token: ${{ secrets.GITHUB_TOKEN }}
alert-threshold: "150%"
comment-always: true
comment-on-alert: true
summary-always: true

- name: Build playground compiler
if: matrix.build_playground
run: |
Expand Down
4 changes: 4 additions & 0 deletions dune-project
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
(and
:with-test
(= 0.26.2)))
(yojson
(and
:with-test
(= 2.2.2)))
(ocaml-lsp-server
(and
:with-dev-setup
Expand Down
1 change: 1 addition & 0 deletions rescript.opam
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ bug-reports: "https://github.com/rescript-lang/rescript-compiler/issues"
depends: [
"ocaml" {>= "4.10"}
"ocamlformat" {with-test & = "0.26.2"}
"yojson" {with-test & = "2.2.2"}
"ocaml-lsp-server" {with-dev-setup & = "1.19.0"}
"cppo" {= "1.6.9"}
"js_of_ocaml" {= "5.8.1"}
Expand Down
159 changes: 75 additions & 84 deletions tests/syntax_benchmarks/Benchmark.ml
Original file line number Diff line number Diff line change
Expand Up @@ -75,59 +75,31 @@ end = struct
end

module Benchmark : sig
type t
type test_result = {ms_per_run: float; allocs_per_run: int}

val make : name:string -> f:(t -> unit) -> unit -> t
val launch : t -> unit
val report : t -> unit
val run : (unit -> unit) -> num_iterations:int -> test_result
end = struct
type t = {
name: string;
mutable start: Time.t;
mutable n: int; (* current iterations count *)
mutable duration: Time.t;
bench_func: t -> unit;
mutable n: int; (* current iteration count *)
mutable total_duration: Time.t;
bench_func: unit -> unit;
mutable timer_on: bool;
(* mutable result: benchmarkResult; *)
(* The initial states *)
mutable start_allocs: float;
mutable start_bytes: float;
(* The net total of this test after being run. *)
mutable net_allocs: float;
mutable net_bytes: float;
mutable total_allocs: float;
}

let report b =
print_endline (Format.sprintf "Benchmark: %s" b.name);
print_endline (Format.sprintf "Nbr of iterations: %d" b.n);
print_endline
(Format.sprintf "Benchmark ran during: %fms" (Time.print b.duration));
print_endline
(Format.sprintf "Avg time/op: %fms"
(Time.print b.duration /. float_of_int b.n));
print_endline
(Format.sprintf "Allocs/op: %d"
(int_of_float (b.net_allocs /. float_of_int b.n)));
print_endline
(Format.sprintf "B/op: %d"
(int_of_float (b.net_bytes /. float_of_int b.n)));

(* return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() *)
print_newline ();
()
type test_result = {ms_per_run: float; allocs_per_run: int}

let make ~name ~f () =
let make f =
{
name;
start = Time.zero;
n = 0;
bench_func = f;
duration = Time.zero;
total_duration = Time.zero;
timer_on = false;
start_allocs = 0.;
start_bytes = 0.;
net_allocs = 0.;
net_bytes = 0.;
total_allocs = 0.;
}

(* total amount of memory allocated by the program since it started in words *)
Expand All @@ -139,79 +111,74 @@ end = struct
if not b.timer_on then (
let allocated_words = mallocs () in
b.start_allocs <- allocated_words;
b.start_bytes <- allocated_words *. 8.;
b.start <- Time.now ();
b.timer_on <- true)

let stop_timer b =
if b.timer_on then (
let allocated_words = mallocs () in
let diff = Time.diff b.start (Time.now ()) in
b.duration <- Time.add b.duration diff;
b.net_allocs <- b.net_allocs +. (allocated_words -. b.start_allocs);
b.net_bytes <- b.net_bytes +. ((allocated_words *. 8.) -. b.start_bytes);
b.total_duration <- Time.add b.total_duration diff;
b.total_allocs <- b.total_allocs +. (allocated_words -. b.start_allocs);
b.timer_on <- false)

let reset_timer b =
if b.timer_on then (
let allocated_words = mallocs () in
b.start_allocs <- allocated_words;
b.net_allocs <- allocated_words *. 8.;
b.start <- Time.now ());
b.net_allocs <- 0.;
b.net_bytes <- 0.
b.start <- Time.now ())

let run_iteration b n =
Gc.full_major ();
b.n <- n;
reset_timer b;
start_timer b;
b.bench_func b;
b.bench_func ();
stop_timer b

let launch b =
(* 150 runs * all the benchmarks means around 1m of benchmark time *)
for n = 1 to 150 do
let run f ~num_iterations =
let b = make f in
for n = 1 to num_iterations do
run_iteration b n
done
done;
{
ms_per_run = Time.print b.total_duration /. float_of_int b.n;
allocs_per_run = int_of_float (b.total_allocs /. float_of_int b.n);
}
end

module Benchmarks : sig
val run : unit -> unit
end = struct
type action = Parse | Print

let string_of_action action =
match action with
| Parse -> "parser"
| Print -> "printer"

(* TODO: we could at Reason here *)
type lang = Rescript
let string_of_lang lang =
match lang with
| Rescript -> "rescript"
| Parse -> "Parse"
| Print -> "Print"

let parse_rescript src filename =
let p = Parser.make src filename in
let structure = ResParser.parse_implementation p in
assert (p.diagnostics == []);
structure

let benchmark filename lang action =
let src = IO.read_file filename in
let name =
filename ^ " " ^ string_of_lang lang ^ " " ^ string_of_action action
in
let data_dir = "tests/syntax_benchmarks/data"
let num_iterations = 150

let benchmark (filename, action) =
let path = Filename.concat data_dir filename in
let src = IO.read_file path in
let benchmark_fn =
match (lang, action) with
| Rescript, Parse ->
fun _ ->
let _ = Sys.opaque_identity (parse_rescript src filename) in
match action with
| Parse ->
fun () ->
let _ = Sys.opaque_identity (parse_rescript src path) in
()
| Rescript, Print ->
let p = Parser.make src filename in
| Print ->
let p = Parser.make src path in
let ast = ResParser.parse_implementation p in
fun _ ->
fun () ->
let _ =
Sys.opaque_identity
(let cmt_tbl = CommentTable.make () in
Expand All @@ -221,21 +188,45 @@ end = struct
in
()
in
let b = Benchmark.make ~name ~f:benchmark_fn () in
Benchmark.launch b;
Benchmark.report b
Benchmark.run benchmark_fn ~num_iterations

let specs =
[
("RedBlackTree.res", Parse);
("RedBlackTree.res", Print);
("RedBlackTreeNoComments.res", Print);
("Napkinscript.res", Parse);
("Napkinscript.res", Print);
("HeroGraphic.res", Parse);
("HeroGraphic.res", Print);
]

let run () =
let data_dir = "tests/syntax_benchmarks/data" in
benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Parse;
benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Print;
benchmark
(Filename.concat data_dir "RedBlackTreeNoComments.res")
Rescript Print;
benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Parse;
benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Print;
benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Parse;
benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Print
List.to_seq specs
|> Seq.flat_map (fun spec ->
let filename, action = spec in
let test_name = string_of_action action ^ " " ^ filename in
let {Benchmark.ms_per_run; allocs_per_run} = benchmark spec in
[
`Assoc
[
("name", `String (Format.sprintf "%s - time/run" test_name));
("unit", `String "ms");
("value", `Float ms_per_run);
];
`Assoc
[
("name", `String (Format.sprintf "%s - allocs/run" test_name));
("unit", `String "words");
("value", `Int allocs_per_run);
];
]
|> List.to_seq)
|> Seq.iteri (fun i json ->
print_endline (if i == 0 then "[" else ",");
print_string (Yojson.to_string json));
print_newline ();
print_endline "]"
end

let () = Benchmarks.run ()
3 changes: 2 additions & 1 deletion tests/syntax_benchmarks/dune
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
(enabled_if
(and
(<> %{profile} browser)
(>= %{ocaml_version} "4.14.0")
(or
(= %{system} macosx)
; or one of Linuxes (see https://github.com/ocaml/ocaml/issues/10613)
Expand All @@ -22,6 +23,6 @@
(foreign_stubs
(language c)
(names time))
(libraries syntax))
(libraries syntax yojson))

(data_only_dirs data)
2 changes: 1 addition & 1 deletion tests/syntax_benchmarks/time.c
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ CAMLprim value caml_mach_absolute_time(value unit) {
#elif defined(__linux__)
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
result = now.tv_sec * 1000 + now.tv_nsec / 1000000;
result = now.tv_sec * 1000000000 + now.tv_nsec;
#endif

return caml_copy_int64(result);
Expand Down

0 comments on commit e1b7fb7

Please sign in to comment.