From 147b1e3e967803a8ac4505857e3e67bee5bb020a Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Tue, 7 Sep 2021 14:20:44 -0400 Subject: [PATCH 1/3] feat: Add Jogger load_benchmark Fixes Add Jogger version of load_benchmarks #15 Updated docs to include new Jogger method --- docs/src/jogger.md | 1 + src/jogger.jl | 16 +++++++++++++++- test/Project.toml | 1 + test/smoke.jl | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/src/jogger.md b/docs/src/jogger.md index be7be24..03e4153 100644 --- a/docs/src/jogger.md +++ b/docs/src/jogger.md @@ -30,5 +30,6 @@ JogExample.benchmark JogExample.warmup JogExample.run JogExample.save_benchmarks +JogExample.load_benchmarks JogExample.BENCHMARK_DIR ``` diff --git a/src/jogger.jl b/src/jogger.jl index 0107e5b..d3c626b 100644 --- a/src/jogger.jl +++ b/src/jogger.jl @@ -128,13 +128,27 @@ macro jog(pkg) Returns the path to the saved results - Results can be loaded with [`PkgJogger.load_benchmarks(filename)`](@ref) + Results can be loaded with [`PkgJogger.load_benchmarks(filename)`](@ref) or + [`$($modname).load_benchmarks(uuid)`](@ref) """ function save_benchmarks(results) filename = joinpath(BENCHMARK_DIR, "trial", "$(UUIDs.uuid4()).json.gz") PkgJogger.save_benchmarks(filename, results) filename end + + """ + load_benchmarks(uuid::String)::Dict + load_benchmarks(uuid::UUID)::Dict + + Loads benchmarking results for $($pkg) from `BENCHMARK_DIR/trial` + """ + load_benchmarks(uuid::UUIDs.UUID) = load_benchmarks(string(uuid)) + function load_benchmarks(uuid::AbstractString) + path = joinpath(BENCHMARK_DIR, "trial", uuid * ".json.gz") + @assert isfile(path) "Missing benchmarking results for $uuid, expected path: $path" + PkgJogger.load_benchmarks(path) + end end end end diff --git a/test/Project.toml b/test/Project.toml index eaa7bf5..0dc759c 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -4,3 +4,4 @@ Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" diff --git a/test/smoke.jl b/test/smoke.jl index 0aa4f44..4d7c0ac 100644 --- a/test/smoke.jl +++ b/test/smoke.jl @@ -1,5 +1,6 @@ using Test using PkgJogger +using UUIDs import BenchmarkTools include("utils.jl") @@ -29,6 +30,21 @@ include("utils.jl") test_loaded_results(r2) @test r == r2["benchmarks"] + # Load with JogPkgJogger + @testset "Jogger's load_benchmarks" begin + uuid = splitpath(file)[end] |> x -> split(x, ".")[1] + r3 = JogPkgJogger.load_benchmarks(uuid) + r4 = JogPkgJogger.load_benchmarks(UUID(uuid)) + @test r3 == r4 + @test r3["benchmarks"] == r + @test r4["benchmarks"] == r + @test r2 == r3 == r4 + + # Check that we error for invalid uuids + @test_throws AssertionError JogPkgJogger.load_benchmarks("not-a-uuid") + @test_throws AssertionError JogPkgJogger.load_benchmarks(UUIDs.uuid4()) + end + # If this is a git repo, there should be a git entry if isdir(joinpath(PKG_JOGGER_PATH, ".git")) @test r2["git"] !== nothing From 25df07799b5635b39fd08e861394dce4b0b0d7ff Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Tue, 7 Sep 2021 15:31:03 -0400 Subject: [PATCH 2/3] feat: Add judge method to jogger Fixes Add Jogger version of judge #16 --- Project.toml | 1 + src/PkgJogger.jl | 1 + src/jogger.jl | 19 +++++++++++++++++++ src/utils.jl | 33 +++++++++++++++++++++++++++++++++ test/Project.toml | 1 + test/judging.jl | 23 +++++++++++++++++++++++ test/runtests.jl | 1 + test/smoke.jl | 5 ++++- test/utils.jl | 8 ++++++++ 9 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 test/judging.jl diff --git a/Project.toml b/Project.toml index e66792e..2fdc028 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,7 @@ LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] diff --git a/src/PkgJogger.jl b/src/PkgJogger.jl index a1f247a..fb9c75c 100644 --- a/src/PkgJogger.jl +++ b/src/PkgJogger.jl @@ -9,6 +9,7 @@ using Pkg using UUIDs using Dates using LibGit2 +using Statistics export @jog diff --git a/src/jogger.jl b/src/jogger.jl index d3c626b..7fd0437 100644 --- a/src/jogger.jl +++ b/src/jogger.jl @@ -138,6 +138,7 @@ macro jog(pkg) end """ + load_benchmarks(filename::String)::Dict load_benchmarks(uuid::String)::Dict load_benchmarks(uuid::UUID)::Dict @@ -145,10 +146,28 @@ macro jog(pkg) """ load_benchmarks(uuid::UUIDs.UUID) = load_benchmarks(string(uuid)) function load_benchmarks(uuid::AbstractString) + # Check if input is a filename + isfile(uuid) && return PkgJogger.load_benchmarks(uuid) + + # Check if a valid benchmark uuid path = joinpath(BENCHMARK_DIR, "trial", uuid * ".json.gz") @assert isfile(path) "Missing benchmarking results for $uuid, expected path: $path" PkgJogger.load_benchmarks(path) end + + """ + judge(new, old; metric=Statistics.median, kwargs...) + + Compares benchmarking results from `new` vs `old` for regressions/improvements + using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge` + + Identical to [`PkgJogger.judge`](@ref), but accepts UUIDs for `new` and `old` + """ + function judge(new, old; kwargs...) + PkgJogger.judge(load_benchmarks(new), _get_benchmarks(old); kwargs...) + end + _get_benchmarks(b::AbstractString) = load_benchmarks(b) + _get_benchmarks(b) = PkgJogger._get_benchmarks(b) end end end diff --git a/src/utils.jl b/src/utils.jl index d6dffa3..8639b72 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -47,3 +47,36 @@ function locate_benchmarks(dir) suite end locate_benchmarks(pkg::Module) = benchmark_dir(pkg) |> locate_benchmarks + + +""" + judge(new, old; metric=Statistics.median, kwargs...) + +Compares benchmarking results from `new` vs `old` for regressions/improvements +using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge` + +Effectively a convenience wrapper around `load_benchmarks` and `BenchmarkTools.judge` + +`new` and `old` can be any one of the following: + - Filename of benchmarking results saved by PkgJogger + - A `Dict` as returned by [`PkgJogger.load_benchmarks(filename)`](@ref) + - A `BenchmarkTools.BenchmarkGroup` with benchmarking results +""" +function judge( + new::BenchmarkTools.BenchmarkGroup, + old::BenchmarkTools.BenchmarkGroup; + metric = Statistics.median, + kwargs... +) + new_estimate = metric(new) + old_estimate = metric(old) + BenchmarkTools.judge(new_estimate, old_estimate; kwargs...) +end +function judge(new, old; kwargs...) + new_results = _get_benchmarks(new) + old_results = _get_benchmarks(old) + judge(new_results, old_results; kwargs...) +end + +_get_benchmarks(b::Dict) = b["benchmarks"]::BenchmarkTools.BenchmarkGroup +_get_benchmarks(filename::String) = load_benchmarks(filename) |> _get_benchmarks diff --git a/test/Project.toml b/test/Project.toml index 0dc759c..4cba04f 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/judging.jl b/test/judging.jl new file mode 100644 index 0000000..1aa9b10 --- /dev/null +++ b/test/judging.jl @@ -0,0 +1,23 @@ +using PkgJogger +using BenchmarkTools + +include("utils.jl") + +# Add Example +using Pkg +Pkg.develop(path="Example.jl/") +using Example +@jog Example + +# Run Benchmarks for testing +r1 = JogExample.benchmark() |> JogExample.save_benchmarks +r2 = JogExample.benchmark() |> JogExample.save_benchmarks + +# Get UUIDs +r1_uuid = get_uuid(r1) +r2_uuid = get_uuid(r2) + +@test typeof(PkgJogger.judge(r1, r2)) <: BenchmarkGroup +@testset "Test loading" for new=[r1, r1_uuid], old=[r2, r2_uuid] + @test typeof(JogExample.judge(new, old)) <: BenchmarkGroup +end diff --git a/test/runtests.jl b/test/runtests.jl index af7c606..38a2702 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,5 +13,6 @@ using PkgJogger # Run the rest of the unit testing suite @safetestset "Smoke Tests" begin include("smoke.jl") end + @safetestset "Judging" begin include("judging.jl") end @safetestset "CI Workflow" begin include("ci.jl") end end diff --git a/test/smoke.jl b/test/smoke.jl index 4d7c0ac..160f403 100644 --- a/test/smoke.jl +++ b/test/smoke.jl @@ -32,7 +32,7 @@ include("utils.jl") # Load with JogPkgJogger @testset "Jogger's load_benchmarks" begin - uuid = splitpath(file)[end] |> x -> split(x, ".")[1] + uuid = get_uuid(file) r3 = JogPkgJogger.load_benchmarks(uuid) r4 = JogPkgJogger.load_benchmarks(UUID(uuid)) @test r3 == r4 @@ -45,6 +45,9 @@ include("utils.jl") @test_throws AssertionError JogPkgJogger.load_benchmarks(UUIDs.uuid4()) end + # Test Judging + @test_nowarn JogPkgJogger.judge(file, file) + # If this is a git repo, there should be a git entry if isdir(joinpath(PKG_JOGGER_PATH, ".git")) @test r2["git"] !== nothing diff --git a/test/utils.jl b/test/utils.jl index d978c29..8d2de21 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -67,4 +67,12 @@ function test_subfile(parent, child) end end +""" + get_uuid(filename) + +Extract benchmark UUID from filename +""" +function get_uuid(filename) + splitpath(filename)[end] |> x -> split(x, ".")[1] +end From 6d8f956790c5d3b90ee822e55f14f88e771f0d31 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Tue, 7 Sep 2021 15:39:08 -0400 Subject: [PATCH 3/3] docs: Add judge to documentation --- docs/src/jogger.md | 1 + docs/src/reference.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/src/jogger.md b/docs/src/jogger.md index 03e4153..eaa5699 100644 --- a/docs/src/jogger.md +++ b/docs/src/jogger.md @@ -31,5 +31,6 @@ JogExample.warmup JogExample.run JogExample.save_benchmarks JogExample.load_benchmarks +JogExample.judge JogExample.BENCHMARK_DIR ``` diff --git a/docs/src/reference.md b/docs/src/reference.md index 5a0150f..cca6c52 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -3,6 +3,7 @@ ```@docs PkgJogger.benchmark_dir PkgJogger.locate_benchmarks +PkgJogger.judge ``` ## Internal