diff --git a/docs/src/reference.md b/docs/src/reference.md index cca6c52..e267a4e 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -4,6 +4,7 @@ PkgJogger.benchmark_dir PkgJogger.locate_benchmarks PkgJogger.judge +PkgJogger.tune! ``` ## Internal diff --git a/src/jogger.jl b/src/jogger.jl index cf52ad5..af959c5 100644 --- a/src/jogger.jl +++ b/src/jogger.jl @@ -88,15 +88,24 @@ macro jog(pkg) suite end + # Dispatch calls to tune! here so we can use the jogger variant of load_benchmarks + __tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...) = PkgJogger.tune!(group, ref; kwargs...) + __tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = PkgJogger.tune!(group, load_benchmarks(ref); kwargs...) + __tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = BenchmarkTools.tune!(group; kwargs...) + """ - benchmark(; verbose = false) + benchmark(; ref = nothing, verbose = false) Warmup, tune and run the benchmarking suite for $($pkg) + + To reuse prior tuning results set `ref` to a BenchmarkGroup or suitable identifier + for [`$($modname).load_benchmarks`](@ref). See [`PkgJogger.tune!`](@ref) for + more information about re-using tuning results. """ - function benchmark(; verbose = false) + function benchmark(; ref = nothing, verbose = false) s = suite() BenchmarkTools.warmup(s; verbose) - BenchmarkTools.tune!(s; verbose = verbose) + __tune!(s, ref; verbose = verbose) BenchmarkTools.run(s; verbose = verbose) end diff --git a/src/utils.jl b/src/utils.jl index d84905b..832c16b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -94,3 +94,37 @@ end _get_benchmarks(b::BenchmarkTools.BenchmarkGroup) = b _get_benchmarks(b::Dict) = b["benchmarks"]::BenchmarkTools.BenchmarkGroup _get_benchmarks(filename::String) = load_benchmarks(filename) |> _get_benchmarks + +""" + tune!(group::BenchmarkGroup, ref::BenchmarkGroup; verbose::Bool=false) + +Tunes a BenchmarkGroup, only tunning benchmarks not found in `ref`, otherwise reuse tuning +results from the reference BenchmarkGroup, by copying over all benchmark parameters from `ref`. + +This can reduce benchmarking runtimes significantly by only tuning new benchmarks. But does +ignore the following: + - Changes to benchmarking parameters (ie. memory_tolerance) between `group` and `ref` + - Significant changes in performance, such that re-tunning is warranted + - Other changes (ie. changing machines), such that re-tunning is warranted? +""" +function tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...) + ids = keys(group) |> collect + ref_ids = keys(ref) |> collect + + # Tune new benchmarks + for id in setdiff(ids, ref_ids) + tune!(group[id]; kwargs...) + end + + # Reuse tunning from prior benchmarks + for id in intersect(ids, ref_ids) + tune!(group[id], ref[id]; kwargs...) + end + + return group +end +tune!(b::BenchmarkTools.Benchmark, ref; kwargs...) = b.params = copy(ref.params) +tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = tune!(group; kwargs...) +tune!(group::BenchmarkTools.BenchmarkGroup; kwargs...) = BenchmarkTools.tune!(group; kwargs...) +tune!(group::BenchmarkTools.BenchmarkGroup, ref::Dict; kwargs...) = tune!(group, ref["benchmarks"]; kwargs...) +tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = tune!(group, load_benchmarks(ref); kwargs...) diff --git a/test/smoke.jl b/test/smoke.jl index d0a922e..3f6a6d5 100644 --- a/test/smoke.jl +++ b/test/smoke.jl @@ -46,6 +46,13 @@ include("utils.jl") @test_throws AssertionError JogExample.load_benchmarks(UUIDs.uuid4()) end + # Test Retuning + @testset "Reusing tune! results" begin + test_benchmark(JogPkgJogger.benchmark(ref = r), r) + test_benchmark(JogPkgJogger.benchmark(ref = get_uuid(file)), r) + test_benchmark(JogPkgJogger.benchmark(ref = file), r) + end + # Test Judging @test_nowarn JogExample.judge(file, file) diff --git a/test/utils.jl b/test/utils.jl index 15c6815..a948f6d 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -52,3 +52,20 @@ function get_uuid(filename) splitpath(filename)[end] |> x -> split(x, ".")[1] end +""" + test_benchmark(target::BenchmarkGroup, ref) + +Checks that target and ref are from equivalent benchmarking suite +""" +function test_benchmark(target, ref::BenchmarkGroup) + @test typeof(target) <: BenchmarkGroup + @test isempty(setdiff(keys(target), keys(ref))) + map(test_benchmark, target, ref) +end +test_benchmark(target, ref) = @test typeof(target) <: typeof(ref) +function test_benchmark(target, ref::BenchmarkTools.Trial) + @test typeof(target) <: BenchmarkTools.Trial + @test params(target) == params(ref) +end + +