diff --git a/docs/src/jogger.md b/docs/src/jogger.md index 81a6d41..0856c0e 100644 --- a/docs/src/jogger.md +++ b/docs/src/jogger.md @@ -88,8 +88,11 @@ Often it's useful to run only a subset of the full benchmarking suite. For example, to run only the benchmarks within `benchmark/bench_filename.jl`: `JogExample.run("bench_filename")`. -Most jogger functions support filtering ([`judge`](@ref JogExample.judge) is a notable exception). -Greater support is planned. +```julia +using PkgJogger, Example +@jog Example +JogExample.run("bench_filename.jl") +``` !!! compat PkgJogger 0.6.0 Support for filtering via [`JogExample.suite`](@ref) was added in v0.6.0 diff --git a/src/jogger.jl b/src/jogger.jl index 8b4fcec..dd658b4 100644 --- a/src/jogger.jl +++ b/src/jogger.jl @@ -214,11 +214,14 @@ macro jog(pkg) load_benchmarks(id) = PkgJogger.load_benchmarks(joinpath(BENCHMARK_DIR, "trial"), id) """ - judge(new, old; metric=Statistics.median, kwargs...) + judge(new, old, [select...]; metric=Statistics.median, kwargs...) Compares benchmarking results from `new` vs `old` for regressions/improvements using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge` + Optionally, filter results using `select...`, see [`$($mod_str).suite`](@ref) for + details. + Identical to [`PkgJogger.judge`](@ref), but accepts any identifier supported by [`$($mod_str).load_benchmarks`](@ref) @@ -230,6 +233,11 @@ macro jog(pkg) [...] ``` + ```julia + # Only judge results in `bench_foo.jl` + $($mod_str).judge(:latest, :oldest, "bench_foo.jl") + ``` + ```julia # Judge results by UUID $($mod_str).judge("$(UUIDs.uuid4())", "$(UUIDs.uuid4())") @@ -242,8 +250,8 @@ macro jog(pkg) [...] ``` """ - function judge(new, old; kwargs...) - PkgJogger.judge(_get_benchmarks(new), _get_benchmarks(old); kwargs...) + function judge(new, old, select...; kwargs...) + PkgJogger.judge(_get_benchmarks(new), _get_benchmarks(old), select...; kwargs...) end _get_benchmarks(b) = load_benchmarks(b) _get_benchmarks(b::Dict) = PkgJogger._get_benchmarks(b) diff --git a/src/utils.jl b/src/utils.jl index 27baec5..bb4fe23 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -117,9 +117,9 @@ function judge( old_estimate = metric(old) BenchmarkTools.judge(new_estimate, old_estimate; kwargs...) end -function judge(new, old; kwargs...) - new_results = _get_benchmarks(new) - old_results = _get_benchmarks(old) +function judge(new, old, select...; kwargs...) + new_results = getsuite(_get_benchmarks(new), select...) + old_results = getsuite(_get_benchmarks(old), select...) judge(new_results, old_results; kwargs...) end diff --git a/test/judge_tests.jl b/test/judge_tests.jl index 647f774..1c436c3 100644 --- a/test/judge_tests.jl +++ b/test/judge_tests.jl @@ -12,9 +12,9 @@ function gen_example() return results, filename, dict, uuid end -function test_judge(f, new, old) - @inferred f(new, old) - judgement = f(new, old) +function test_judge(f, new, old, args...) + @inferred f(new, old, args...) + judgement = f(new, old, args...) @test typeof(judgement) <: BenchmarkGroup return judgement end @@ -25,6 +25,7 @@ old = gen_example() @testset "JogPkgName.judge($(typeof(n)), $(typeof(o)))" for (n, o) in Iterators.product(new, old) test_judge(jogger.judge, n, o) + test_judge(jogger.judge, n, o, :, "1ms") end @testset "PkgJogger.judge($(typeof(n)), $(typeof(o)))" for (n, o) in Iterators.product(new[1:3], old[1:3])