From d345569005050a0d21639e6d8584dee53c168ca4 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Wed, 29 Sep 2021 10:07:11 -0400 Subject: [PATCH 1/7] feat: Add support for benchmarks in sub folders Can now have benchmarks in subfolders ie. - benchmarks/ - bench_foo.jl - bench_group/ - bench_foo2.jl Reworked jogger suite() to use generated code and gensym names Fixes: #19, sets up option to define suite module in `BenchModule` for #5 --- src/PkgJogger.jl | 2 +- src/jogger.jl | 34 ++++++++++++++++++++-------------- src/utils.jl | 34 ++++++++++++++++++++++++---------- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/PkgJogger.jl b/src/PkgJogger.jl index fb9c75c..ce49be8 100644 --- a/src/PkgJogger.jl +++ b/src/PkgJogger.jl @@ -26,8 +26,8 @@ const JOGGER_PKGS = [ PkgId(UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4"), "UUIDs"), ] -include("jogger.jl") include("utils.jl") +include("jogger.jl") include("ci.jl") end diff --git a/src/jogger.jl b/src/jogger.jl index fd4405a..cf52ad5 100644 --- a/src/jogger.jl +++ b/src/jogger.jl @@ -46,10 +46,11 @@ macro jog(pkg) # Generate modules suite_modules = Expr[] - benchmarks = Symbol[] - for (name, file) in locate_benchmarks(bench_dir) - push!(suite_modules, build_module(name, file)) - push!(benchmarks, Symbol(name)) + suite_expressions = Expr[] + for s in locate_benchmarks(bench_dir) + suite_expr, suite_module = build_module(s) + push!(suite_modules, suite_module) + push!(suite_expressions, suite_expr) end # Flatten out modules into a Vector{Expr} @@ -83,9 +84,7 @@ macro jog(pkg) """ function suite() suite = BenchmarkTools.BenchmarkGroup() - for (n, m) in zip([$(string.(benchmarks)...)], [$(benchmarks...)]) - suite[n] = m.suite - end + $(suite_expressions...) suite end @@ -173,17 +172,24 @@ macro jog(pkg) end """ - build_module(name, file) + build_module(s::BenchModule) -Construct a module wrapping the BenchmarkGroup defined by `file` with `name` +Construct a module wrapping the BenchmarkGroup defined by `s::BenchModule` """ -function build_module(name, file) - modname = Symbol(name) - exp = quote +function build_module(s::BenchModule) + modname = gensym(s.name[end]) + module_expr = quote module $modname __revise_mode__ = :eval - include($file) + include($(s.filename)) end - Revise.track($modname, $file) + Revise.track($modname, $(s.filename)) end + + # Build Expression for accessing suite + suite_expr = quote + suite[$(s.name)] = $(modname).suite + end + + return suite_expr, module_expr end diff --git a/src/utils.jl b/src/utils.jl index 53063b6..ed55f32 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,3 +1,8 @@ +struct BenchModule + filename::String + name::Vector{String} +end + """ benchmark_dir(pkg::Module) benchmark_dir(pkg::PackageSpec) @@ -32,23 +37,32 @@ end """ locate_benchmarks(pkg::Module) - locate_benchmarks(bench_dir::String) + locate_benchmarks(path::String, name=String[]) -Returns a dict of `name => filename` of identified benchmark files +Returns a list of `BenchModule` for identified benchmark files """ -function locate_benchmarks(dir) - suite = Dict{String, String}() - for file in readdir(dir; join=true) - m = match(r"bench_(.*?)\.jl$", file) - if m !== nothing - suite[m.captures[1]] = file +function locate_benchmarks(path, name=String[]) + suite = BenchModule[] + for file in readdir(path) + # Check that path is named 'bench_*' + !startswith(file, "bench_") && continue + + # Check if file is a valid target to add + cur_name = [name..., file] + filename = joinpath(path, file) + if isfile(filename) && endswith(file, ".jl") + # File is a julia file named bench_*.jl + push!(suite, BenchModule(filename, cur_name)) + elseif isdir(filename) + # Subdirectory named bench_* -> Look for more modules + append!(suite, locate_benchmarks(filename, cur_name)) end end - suite + @info suite + return suite end locate_benchmarks(pkg::Module) = benchmark_dir(pkg) |> locate_benchmarks - """ judge(new, old; metric=Statistics.median, kwargs...) From 964fa8729625bb0dd21c0125a6867a55d4e0759f Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Wed, 29 Sep 2021 13:23:13 -0400 Subject: [PATCH 2/7] fix: remove extraneous @info --- src/utils.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils.jl b/src/utils.jl index ed55f32..d84905b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -58,7 +58,6 @@ function locate_benchmarks(path, name=String[]) append!(suite, locate_benchmarks(filename, cur_name)) end end - @info suite return suite end locate_benchmarks(pkg::Module) = benchmark_dir(pkg) |> locate_benchmarks From e95c89b8e32a3b55b3b979f5e850233bcd2561b7 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Thu, 4 Nov 2021 11:34:46 -0400 Subject: [PATCH 3/7] fix: update tests to nameing change --- test/ci.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ci.jl b/test/ci.jl index 68d16e8..ad391c4 100644 --- a/test/ci.jl +++ b/test/ci.jl @@ -66,6 +66,6 @@ end results = run_ci_workflow(project) # Check timer results are decent (sleep isn't very accurate) - isapprox((time∘minimum)(results["benchmarks"][["timer", "1ms"]]), 1e6; atol=3e6) - isapprox((time∘minimum)(results["benchmarks"][["timer", "2ms"]]), 2e6; atol=3e6) + isapprox((time∘minimum)(results["benchmarks"][["bench_timer.jl", "1ms"]]), 1e6; atol=3e6) + isapprox((time∘minimum)(results["benchmarks"][["bench_timer.jl", "2ms"]]), 2e6; atol=3e6) end From 356e695cee36e2f783cb0f0612fa281d68b84973 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Thu, 4 Nov 2021 11:36:55 -0400 Subject: [PATCH 4/7] testing: Reduce default benchmarking time Cuts unit testing time in half (110s to 51s) --- test/utils.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/utils.jl b/test/utils.jl index 97e2323..15c6815 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,6 +1,9 @@ using BenchmarkTools using Test +# Reduce Benchmarking Duration for faster testing +BenchmarkTools.DEFAULT_PARAMETERS.seconds = 0.1 + const PKG_JOGGER_PATH = joinpath(@__DIR__, "..") |> abspath function test_loaded_results(r::Dict) From 3e3c787ba3ac90ea3b1187009cebdde7ec1dd115 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Thu, 4 Nov 2021 14:34:03 -0400 Subject: [PATCH 5/7] refactor: Use jogger instead of interpolating name Does the same thing with less macro-foo --- src/ci.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ci.jl b/src/ci.jl index c63d2e4..56eee7c 100644 --- a/src/ci.jl +++ b/src/ci.jl @@ -27,14 +27,13 @@ function ci() # Run in sandbox pkgname = Symbol(pkg.name) - jogger = Symbol(:Jog, pkg.name) sandbox(pkg) do @eval Main begin using PkgJogger using $pkgname jogger = @jog $pkgname - result = $jogger.benchmark() - filename = $jogger.save_benchmarks(result) + result = jogger.benchmark() + filename = jogger.save_benchmarks(result) @info "Saved benchmarks to $filename" end end From 7eb510621715e2212d497a606277ee432bf8bcea Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Sat, 6 Nov 2021 12:35:47 -0400 Subject: [PATCH 6/7] test: Check that benchmarking files get found Tests adding `bench_*.jl` and `bench_*/` as well as non-bench files to the benchmarking directory --- test/locate_benchmarks.jl | 96 +++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 test/locate_benchmarks.jl diff --git a/test/locate_benchmarks.jl b/test/locate_benchmarks.jl new file mode 100644 index 0000000..28b789a --- /dev/null +++ b/test/locate_benchmarks.jl @@ -0,0 +1,96 @@ +using Test +using BenchmarkTools +using PkgJogger +using Example + +function check_suite(jogger; add=nothing) + # Default Suite + reference = [ + ["bench_timer.jl", "1ms"], + ["bench_timer.jl", "2ms"], + ] |> Set + + if add !== nothing + reference = union(reference, add) + end + + # Get suite of jogger + suite = Set(jogger.suite() |> leaves |> x -> map(first, x)) + @test suite == reference +end + +function add_benchmark(pkg, path) + + # Create Dummy Benchmark + filename = joinpath(PkgJogger.benchmark_dir(pkg), path) + dir = dirname(filename) + cleanup = isdir(dir) ? () -> rm(filename) : () -> rm(dir; recursive=true) + mkpath(dir) + + open(filename, "w") do io + """ + using BenchmarkTools + + suite = BenchmarkGroup() + suite["foo"] = @benchmarkable sin(rand()) + """ |> s -> write(io, s) + end + + suite = Set([[splitpath(path)..., "foo"]]) + return suite, cleanup +end + + +@testset "default suite" begin + jogger = @eval @jog Example + check_suite(jogger) +end + +@testset "Add benchmarks" begin + suite, cleanup = add_benchmark(Example, "bench_foo_$(rand(UInt16)).jl") + jogger = @eval @jog Example + check_suite(jogger; add=suite) + + # Add a non-benchmarking file (Should not be added) + cleanup2 = add_benchmark(Example, "foo_$(rand(UInt16)).jl")[2] + check_suite(jogger; add=suite) + + # Add another file (should not be added) + suite3, cleanup3 = add_benchmark(Example, "bench_foo_$(rand(UInt16)).jl") + check_suite(jogger; add=suite) + + # Regenerate jogger to get new suite -> Should now just be suite3 + suite + jogger = @eval @jog Example + check_suite(jogger; add=union(suite, suite3)) + + cleanup() + cleanup2() + cleanup3() +end + +@testset "Benchmarks in subfolders" begin + # Add a subfolder -> Don't track + jogger = @eval @jog Example + tempdir = mktempdir(jogger.BENCHMARK_DIR; cleanup=false) + check_suite(jogger) + rm(tempdir) + + # Add an empty bench_ subfolder -> Ignore + tempdir = mktempdir(jogger.BENCHMARK_DIR; prefix="bench_", cleanup=false) + check_suite(jogger) + rm(tempdir) + + # Add a benchmark to a subfolder -> track + path = joinpath("bench_subdir_$(rand(UInt16))", "bench_foo_$(rand(UInt16)).jl") + suite, cleanup = add_benchmark(Example, path) + check_suite(@eval @jog Example; add=suite) + cleanup() + + # Two Levels Deep + dir = "bench_subdir_$(rand(UInt16))" + suite, cleanup = add_benchmark(Example, joinpath(dir, "bench_foo_$(rand(UInt16)).jl")) + union!(suite, add_benchmark(Example, joinpath(dir, "bench_l2", "bench_foo_$(rand(UInt16)).jl"))[1]) + add_benchmark(Example, joinpath(dir, "skip_me.jl")) + check_suite(@eval @jog Example; add=suite) + cleanup() +end diff --git a/test/runtests.jl b/test/runtests.jl index 2bfeff6..d05df34 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -16,10 +16,10 @@ using PkgJogger if example_path ∉ LOAD_PATH push!(LOAD_PATH, example_path) end - using Example # Run the rest of the unit testing suite @safetestset "Smoke Tests" begin include("smoke.jl") end @safetestset "Judging" begin include("judging.jl") end @safetestset "CI Workflow" begin include("ci.jl") end + @safetestset "Locate Benchmarks" begin include("locate_benchmarks.jl") end end From 8ef0c18a2a693804b234756cf7cee67eb235799f Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Sat, 6 Nov 2021 13:41:00 -0400 Subject: [PATCH 7/7] doc: Benchmark directory and file structure Expand docs on the expected structure of the directory and individual benchmarking files --- docs/src/jogger.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/docs/src/jogger.md b/docs/src/jogger.md index e202883..8661e6a 100644 --- a/docs/src/jogger.md +++ b/docs/src/jogger.md @@ -18,6 +18,71 @@ Similarly, `@jog AwesomePkg` would create a module named `JogAwesomePkg`: PkgJogger.@jog ``` +## Benchmark Directory Structure + +`PkgJogger` will recursively search the a package's `benchmark/` directory +for benchmarking files `bench_*.jl` or directories `bench_*/`. + +For example, the following directory: + +``` +. ++-- Project.toml ++-- src/ +| +-- PkgName.j; +| ... ++-- benchmark + +-- bench_matrix.jl # Will be included + .... + +-- subdir/ # ignored + +-- bench_ui/ # This gets added + +-- bench_foo.jl + .... +``` + +Results in a benchmarking suite of: +``` +1-element BenchmarkTools.BenchmarkGroup: + "bench_matrix.jl" => Suite from "benchmark/bench_matrix.jl" + ... # Other benchmark/bench_*.jl files + "bench_ui" => BenchmarkTools.BenchmarkGroup: + "bench_foo.jl" => Suite from "benchmark/bench_ui/bench_foo.jl" + ... # Other benchmark/bench_ui/bench_*.jl files +``` + +## Benchmark Files + +`PkgJogger` expects the following structure for benchmarking files: + +```julia +# PkgJogger will wrap this file into a module, thus it needs to declare all of +# it's `using` and `import` statements. +using BenchmarkTools +using OtherPkg +using AweseomePkg + +# PkgJogger assumes the first `suite` variable is the benchmark suite for this file +suite = BenchmarkGroup() + +# This will add a benchmark "foo" to the benchmarking suite with a key of: +# ["bench_filename.jl", "foo"] +suite["foo"] = @benchmarkable ... + +# Further nesting within the file's `suite` is possible +s = suite["baz"] = BenchmarkGroup() +s["bar"] = @benchmarkable ... # Key of ["bench_filename.jl", "baz", "bar"] +``` + +In the example, we assume the benchmarking file is `benchmark/bench_filename.jl`. +If it was located in a subdirectory `benchmark/bench_subdir` the resulting suite +would have keys of `["bench_subdir", "bench_filename.jl", ...]`, instead of +`["bench_filename.jl", ...]`. as shown. + +> A side effect of this structure is that each benchmarking file is self-contained +> and independent of other benchmarking files. This means that if you want to +> run the suite of a single file, you can `include` the file and run it with: +> `tune!(suite); run(suite)` + ## Jogger Reference Jogger modules provide helper methods for working with their package's