Merge branch 'jogger_rework'

docs/src/jogger.md

@@ -18,6 +18,71 @@ Similarly, `@jog AwesomePkg` would create a module named `JogAwesomePkg`:
 PkgJogger.@jog
 ```
 
+## Benchmark Directory Structure
+
+`PkgJogger` will recursively search the a package's `benchmark/` directory
+for benchmarking files `bench_*.jl` or directories `bench_*/`.
+
+For example, the following directory:
+
+```
+.
++-- Project.toml
++-- src/
+|   +-- PkgName.j;
+|   ...
++-- benchmark
+    +-- bench_matrix.jl # Will be included
+    ....
+    +-- subdir/     # ignored
+    +-- bench_ui/   # This gets added
+        +-- bench_foo.jl
+        ....
+```
+
+Results in a benchmarking suite of:
+```
+1-element BenchmarkTools.BenchmarkGroup:
+    "bench_matrix.jl" => Suite from "benchmark/bench_matrix.jl"
+    ... # Other benchmark/bench_*.jl files
+    "bench_ui" => BenchmarkTools.BenchmarkGroup:
+        "bench_foo.jl" => Suite from "benchmark/bench_ui/bench_foo.jl"
+        ... # Other benchmark/bench_ui/bench_*.jl files
+```
+
+## Benchmark Files
+
+`PkgJogger` expects the following structure for benchmarking files:
+
+```julia
+# PkgJogger will wrap this file into a module, thus it needs to declare all of
+# it's `using` and `import` statements.
+using BenchmarkTools
+using OtherPkg
+using AweseomePkg
+
+# PkgJogger assumes the first `suite` variable is the benchmark suite for this file
+suite = BenchmarkGroup()
+
+# This will add a benchmark "foo" to the benchmarking suite with a key of:
+# ["bench_filename.jl", "foo"]
+suite["foo"] = @benchmarkable ...
+
+# Further nesting within the file's `suite` is possible
+s = suite["baz"] = BenchmarkGroup()
+s["bar"] = @benchmarkable ... # Key of ["bench_filename.jl", "baz", "bar"]
+```
+
+In the example, we assume the benchmarking file is `benchmark/bench_filename.jl`.
+If it was located in a subdirectory `benchmark/bench_subdir` the resulting suite
+would have keys of `["bench_subdir", "bench_filename.jl", ...]`, instead of
+`["bench_filename.jl", ...]`. as shown.
+
+> A side effect of this structure is that each benchmarking file is self-contained
+> and independent of other benchmarking files. This means that if you want to
+> run the suite of a single file, you can `include` the file and run it with:
+> `tune!(suite); run(suite)`
+
 ## Jogger Reference
 
 Jogger modules provide helper methods for working with their package's

src/PkgJogger.jl

@@ -26,8 +26,8 @@ const JOGGER_PKGS = [
     PkgId(UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4"), "UUIDs"),
 ]
 
-include("jogger.jl")
 include("utils.jl")
+include("jogger.jl")
 include("ci.jl")
 
 end

src/ci.jl

@@ -27,14 +27,13 @@ function ci()
 
     # Run in sandbox
     pkgname = Symbol(pkg.name)
-    jogger = Symbol(:Jog, pkg.name)
     sandbox(pkg) do
         @eval Main begin
             using PkgJogger
             using $pkgname
             jogger = @jog $pkgname
-            result = $jogger.benchmark()
-            filename = $jogger.save_benchmarks(result)
+            result = jogger.benchmark()
+            filename = jogger.save_benchmarks(result)
             @info "Saved benchmarks to $filename"
         end
     end

src/jogger.jl

@@ -46,10 +46,11 @@ macro jog(pkg)
 
     # Generate modules
     suite_modules = Expr[]
-    benchmarks = Symbol[]
-    for (name, file) in locate_benchmarks(bench_dir)
-        push!(suite_modules, build_module(name, file))
-        push!(benchmarks, Symbol(name))
+    suite_expressions = Expr[]
+    for s in locate_benchmarks(bench_dir)
+        suite_expr, suite_module = build_module(s)
+        push!(suite_modules, suite_module)
+        push!(suite_expressions, suite_expr)
     end
 
     # Flatten out modules into a Vector{Expr}
@@ -83,9 +84,7 @@ macro jog(pkg)
             """
             function suite()
                 suite = BenchmarkTools.BenchmarkGroup()
-                for (n, m) in zip([$(string.(benchmarks)...)], [$(benchmarks...)])
-                    suite[n] = m.suite
-                end
+                $(suite_expressions...)
                 suite
             end
 
@@ -173,17 +172,24 @@ macro jog(pkg)
 end
 
 """
-    build_module(name, file)
+    build_module(s::BenchModule)
 
-Construct a module wrapping the BenchmarkGroup defined by `file` with `name`
+Construct a module wrapping the BenchmarkGroup defined by `s::BenchModule`
 """
-function build_module(name, file)
-    modname = Symbol(name)
-    exp = quote
+function build_module(s::BenchModule)
+    modname = gensym(s.name[end])
+    module_expr = quote
         module $modname
             __revise_mode__ = :eval
-            include($file)
+            include($(s.filename))
         end
-        Revise.track($modname, $file)
+        Revise.track($modname, $(s.filename))
     end
+
+    # Build Expression for accessing suite
+    suite_expr = quote
+        suite[$(s.name)] = $(modname).suite
+    end
+
+    return suite_expr, module_expr
 end

src/utils.jl

@@ -1,3 +1,8 @@
+struct BenchModule
+    filename::String
+    name::Vector{String}
+end
+
 """
     benchmark_dir(pkg::Module)
     benchmark_dir(pkg::PackageSpec)
@@ -32,23 +37,31 @@ end
 
 """
     locate_benchmarks(pkg::Module)
-    locate_benchmarks(bench_dir::String)
+    locate_benchmarks(path::String, name=String[])
 
-Returns a dict of `name => filename` of identified benchmark files
+Returns a list of `BenchModule` for identified benchmark files
 """
-function locate_benchmarks(dir)
-    suite = Dict{String, String}()
-    for file in readdir(dir; join=true)
-        m = match(r"bench_(.*?)\.jl$", file)
-        if m !== nothing
-            suite[m.captures[1]] = file
+function locate_benchmarks(path, name=String[])
+    suite = BenchModule[]
+    for file in readdir(path)
+        # Check that path is named 'bench_*'
+        !startswith(file, "bench_") && continue
+
+        # Check if file is a valid target to add
+        cur_name = [name..., file]
+        filename = joinpath(path, file)
+        if isfile(filename) && endswith(file, ".jl")
+            # File is a julia file named bench_*.jl
+            push!(suite, BenchModule(filename, cur_name))
+        elseif isdir(filename)
+            # Subdirectory named bench_* -> Look for more modules
+            append!(suite, locate_benchmarks(filename, cur_name))
         end
     end
-    suite
+    return suite
 end
 locate_benchmarks(pkg::Module) = benchmark_dir(pkg) |> locate_benchmarks
 
-
 """
     judge(new, old; metric=Statistics.median, kwargs...)
 

test/ci.jl

@@ -66,6 +66,6 @@ end
     results = run_ci_workflow(project)
 
     # Check timer results are decent (sleep isn't very accurate)
-    isapprox((time∘minimum)(results["benchmarks"][["timer", "1ms"]]), 1e6; atol=3e6)
-    isapprox((time∘minimum)(results["benchmarks"][["timer", "2ms"]]), 2e6; atol=3e6)
+    isapprox((time∘minimum)(results["benchmarks"][["bench_timer.jl", "1ms"]]), 1e6; atol=3e6)
+    isapprox((time∘minimum)(results["benchmarks"][["bench_timer.jl", "2ms"]]), 2e6; atol=3e6)
 end

test/locate_benchmarks.jl

@@ -0,0 +1,96 @@
+using Test
+using BenchmarkTools
+using PkgJogger
+using Example
+
+function check_suite(jogger; add=nothing)
+    # Default Suite
+    reference = [
+        ["bench_timer.jl", "1ms"],
+        ["bench_timer.jl", "2ms"],
+    ] |> Set
+
+    if add !== nothing
+        reference = union(reference, add)
+    end
+
+    # Get suite of jogger
+    suite = Set(jogger.suite() |> leaves |> x -> map(first, x))
+    @test suite == reference
+end
+
+function add_benchmark(pkg, path)
+
+    # Create Dummy Benchmark
+    filename = joinpath(PkgJogger.benchmark_dir(pkg), path)
+    dir = dirname(filename)
+    cleanup = isdir(dir) ? () -> rm(filename) : () -> rm(dir; recursive=true)
+    mkpath(dir)
+
+    open(filename, "w") do io
+        """
+        using BenchmarkTools
+
+        suite = BenchmarkGroup()
+        suite["foo"] = @benchmarkable sin(rand())
+        """ |> s -> write(io, s)
+    end
+
+    suite = Set([[splitpath(path)..., "foo"]])
+    return suite, cleanup
+end
+
+
+@testset "default suite" begin
+    jogger = @eval @jog Example
+    check_suite(jogger)
+end
+
+@testset "Add benchmarks" begin
+    suite, cleanup = add_benchmark(Example, "bench_foo_$(rand(UInt16)).jl")
+    jogger = @eval @jog Example
+    check_suite(jogger; add=suite)
+
+    # Add a non-benchmarking file (Should not be added)
+    cleanup2 = add_benchmark(Example, "foo_$(rand(UInt16)).jl")[2]
+    check_suite(jogger; add=suite)
+
+    # Add another file (should not be added)
+    suite3, cleanup3 = add_benchmark(Example, "bench_foo_$(rand(UInt16)).jl")
+    check_suite(jogger; add=suite)
+
+    # Regenerate jogger to get new suite -> Should now just be suite3 + suite
+    jogger = @eval @jog Example
+    check_suite(jogger; add=union(suite, suite3))
+
+    cleanup()
+    cleanup2()
+    cleanup3()
+end
+
+@testset "Benchmarks in subfolders" begin
+    # Add a subfolder -> Don't track
+    jogger = @eval @jog Example
+    tempdir = mktempdir(jogger.BENCHMARK_DIR; cleanup=false)
+    check_suite(jogger)
+    rm(tempdir)
+
+    # Add an empty bench_ subfolder -> Ignore
+    tempdir = mktempdir(jogger.BENCHMARK_DIR; prefix="bench_", cleanup=false)
+    check_suite(jogger)
+    rm(tempdir)
+
+    # Add a benchmark to a subfolder -> track
+    path = joinpath("bench_subdir_$(rand(UInt16))", "bench_foo_$(rand(UInt16)).jl")
+    suite, cleanup = add_benchmark(Example, path)
+    check_suite(@eval @jog Example; add=suite)
+    cleanup()
+
+    # Two Levels Deep
+    dir = "bench_subdir_$(rand(UInt16))"
+    suite, cleanup = add_benchmark(Example, joinpath(dir, "bench_foo_$(rand(UInt16)).jl"))
+    union!(suite, add_benchmark(Example, joinpath(dir, "bench_l2", "bench_foo_$(rand(UInt16)).jl"))[1])
+    add_benchmark(Example, joinpath(dir, "skip_me.jl"))
+    check_suite(@eval @jog Example; add=suite)
+    cleanup()
+end

test/runtests.jl

@@ -16,10 +16,10 @@ using PkgJogger
     if example_path ∉ LOAD_PATH
         push!(LOAD_PATH, example_path)
     end
-    using Example
 
     # Run the rest of the unit testing suite
     @safetestset "Smoke Tests" begin include("smoke.jl") end
     @safetestset "Judging" begin include("judging.jl") end
     @safetestset "CI Workflow" begin include("ci.jl") end
+    @safetestset "Locate Benchmarks" begin include("locate_benchmarks.jl") end
 end

test/utils.jl

@@ -1,6 +1,9 @@
 using BenchmarkTools
 using Test
 
+# Reduce Benchmarking Duration for faster testing
+BenchmarkTools.DEFAULT_PARAMETERS.seconds = 0.1
+
 const PKG_JOGGER_PATH = joinpath(@__DIR__, "..") |> abspath
 
 function test_loaded_results(r::Dict)