From d345569005050a0d21639e6d8584dee53c168ca4 Mon Sep 17 00:00:00 2001
From: Alexius Wadell <awadell@gmail.com>
Date: Wed, 29 Sep 2021 10:07:11 -0400
Subject: [PATCH] feat: Add support for benchmarks in sub folders

Can now have benchmarks in subfolders ie.
- benchmarks/
 - bench_foo.jl
 - bench_group/
   - bench_foo2.jl

  Reworked jogger suite() to use generated code and gensym names

  Fixes: #19, sets up option to define suite module in `BenchModule`
  for #5
---
 src/PkgJogger.jl |  2 +-
 src/jogger.jl    | 34 ++++++++++++++++++++--------------
 src/utils.jl     | 34 ++++++++++++++++++++++++----------
 3 files changed, 45 insertions(+), 25 deletions(-)

diff --git a/src/PkgJogger.jl b/src/PkgJogger.jl
index fb9c75c..ce49be8 100644
--- a/src/PkgJogger.jl
+++ b/src/PkgJogger.jl
@@ -26,8 +26,8 @@ const JOGGER_PKGS = [
     PkgId(UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4"), "UUIDs"),
 ]
 
-include("jogger.jl")
 include("utils.jl")
+include("jogger.jl")
 include("ci.jl")
 
 end
diff --git a/src/jogger.jl b/src/jogger.jl
index fd4405a..cf52ad5 100644
--- a/src/jogger.jl
+++ b/src/jogger.jl
@@ -46,10 +46,11 @@ macro jog(pkg)
 
     # Generate modules
     suite_modules = Expr[]
-    benchmarks = Symbol[]
-    for (name, file) in locate_benchmarks(bench_dir)
-        push!(suite_modules, build_module(name, file))
-        push!(benchmarks, Symbol(name))
+    suite_expressions = Expr[]
+    for s in locate_benchmarks(bench_dir)
+        suite_expr, suite_module = build_module(s)
+        push!(suite_modules, suite_module)
+        push!(suite_expressions, suite_expr)
     end
 
     # Flatten out modules into a Vector{Expr}
@@ -83,9 +84,7 @@ macro jog(pkg)
             """
             function suite()
                 suite = BenchmarkTools.BenchmarkGroup()
-                for (n, m) in zip([$(string.(benchmarks)...)], [$(benchmarks...)])
-                    suite[n] = m.suite
-                end
+                $(suite_expressions...)
                 suite
             end
 
@@ -173,17 +172,24 @@ macro jog(pkg)
 end
 
 """
-    build_module(name, file)
+    build_module(s::BenchModule)
 
-Construct a module wrapping the BenchmarkGroup defined by `file` with `name`
+Construct a module wrapping the BenchmarkGroup defined by `s::BenchModule`
 """
-function build_module(name, file)
-    modname = Symbol(name)
-    exp = quote
+function build_module(s::BenchModule)
+    modname = gensym(s.name[end])
+    module_expr = quote
         module $modname
             __revise_mode__ = :eval
-            include($file)
+            include($(s.filename))
         end
-        Revise.track($modname, $file)
+        Revise.track($modname, $(s.filename))
     end
+
+    # Build Expression for accessing suite
+    suite_expr = quote
+        suite[$(s.name)] = $(modname).suite
+    end
+
+    return suite_expr, module_expr
 end
diff --git a/src/utils.jl b/src/utils.jl
index 53063b6..ed55f32 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -1,3 +1,8 @@
+struct BenchModule
+    filename::String
+    name::Vector{String}
+end
+
 """
     benchmark_dir(pkg::Module)
     benchmark_dir(pkg::PackageSpec)
@@ -32,23 +37,32 @@ end
 
 """
     locate_benchmarks(pkg::Module)
-    locate_benchmarks(bench_dir::String)
+    locate_benchmarks(path::String, name=String[])
 
-Returns a dict of `name => filename` of identified benchmark files
+Returns a list of `BenchModule` for identified benchmark files
 """
-function locate_benchmarks(dir)
-    suite = Dict{String, String}()
-    for file in readdir(dir; join=true)
-        m = match(r"bench_(.*?)\.jl$", file)
-        if m !== nothing
-            suite[m.captures[1]] = file
+function locate_benchmarks(path, name=String[])
+    suite = BenchModule[]
+    for file in readdir(path)
+        # Check that path is named 'bench_*'
+        !startswith(file, "bench_") && continue
+
+        # Check if file is a valid target to add
+        cur_name = [name..., file]
+        filename = joinpath(path, file)
+        if isfile(filename) && endswith(file, ".jl")
+            # File is a julia file named bench_*.jl
+            push!(suite, BenchModule(filename, cur_name))
+        elseif isdir(filename)
+            # Subdirectory named bench_* -> Look for more modules
+            append!(suite, locate_benchmarks(filename, cur_name))
         end
     end
-    suite
+    @info suite
+    return suite
 end
 locate_benchmarks(pkg::Module) = benchmark_dir(pkg) |> locate_benchmarks
 
-
 """
     judge(new, old; metric=Statistics.median, kwargs...)