Merge branch 'cleanup'

README.md

@@ -3,3 +3,17 @@
 [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://awadell1.github.io/PkgJogger.jl/stable)
 [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://awadell1.github.io/PkgJogger.jl/dev)
 [![Build Status](https://github.com/awadell1/PkgJogger.jl/workflows/CI/badge.svg)](https://github.com/awadell1/PkgJogger.jl/actions)
+
+PkgJogger is a benchmarking framework for Julia with the following features:
+
+- Just write benchmarks files:`benchmark/bench_*.jl`
+
+    PkgJogger will wrap each benchmark file into a separate module, and return
+    a top level module with helper methods for running the suite
+
+    Individual benchmark files only need to define a `suite::BenchmarkGroup`
+
+- Revise, benchmark, and revise again
+
+    PkgJogger uses [Revise.jl](https://github.com/timholy/Revise.jl) to track changes to benchmarking files and updates
+    the suite as you edit. No more waiting your benchmarks to precompile!

docs/src/index.md

@@ -4,11 +4,50 @@ CurrentModule = PkgJogger
 
 # PkgJogger
 
-Documentation for [PkgJogger](https://github.com/awadell1/PkgJogger.jl).
+*A Benchmarking Framework for Julia*
 
-```@index
+PkgJogger makes benchmarking easy by providing a framework for running [BenchmarkTool.jl](https://github.com/JuliaCI/BenchmarkTools.jl) benchmarks, while handling all the boilerplate setup.
+
+## Just write benchmarks
+
+Create a `benchmark/bench_*.jl` file, define a `suite` and go!
+
+```julia
+using Benchmark
+using AwesomePkg
+suite = BenchmarkGroup()
+suite["fast"] = @benchmarkable fast_code()
 ```
 
-```@autodocs
-Modules = [PkgJogger]
+PkgJogger will wrap each `benchmark/bench_*.jl` in it's own module and bundle them into `JogAwesomePkg`
+
+```julia
+using AwesomePkg
+using PkgJogger
+
+# Creates the JogAwesomePkg module
+@jog AwesomePkg
+
+# Warmup, tune and run all of AwesomePkg's benchmarks
+JogAwesomePkg.benchmark()
 ```
+
+## Benchmark, Revise and Benchmark Again!
+
+PkgJogger uses [Revise.jl](https://github.com/timholy/Revise.jl) to track
+changes to your `benchmark/bench_*.jl` files and reload your suite as you edit.
+No more waiting for your benchmarks to precompile!
+
+Tracked Changes:
+
+- Changing your benchmarked function
+- Changing benchmarking parameters (ie. `seconds` or `samples`)
+- Adding new benchmarks
+
+Current Limitations:
+
+- New benchmark files are not tracked
+- Deleted benchmarks will stick around
+- Renamed benchmarks will create a new benchmark, and retain the old name
+
+To get around the above, run `@jog PkgName` to get a updated jogger.

src/PkgJogger.jl

@@ -5,112 +5,7 @@ using BenchmarkTools
 
 export @jog
 
-macro jog(pkg)
-    # Module Name
-    modname = Symbol(:Jog, pkg)
-
-    # Locate benchmark folder
-    bench_dir = @eval benchmark_dir($pkg)
-
-    # Generate modules
-    suite_modules = Expr[]
-    benchmarks = Symbol[]
-    for (name, file) in locate_benchmarks(bench_dir)
-        push!(suite_modules, build_module(name, file))
-        push!(benchmarks, Symbol(name))
-    end
-
-    # Dispatch to PkgJogger functions
-    dispatch_funcs = Expr[]
-    for (m, f) in [(:BenchmarkTools, :run), (:BenchmarkTools, :warmup), (:PkgJogger, :benchmark)]
-        exp = quote
-            $f(args...; kwargs...) = $(m).$(f)(suite(), args...; kwargs...)
-        end
-        push!(dispatch_funcs, exp)
-    end
-
-    # Flatten out modules into a Vector{Expr}
-    suite_exp = getfield(MacroTools.flatten(quote $(suite_modules...) end), :args)
-
-    # Generate Module for Jogging pkg
-    quote
-        @eval module $modname
-            using $pkg
-            using BenchmarkTools
-            using PkgJogger
-            using Revise
-
-            # Set Revise Mode and put submodules here
-            __revise_mode__ = :eval
-            $(suite_exp...)
-
-            """
-                suite()
-
-            Gets the BenchmarkTools suite for $($pkg)
-            """
-            function suite()
-                suite = BenchmarkGroup()
-                for (n, m) in zip([$(string.(benchmarks)...)], [$(benchmarks...)])
-                    suite[n] = m.suite
-                end
-                suite
-            end
-
-            $(dispatch_funcs...)
-        end
-    end
-end
-
-"""
-    benchmark_dir(pkg)
-
-Expected location of benchmarks for `pkg`
-"""
-function benchmark_dir(pkg::Module)
-    pkg_dir = joinpath(dirname(pathof(pkg)), "..")
-    joinpath(pkg_dir, "benchmark") |> abspath
-end
-
-
-function locate_benchmarks(dir)
-    suite = Dict{String, String}()
-    for file in readdir(dir; join=true)
-        m = match(r"bench_(.*?)\.jl$", file)
-        if m !== nothing
-            suite[m.captures[1]] = file
-        end
-    end
-    suite
-end
-
-"""
-    build_module(name, file)
-
-Construct a module wrapping the BenchmarkGroup defined by `file` with `name`
-"""
-function build_module(name, file)
-    modname = Symbol(name)
-    exp = quote
-        module $modname
-            __revise_mode__ = :eval
-            include($file)
-        end
-        Revise.track($modname, $file)
-    end
-end
-
-
-"""
-    benchmark(s::BenchmarkGroup)
-
-Warmup, tune and run a benchmark suite
-"""
-function benchmark(s::BenchmarkTools.BenchmarkGroup; kwargs...)
-    warmup(s)
-    tune!(s)
-    BenchmarkTools.run(s)
-end
-
+include("dispatch.jl")
+include("jogger.jl")
 
 end

src/dispatch.jl

@@ -0,0 +1,19 @@
+# This file contains functions that the generated modules will dispatch to
+
+# List of Module => function that `JogPkgName` will dispatch to
+const DISPATCH_METHODS = [
+    :BenchmarkTools => :run,
+    :BenchmarkTools => :warmup,
+    :PkgJogger => :benchmark
+]
+
+"""
+    benchmark(s::BenchmarkGroup)
+
+Warmup, tune and run a benchmark suite
+"""
+function benchmark(s::BenchmarkTools.BenchmarkGroup; kwargs...)
+    warmup(s)
+    tune!(s)
+    BenchmarkTools.run(s)
+end

src/jogger.jl

@@ -0,0 +1,106 @@
+# This file contains functions related to building the JogPkgName module
+
+"""
+    @jog PkgName
+
+Creates a module named `JogPkgName` of benchmarks for `PkgName` pulled from
+`PKG_DIR/benchmark/bench_*.jl`
+
+Methods:
+ - `suite()`        Return a `BenchmarkGroup` of the benchmarks for `PkgName`
+ - `benchmark()`    Warmup, tune and run the suite
+"""
+macro jog(pkg)
+    # Module Name
+    modname = Symbol(:Jog, pkg)
+
+    # Locate benchmark folder
+    bench_dir = @eval benchmark_dir($pkg)
+
+    # Generate modules
+    suite_modules = Expr[]
+    benchmarks = Symbol[]
+    for (name, file) in locate_benchmarks(bench_dir)
+        push!(suite_modules, build_module(name, file))
+        push!(benchmarks, Symbol(name))
+    end
+
+    # Dispatch to PkgJogger functions
+    dispatch_funcs = Expr[]
+    for (m, f) in DISPATCH_METHODS
+        exp = quote
+            $f(args...; kwargs...) = $(m).$(f)(suite(), args...; kwargs...)
+        end
+        push!(dispatch_funcs, exp)
+    end
+
+    # Flatten out modules into a Vector{Expr}
+    suite_exp = getfield(MacroTools.flatten(quote $(suite_modules...) end), :args)
+
+    # Generate Module for Jogging pkg
+    quote
+        @eval module $modname
+            using $pkg
+            using BenchmarkTools
+            using PkgJogger
+            using Revise
+
+            # Set Revise Mode and put submodules here
+            __revise_mode__ = :eval
+            $(suite_exp...)
+
+            """
+                suite()
+
+            Gets the BenchmarkTools suite for $($pkg)
+            """
+            function suite()
+                suite = BenchmarkGroup()
+                for (n, m) in zip([$(string.(benchmarks)...)], [$(benchmarks...)])
+                    suite[n] = m.suite
+                end
+                suite
+            end
+
+            $(dispatch_funcs...)
+        end
+    end
+end
+
+"""
+    benchmark_dir(pkg)
+
+Expected location of benchmarks for `pkg`
+"""
+function benchmark_dir(pkg::Module)
+    pkg_dir = joinpath(dirname(pathof(pkg)), "..")
+    joinpath(pkg_dir, "benchmark") |> abspath
+end
+
+
+function locate_benchmarks(dir)
+    suite = Dict{String, String}()
+    for file in readdir(dir; join=true)
+        m = match(r"bench_(.*?)\.jl$", file)
+        if m !== nothing
+            suite[m.captures[1]] = file
+        end
+    end
+    suite
+end
+
+"""
+    build_module(name, file)
+
+Construct a module wrapping the BenchmarkGroup defined by `file` with `name`
+"""
+function build_module(name, file)
+    modname = Symbol(name)
+    exp = quote
+        module $modname
+            __revise_mode__ = :eval
+            include($file)
+        end
+        Revise.track($modname, $file)
+    end
+end

test/smoke.jl

@@ -19,3 +19,12 @@ benchmark_dir = PkgJogger.benchmark_dir(PkgJogger)
     r = JogPkgJogger.run()
     @test typeof(r) <: BenchmarkTools.BenchmarkGroup
 end
+
+@testset "Jogger Methods" begin
+    @jog PkgJogger
+    @test @isdefined JogPkgJogger
+
+    @testset "JogPkgJogger.$f" for (m, f) in PkgJogger.DISPATCH_METHODS
+        @test isdefined(JogPkgJogger, f)
+    end
+end