From c9bdfe7426a23e31dfcc621dd941ae99b4a9af71 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Wed, 11 Aug 2021 11:35:34 -0400 Subject: [PATCH 1/4] refactor: Split code into files --- src/PkgJogger.jl | 109 +---------------------------------------------- src/dispatch.jl | 19 +++++++++ src/jogger.jl | 106 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 107 deletions(-) create mode 100644 src/dispatch.jl create mode 100644 src/jogger.jl diff --git a/src/PkgJogger.jl b/src/PkgJogger.jl index 42c3b04..00535ab 100644 --- a/src/PkgJogger.jl +++ b/src/PkgJogger.jl @@ -5,112 +5,7 @@ using BenchmarkTools export @jog -macro jog(pkg) - # Module Name - modname = Symbol(:Jog, pkg) - - # Locate benchmark folder - bench_dir = @eval benchmark_dir($pkg) - - # Generate modules - suite_modules = Expr[] - benchmarks = Symbol[] - for (name, file) in locate_benchmarks(bench_dir) - push!(suite_modules, build_module(name, file)) - push!(benchmarks, Symbol(name)) - end - - # Dispatch to PkgJogger functions - dispatch_funcs = Expr[] - for (m, f) in [(:BenchmarkTools, :run), (:BenchmarkTools, :warmup), (:PkgJogger, :benchmark)] - exp = quote - $f(args...; kwargs...) = $(m).$(f)(suite(), args...; kwargs...) - end - push!(dispatch_funcs, exp) - end - - # Flatten out modules into a Vector{Expr} - suite_exp = getfield(MacroTools.flatten(quote $(suite_modules...) end), :args) - - # Generate Module for Jogging pkg - quote - @eval module $modname - using $pkg - using BenchmarkTools - using PkgJogger - using Revise - - # Set Revise Mode and put submodules here - __revise_mode__ = :eval - $(suite_exp...) - - """ - suite() - - Gets the BenchmarkTools suite for $($pkg) - """ - function suite() - suite = BenchmarkGroup() - for (n, m) in zip([$(string.(benchmarks)...)], [$(benchmarks...)]) - suite[n] = m.suite - end - suite - end - - $(dispatch_funcs...) - end - end -end - -""" - benchmark_dir(pkg) - -Expected location of benchmarks for `pkg` -""" -function benchmark_dir(pkg::Module) - pkg_dir = joinpath(dirname(pathof(pkg)), "..") - joinpath(pkg_dir, "benchmark") |> abspath -end - - -function locate_benchmarks(dir) - suite = Dict{String, String}() - for file in readdir(dir; join=true) - m = match(r"bench_(.*?)\.jl$", file) - if m !== nothing - suite[m.captures[1]] = file - end - end - suite -end - -""" - build_module(name, file) - -Construct a module wrapping the BenchmarkGroup defined by `file` with `name` -""" -function build_module(name, file) - modname = Symbol(name) - exp = quote - module $modname - __revise_mode__ = :eval - include($file) - end - Revise.track($modname, $file) - end -end - - -""" - benchmark(s::BenchmarkGroup) - -Warmup, tune and run a benchmark suite -""" -function benchmark(s::BenchmarkTools.BenchmarkGroup; kwargs...) - warmup(s) - tune!(s) - BenchmarkTools.run(s) -end - +include("dispatch.jl") +include("jogger.jl") end diff --git a/src/dispatch.jl b/src/dispatch.jl new file mode 100644 index 0000000..ac4adc9 --- /dev/null +++ b/src/dispatch.jl @@ -0,0 +1,19 @@ +# This file contains functions that the generated modules will dispatch to + +# List of Module => function that `JogPkgName` will dispatch to +const DISPATCH_METHODS = [ + :BenchmarkTools => :run, + :BenchmarkTools => :warmup, + :PkgJogger => :benchmark +] + +""" + benchmark(s::BenchmarkGroup) + +Warmup, tune and run a benchmark suite +""" +function benchmark(s::BenchmarkTools.BenchmarkGroup; kwargs...) + warmup(s) + tune!(s) + BenchmarkTools.run(s) +end diff --git a/src/jogger.jl b/src/jogger.jl new file mode 100644 index 0000000..fb3236e --- /dev/null +++ b/src/jogger.jl @@ -0,0 +1,106 @@ +# This file contains functions related to building the JogPkgName module + +""" + @jog PkgName + +Creates a module named `JogPkgName` of benchmarks for `PkgName` pulled from +`PKG_DIR/benchmark/` + +Methods: + - `suite()` Return a `BenchmarkGroup` of the benchmarks for `PkgName` + - `benchmark()` Warmup, tune and run the suite +""" +macro jog(pkg) + # Module Name + modname = Symbol(:Jog, pkg) + + # Locate benchmark folder + bench_dir = @eval benchmark_dir($pkg) + + # Generate modules + suite_modules = Expr[] + benchmarks = Symbol[] + for (name, file) in locate_benchmarks(bench_dir) + push!(suite_modules, build_module(name, file)) + push!(benchmarks, Symbol(name)) + end + + # Dispatch to PkgJogger functions + dispatch_funcs = Expr[] + for (m, f) in DISPATCH_METHODS + exp = quote + $f(args...; kwargs...) = $(m).$(f)(suite(), args...; kwargs...) + end + push!(dispatch_funcs, exp) + end + + # Flatten out modules into a Vector{Expr} + suite_exp = getfield(MacroTools.flatten(quote $(suite_modules...) end), :args) + + # Generate Module for Jogging pkg + quote + @eval module $modname + using $pkg + using BenchmarkTools + using PkgJogger + using Revise + + # Set Revise Mode and put submodules here + __revise_mode__ = :eval + $(suite_exp...) + + """ + suite() + + Gets the BenchmarkTools suite for $($pkg) + """ + function suite() + suite = BenchmarkGroup() + for (n, m) in zip([$(string.(benchmarks)...)], [$(benchmarks...)]) + suite[n] = m.suite + end + suite + end + + $(dispatch_funcs...) + end + end +end + +""" + benchmark_dir(pkg) + +Expected location of benchmarks for `pkg` +""" +function benchmark_dir(pkg::Module) + pkg_dir = joinpath(dirname(pathof(pkg)), "..") + joinpath(pkg_dir, "benchmark") |> abspath +end + + +function locate_benchmarks(dir) + suite = Dict{String, String}() + for file in readdir(dir; join=true) + m = match(r"bench_(.*?)\.jl$", file) + if m !== nothing + suite[m.captures[1]] = file + end + end + suite +end + +""" + build_module(name, file) + +Construct a module wrapping the BenchmarkGroup defined by `file` with `name` +""" +function build_module(name, file) + modname = Symbol(name) + exp = quote + module $modname + __revise_mode__ = :eval + include($file) + end + Revise.track($modname, $file) + end +end From c5bad4e5ad7de27406f558933ec39f07a159d0d6 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Wed, 11 Aug 2021 11:35:48 -0400 Subject: [PATCH 2/4] Add test to check methods are dispatched to --- test/smoke.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/smoke.jl b/test/smoke.jl index a92eea8..12355c3 100644 --- a/test/smoke.jl +++ b/test/smoke.jl @@ -19,3 +19,12 @@ benchmark_dir = PkgJogger.benchmark_dir(PkgJogger) r = JogPkgJogger.run() @test typeof(r) <: BenchmarkTools.BenchmarkGroup end + +@testset "Jogger Methods" begin + @jog PkgJogger + @test @isdefined JogPkgJogger + + @testset "JogPkgJogger.$f" for (m, f) in PkgJogger.DISPATCH_METHODS + @test isdefined(JogPkgJogger, f) + end +end From f7893e1fc31dcf085b75e612ad229968b559fb3b Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Wed, 11 Aug 2021 12:43:35 -0400 Subject: [PATCH 3/4] docs: add some docs --- README.md | 14 ++++++++++++++ docs/src/index.md | 47 +++++++++++++++++++++++++++++++++++++++++++---- src/jogger.jl | 2 +- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 888e632..c4df08c 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,17 @@ [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://awadell1.github.io/PkgJogger.jl/stable) [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://awadell1.github.io/PkgJogger.jl/dev) [![Build Status](https://github.com/awadell1/PkgJogger.jl/workflows/CI/badge.svg)](https://github.com/awadell1/PkgJogger.jl/actions) + +PkgJogger is a benchmarking framework for Julia with the following features: + +- Just write benchmarks files:`benchmark/bench_*.jl` + + PkgJogger will wrap each benchmark file into a separate module, and return + a top level module with helper methods for running the suite + + Individual benchmark files only need to define a `suite::BenchmarkGroup` + +- Revise, benchmark, and revise again + + PkgJogger uses [Revise.jl] to track changes to benchmarking files and updates + the suite as you edit. No more waiting your benchmarks to precompile! diff --git a/docs/src/index.md b/docs/src/index.md index c2f7a2b..ab16ad0 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -4,11 +4,50 @@ CurrentModule = PkgJogger # PkgJogger -Documentation for [PkgJogger](https://github.com/awadell1/PkgJogger.jl). +*A Benchmarking Framework for Julia* -```@index +PkgJogger makes benchmarking easy by providing a framework for running [BenchmarkTool.jl](https://github.com/JuliaCI/BenchmarkTools.jl) benchmarks, while handling all the boilerplate setup. + +## Just write benchmarks + +Create a `benchmark/bench_*.jl` file, define a `suite` and go! + +```julia +using Benchmark +using AwesomePkg +suite = BenchmarkGroup() +suite["fast"] = @benchmarkable fast_code() ``` -```@autodocs -Modules = [PkgJogger] +PkgJogger will wrap each `benchmark/bench_*.jl` in it's own module and bundle them into `JogAwesomePkg` + +```julia +using AwesomePkg +using PkgJogger + +# Creates the JogAwesomePkg module +@jog AwesomePkg + +# Warmup, tune and run all of AwesomePkg's benchmarks +JogAwesomePkg.benchmark() ``` + +## Benchmark, Revise and Benchmark Again! + +PkgJogger uses [Revise.jl](https://github.com/timholy/Revise.jl) to track +changes to your `benchmark/bench_*.jl` files and reload your suite as you edit. +No more waiting for your benchmarks to precompile! + +Tracked Changes: + +- Changing your benchmarked function +- Changing benchmarking parameters (ie. `seconds` or `samples`) +- Adding new benchmarks + +Current Limitations: + +- New benchmark files are not tracked +- Deleted benchmarks will stick around +- Renamed benchmarks will create a new benchmark, and retain the old name + +To get around the above, run `@jog PkgName` to get a updated jogger. diff --git a/src/jogger.jl b/src/jogger.jl index fb3236e..0b1dcc7 100644 --- a/src/jogger.jl +++ b/src/jogger.jl @@ -4,7 +4,7 @@ @jog PkgName Creates a module named `JogPkgName` of benchmarks for `PkgName` pulled from -`PKG_DIR/benchmark/` +`PKG_DIR/benchmark/bench_*.jl` Methods: - `suite()` Return a `BenchmarkGroup` of the benchmarks for `PkgName` From 08319bb0a07557d048273dafcbd3a266fa4639dc Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Wed, 11 Aug 2021 12:49:29 -0400 Subject: [PATCH 4/4] fix: Revise link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c4df08c..b461237 100644 --- a/README.md +++ b/README.md @@ -15,5 +15,5 @@ PkgJogger is a benchmarking framework for Julia with the following features: - Revise, benchmark, and revise again - PkgJogger uses [Revise.jl] to track changes to benchmarking files and updates + PkgJogger uses [Revise.jl](https://github.com/timholy/Revise.jl) to track changes to benchmarking files and updates the suite as you edit. No more waiting your benchmarks to precompile!