Merge branch 'loading'

Project.toml

@@ -12,6 +12,7 @@ LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]

docs/src/jogger.md

@@ -30,5 +30,7 @@ JogExample.benchmark
 JogExample.warmup
 JogExample.run
 JogExample.save_benchmarks
+JogExample.load_benchmarks
+JogExample.judge
 JogExample.BENCHMARK_DIR
 ```

docs/src/reference.md

@@ -3,6 +3,7 @@
 ```@docs
 PkgJogger.benchmark_dir
 PkgJogger.locate_benchmarks
+PkgJogger.judge
 ```
 
 ## Internal

src/PkgJogger.jl

@@ -9,6 +9,7 @@ using Pkg
 using UUIDs
 using Dates
 using LibGit2
+using Statistics
 
 export @jog
 

src/jogger.jl

@@ -128,13 +128,46 @@ macro jog(pkg)
 
             Returns the path to the saved results
 
-            Results can be loaded with [`PkgJogger.load_benchmarks(filename)`](@ref)
+            Results can be loaded with [`PkgJogger.load_benchmarks(filename)`](@ref) or
+            [`$($modname).load_benchmarks(uuid)`](@ref)
             """
             function save_benchmarks(results)
                 filename = joinpath(BENCHMARK_DIR, "trial", "$(UUIDs.uuid4()).json.gz")
                 PkgJogger.save_benchmarks(filename, results)
                 filename
             end
+
+            """
+                load_benchmarks(filename::String)::Dict
+                load_benchmarks(uuid::String)::Dict
+                load_benchmarks(uuid::UUID)::Dict
+
+            Loads benchmarking results for $($pkg) from `BENCHMARK_DIR/trial`
+            """
+            load_benchmarks(uuid::UUIDs.UUID) = load_benchmarks(string(uuid))
+            function load_benchmarks(uuid::AbstractString)
+                # Check if input is a filename
+                isfile(uuid) && return PkgJogger.load_benchmarks(uuid)
+
+                # Check if a valid benchmark uuid
+                path = joinpath(BENCHMARK_DIR, "trial", uuid * ".json.gz")
+                @assert isfile(path) "Missing benchmarking results for $uuid, expected path: $path"
+                PkgJogger.load_benchmarks(path)
+            end
+
+            """
+                judge(new, old; metric=Statistics.median, kwargs...)
+
+            Compares benchmarking results from `new` vs `old` for regressions/improvements
+            using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge`
+
+            Identical to [`PkgJogger.judge`](@ref), but accepts UUIDs for `new` and `old`
+            """
+            function judge(new, old; kwargs...)
+                PkgJogger.judge(load_benchmarks(new), _get_benchmarks(old); kwargs...)
+            end
+            _get_benchmarks(b::AbstractString) = load_benchmarks(b)
+            _get_benchmarks(b) = PkgJogger._get_benchmarks(b)
         end
     end
 end

src/utils.jl

@@ -47,3 +47,36 @@ function locate_benchmarks(dir)
     suite
 end
 locate_benchmarks(pkg::Module) = benchmark_dir(pkg) |> locate_benchmarks
+
+
+"""
+    judge(new, old; metric=Statistics.median, kwargs...)
+
+Compares benchmarking results from `new` vs `old` for regressions/improvements
+using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge`
+
+Effectively a convenience wrapper around `load_benchmarks` and `BenchmarkTools.judge`
+
+`new` and `old` can be any one of the following:
+    - Filename of benchmarking results saved by PkgJogger
+    - A `Dict` as returned by [`PkgJogger.load_benchmarks(filename)`](@ref)
+    - A `BenchmarkTools.BenchmarkGroup` with benchmarking results
+"""
+function judge(
+    new::BenchmarkTools.BenchmarkGroup,
+    old::BenchmarkTools.BenchmarkGroup;
+    metric = Statistics.median,
+    kwargs...
+)
+    new_estimate = metric(new)
+    old_estimate = metric(old)
+    BenchmarkTools.judge(new_estimate, old_estimate; kwargs...)
+end
+function judge(new, old; kwargs...)
+    new_results = _get_benchmarks(new)
+    old_results = _get_benchmarks(old)
+    judge(new_results, old_results; kwargs...)
+end
+
+_get_benchmarks(b::Dict) =  b["benchmarks"]::BenchmarkTools.BenchmarkGroup
+_get_benchmarks(filename::String) = load_benchmarks(filename) |> _get_benchmarks

test/Project.toml

@@ -1,6 +1,8 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

test/judging.jl

@@ -0,0 +1,23 @@
+using PkgJogger
+using BenchmarkTools
+
+include("utils.jl")
+
+# Add Example
+using Pkg
+Pkg.develop(path="Example.jl/")
+using Example
+@jog Example
+
+# Run Benchmarks for testing
+r1 = JogExample.benchmark() |> JogExample.save_benchmarks
+r2 = JogExample.benchmark() |> JogExample.save_benchmarks
+
+# Get UUIDs
+r1_uuid = get_uuid(r1)
+r2_uuid = get_uuid(r2)
+
+@test typeof(PkgJogger.judge(r1, r2)) <: BenchmarkGroup
+@testset "Test loading" for new=[r1, r1_uuid], old=[r2, r2_uuid]
+    @test typeof(JogExample.judge(new, old)) <: BenchmarkGroup
+end

test/runtests.jl

@@ -13,5 +13,6 @@ using PkgJogger
 
     # Run the rest of the unit testing suite
     @safetestset "Smoke Tests" begin include("smoke.jl") end
+    @safetestset "Judging" begin include("judging.jl") end
     @safetestset "CI Workflow" begin include("ci.jl") end
 end

test/smoke.jl

@@ -1,5 +1,6 @@
 using Test
 using PkgJogger
+using UUIDs
 import BenchmarkTools
 
 include("utils.jl")
@@ -29,6 +30,24 @@ include("utils.jl")
     test_loaded_results(r2)
     @test r == r2["benchmarks"]
 
+    # Load with JogPkgJogger
+    @testset "Jogger's load_benchmarks" begin
+        uuid = get_uuid(file)
+        r3 = JogPkgJogger.load_benchmarks(uuid)
+        r4 = JogPkgJogger.load_benchmarks(UUID(uuid))
+        @test r3 == r4
+        @test r3["benchmarks"] == r
+        @test r4["benchmarks"] == r
+        @test r2 == r3 == r4
+
+        # Check that we error for invalid uuids
+        @test_throws AssertionError JogPkgJogger.load_benchmarks("not-a-uuid")
+        @test_throws AssertionError JogPkgJogger.load_benchmarks(UUIDs.uuid4())
+    end
+
+    # Test Judging
+    @test_nowarn JogPkgJogger.judge(file, file)
+
     # If this is a git repo, there should be a git entry
     if isdir(joinpath(PKG_JOGGER_PATH, ".git"))
         @test r2["git"] !== nothing

test/utils.jl

@@ -67,4 +67,12 @@ function test_subfile(parent, child)
     end
 end
 
+"""
+    get_uuid(filename)
+
+Extract benchmark UUID from filename
+"""
+function get_uuid(filename)
+    splitpath(filename)[end] |> x -> split(x, ".")[1]
+end