Merge pull request #48 from awadell1/tui

Breaking: Switch to BSON for storage

.gitignore

@@ -5,3 +5,4 @@ Manifest.toml
 /docs/build/
 /benchmark/trial/
 *.json.gz
+*.bson.gz

Project.toml

@@ -1,9 +1,10 @@
 name = "PkgJogger"
 uuid = "10150987-6cc1-4b76-abee-b1c1cbd91c01"
 authors = ["Alexius Wadell <[email protected]> and contributors"]
-version = "0.3.5"
+version = "0.4.0"
 
 [deps]
+BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -17,6 +18,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
+BSON = "0.3"
 BenchmarkTools = "1"
 CodecZlib = "0.7"
 JSON = "0.21"

README.md

@@ -62,7 +62,7 @@ To get around the above, run `@jog PkgName` to get an updated jogger.
 
 ## Continuous Benchmarking Baked In!
 
-Install PkgJogger, run benchmarks, and save results to a `*.json.gz` with a
+Install PkgJogger, run benchmarks, and save results to a `*.bson.gz` with a
 one-line command.
 
 ```shell
@@ -77,7 +77,7 @@ What gets done:
 - Creates a [jogger](https://awadell1.github.io/PkgJogger.jl/stable/jogger/)
   to run the package's benchmarks.
 - Warmup, tune and run all benchmarks.
-- Save Benchmarking results and more to a compressed `*.json.gz` file.
+- Save Benchmarking results and more to a compressed `*.bson.gz` file.
 
 Or for a more lightweight option, use
 [`@test_bechmarks`](https://awadell1.github.io/PkgJogger.jl/stable/ci/#Testing-Benchmarks)

docs/src/io.md

@@ -11,6 +11,7 @@ These methods build on
   - System Information (Essentially everything in `Sys`)
   - Timestamp when the results get saved
   - Git Information, if run from a Git Repository
+  - The version of PkgJogger used to save the results
 
 Overall the resulting files are ~10x smaller, despite capturing additional information.
 
@@ -28,7 +29,7 @@ using PkgJogger
 @jog AwesomePkg
 results = JogAwesomePkg.benchmark()
 
-# Saves results to BENCH_DIR/trial/UUID.json.gz and returns the filename used
+# Saves results to BENCH_DIR/trial/UUID.bson.gz and returns the filename used
 JogAwesomePkg.save_benchmarks(results)
 
 # Or run and save the benchmarks in a single step, the filename saved to

src/PkgJogger.jl

@@ -5,6 +5,7 @@ using BenchmarkTools
 using Revise
 using CodecZlib
 using JSON
+using BSON
 using Pkg
 using UUIDs
 using Dates
@@ -27,6 +28,10 @@ const JOGGER_PKGS = [
     PkgId(UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4"), "UUIDs"),
 ]
 
+const PKG_JOGGER_VER = VersionNumber(
+    Base.parsed_toml(joinpath(@__DIR__, "..", "Project.toml"))["version"]
+)
+
 include("utils.jl")
 include("jogger.jl")
 include("ci.jl")

src/ci.jl

@@ -88,7 +88,7 @@ end
 """
     save_benchmarks(filename, results::BenchmarkGroup)
 
-Save benchmarking results to `filename.json.gz` for later
+Save benchmarking results to `filename.bson.gz` for later
 analysis.
 
 ## File Contents
@@ -97,9 +97,10 @@ analysis.
 - Timestamp
 - Benchmarking Results
 - Git Commit, 'Is Dirty' status and author datetime
+- PkgJogger Version used to save the file
 
 ## File Format:
-Results are saved as a gzip compressed JSON file and can be loaded
+Results are saved as a gzip compressed BSON file and can be loaded
 with [`PkgJogger.load_benchmarks`](@ref)
 """
 function save_benchmarks(filename, results::BenchmarkTools.BenchmarkGroup)
@@ -108,14 +109,15 @@ function save_benchmarks(filename, results::BenchmarkTools.BenchmarkGroup)
     out = Dict(
         "julia" => julia_info(),
         "system" => system_info(),
-        "datetime" => string(Dates.now()),
+        "datetime" => Dates.now(),
         "benchmarks" => results,
         "git" => git_info(filename),
+        "pkgjogger" => PkgJogger.PKG_JOGGER_VER,
     )
 
     # Write benchmark to disk
     open(GzipCompressorStream, filename, "w") do io
-        JSON.print(io, out)
+        bson(io, out)
     end
 end
 
@@ -173,7 +175,7 @@ function git_info(path)
         return Dict(
             "commit" => LibGit2.GitHash(head) |> string,
             "is_dirty" =>  LibGit2.with(LibGit2.isdirty, GitRepo(ref_dir)),
-            "datetime" => Dates.unix2datetime(author_sig.time) |> string,
+            "datetime" => Dates.unix2datetime(author_sig.time),
         )
     catch e
         if e isa LibGit2
@@ -191,16 +193,73 @@ end
     load_benchmarks(filename::String)::Dict
 
 Load benchmarking results from `filename`
+
+> Prior to v0.4 PkgJogger saved results as `*.json.gz` instead of `*.bson.gz`.
+> This function supports both formats. However, the `*.json.gz` format is
+> deprecated, and may not support all features.
 """
-function load_benchmarks(filename)
+function load_benchmarks(filename::AbstractString)
     # Decompress
-    out = open(JSON.parse, GzipDecompressorStream, filename)
+    if endswith(filename, ".json.gz")
+        @warn "Legacy `*.json.gz` format is deprecated, some features may not be supported"
+        reader = JSON.parse
+    elseif endswith(filename, ".bson.gz")
+        reader = io -> BSON.load(io, @__MODULE__)
+    else
+        error("Unsupported file format: $filename")
+    end
+    out = open(reader, GzipDecompressorStream, filename)
 
-    # Recover BenchmarkTools Types
-    if haskey(out, "benchmarks")
+    # Get PkgJogger version
+    version = haskey(out, "pkgjogger") ? out["pkgjogger"] : missing
+
+    # Recover Benchmarking Results
+    if ismissing(version)
+        @assert haskey(out, "benchmarks") "Missing 'benchmarks' field in $filename"
         out["benchmarks"] = BenchmarkTools.recover(out["benchmarks"])
-    else
-        error("Missing 'benchmarks' field in $filename")
     end
-    out
+
+    return out
 end
+
+# Possible file extensions for a PkgJogger file
+const PKG_JOGGER_EXT = (".bson.gz", ".json.gz")
+
+# Handle dispatch on UUIDs from Jogger
+function load_benchmarks(trial_dir::AbstractString, uuid::UUIDs.UUID)
+    for ext in PKG_JOGGER_EXT
+        full_path = joinpath(trial_dir, string(uuid) * ext)
+        if isfile(full_path)
+            return load_benchmarks(full_path)
+        end
+    end
+    error("Unable to find benchmarking results for $uuid in $trial_dir")
+end
+
+# Handle dispatch on a string from Jogger
+function load_benchmarks(trial_dir, id::AbstractString)
+    # Attempt to parse string as UUID -> If not, assume it is a filename
+    uuid = Base.tryparse(UUID, id)
+    isnothing(uuid) && return load_benchmarks(id)
+    return load_benchmarks(trial_dir, uuid)
+end
+
+# Handle dispatch on a Symbol
+load_benchmarks(trial_dir, s::Symbol) = load_benchmarks(trial_dir, Val(s))
+
+# Load the latest benchmarking results
+load_benchmarks(trial_dir::AbstractString, ::Val{:latest}) =
+    load_benchmarks(argmax(mtime, list_benchmarks(trial_dir)))
+
+# Loads the oldest benchmarking results
+load_benchmarks(trial_dir::AbstractString, ::Val{:oldest}) =
+    load_benchmarks(argmin(mtime, list_benchmarks(trial_dir)))
+
+function list_benchmarks(dir)
+    isdir(dir) || return String[]
+    files = readdir(dir; join=true, sort=false)
+    r = filter(f -> any(e -> endswith(f, e), PKG_JOGGER_EXT), files)
+    @assert !isempty(r) "No benchmarking results found in $dir"
+    return r
+end
+

src/jogger.jl

@@ -143,50 +143,69 @@ macro jog(pkg)
             """
                 save_benchmarks(results::BenchmarkGroup)::String
 
-            Saves benchmarking results for $($pkg) to `BENCHMARK_DIR/trial/uuid4().json.gz`.
+            Saves benchmarking results for $($pkg) to `BENCHMARK_DIR/trial/uuid4().bson.gz`.
 
             Returns the path to the saved results
 
             Results can be loaded with [`PkgJogger.load_benchmarks(filename)`](@ref) or
             [`$($modname).load_benchmarks(uuid)`](@ref)
             """
             function save_benchmarks(results)
-                filename = joinpath(BENCHMARK_DIR, "trial", "$(UUIDs.uuid4()).json.gz")
+                filename = joinpath(BENCHMARK_DIR, "trial", "$(UUIDs.uuid4()).bson.gz")
                 PkgJogger.save_benchmarks(filename, results)
                 filename
             end
 
             """
-                load_benchmarks(filename::String)::Dict
-                load_benchmarks(uuid::String)::Dict
-                load_benchmarks(uuid::UUID)::Dict
+                load_benchmarks(id)::Dict
 
-            Loads benchmarking results for $($pkg) from `BENCHMARK_DIR/trial`
-            """
-            load_benchmarks(uuid::UUIDs.UUID) = load_benchmarks(string(uuid))
-            function load_benchmarks(uuid::AbstractString)
-                # Check if input is a filename
-                isfile(uuid) && return PkgJogger.load_benchmarks(uuid)
+            Loads benchmarking results for $($pkg) from `BENCHMARK_DIR/trial` based on `id`.
+            The following are supported `id` types:
 
-                # Check if a valid benchmark uuid
-                path = joinpath(BENCHMARK_DIR, "trial", uuid * ".json.gz")
-                @assert isfile(path) "Missing benchmarking results for $uuid, expected path: $path"
-                PkgJogger.load_benchmarks(path)
-            end
+                - `filename::String`: Loads results from `filename`
+                - `uuid::Union{String, UUID}`: Loads results with the given UUID
+                - `:latest` loads the latest (By mtime) results from `BENCHMARK_DIR/trial`
+                - `:oldest` loads the oldest (By mtime) results from `BENCHMARK_DIR/trial`
+            """
+            load_benchmarks(id) = PkgJogger.load_benchmarks(joinpath(BENCHMARK_DIR, "trial"), id)
 
             """
                 judge(new, old; metric=Statistics.median, kwargs...)
 
             Compares benchmarking results from `new` vs `old` for regressions/improvements
             using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge`
 
-            Identical to [`PkgJogger.judge`](@ref), but accepts UUIDs for `new` and `old`
+            Identical to [`PkgJogger.judge`](@ref), but accepts any identifier supported by
+            [`$($modname).load_benchmarks`](@ref)
+
+            ## Examples
+
+            ```julia
+            # Judge the latest results vs. the oldest
+            $($modname).judge(:latest, :oldest)
+            [...]
+            ```
+
+            ```julia
+            # Judge results by UUID
+            $($modname).judge("$(UUIDs.uuid4())", "$(UUIDs.uuid4())")
+            [...]
+            ```
+
+            ```julia
+            # Judge using the minimum, instead of the median, time
+            $($modname).judge("path/to/results.bson.gz", "$(UUIDs.uuid4())"; metric=minimum)
+            [...]
+            ```
+
             """
             function judge(new, old; kwargs...)
                 PkgJogger.judge(_get_benchmarks(new), _get_benchmarks(old); kwargs...)
             end
-            _get_benchmarks(b::AbstractString) = load_benchmarks(b)
-            _get_benchmarks(b) = PkgJogger._get_benchmarks(b)
+            _get_benchmarks(b) = load_benchmarks(b)
+            _get_benchmarks(b::Dict) = PkgJogger._get_benchmarks(b)
+            _get_benchmarks(b::BenchmarkTools.BenchmarkGroup) = b
+
         end
     end
 end

src/utils.jl

@@ -91,9 +91,13 @@ function judge(new, old; kwargs...)
     judge(new_results, old_results; kwargs...)
 end
 
+# Internal functions to handle extracting benchmark results from various types
 _get_benchmarks(b::BenchmarkTools.BenchmarkGroup) = b
-_get_benchmarks(b::Dict) =  b["benchmarks"]::BenchmarkTools.BenchmarkGroup
-_get_benchmarks(filename::String) = load_benchmarks(filename) |> _get_benchmarks
+_get_benchmarks(filename::AbstractString) = _get_benchmarks(load_benchmarks(filename))
+function _get_benchmarks(b::Dict)
+    @assert haskey(b, "benchmarks") "Missing 'benchmarks' key in $b"
+    return b["benchmarks"]::BenchmarkTools.BenchmarkGroup
+end
 
 """
     test_benchmarks(s::BenchmarkGroup)
@@ -164,24 +168,28 @@ ignore the following:
     - Significant changes in performance, such that re-tunning is warranted
     - Other changes (ie. changing machines), such that re-tunning is warranted
 """
-function tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...)
-    ids = keys(group) |> collect
-    ref_ids = keys(ref) |> collect
-
-    # Tune new benchmarks
-    for id in setdiff(ids, ref_ids)
-        tune!(group[id]; kwargs...)
-    end
-
-    # Reuse tunning from prior benchmarks
-    for id in intersect(ids, ref_ids)
-        tune!(group[id], ref[id]; kwargs...)
+function tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; pad="", kwargs...)
+    for (k, v) in group
+        if haskey(ref, k)
+            # Load tuning parameters from reference
+            tune!(v, ref[k]; kwargs...)
+        else
+            # Fallback to re-tuning the benchmark
+            tune!(v; kwargs...)
+        end
     end
-
     return group
 end
-tune!(b::BenchmarkTools.Benchmark, ref; kwargs...) = b.params = copy(ref.params)
+
+# Fallback to using BenchmarkTools's tuning if no reference is provided
 tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = tune!(group; kwargs...)
 tune!(group::BenchmarkTools.BenchmarkGroup; kwargs...) = BenchmarkTools.tune!(group; kwargs...)
-tune!(group::BenchmarkTools.BenchmarkGroup, ref::Dict; kwargs...) = tune!(group, get(ref, "benchmarks", nothing); kwargs...)
-tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = tune!(group, load_benchmarks(ref); kwargs...)
+
+# If ref is not a BenchmarkGroup, attempt to get it based on the type
+tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = tune!(group, _get_benchmarks(ref); kwargs...)
+
+# Load tuning results from the reference benchmarking results
+function tune!(b::BenchmarkTools.Benchmark, ref; kwargs...)
+    p = ref isa BenchmarkTools.Parameters ? ref : BenchmarkTools.params(ref)
+    BenchmarkTools.loadparams!(b, p, :evals, :samples)
+end

test/Project.toml

@@ -1,6 +1,9 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"