Revert "feat: Add tune!(group, ref) for retuning suites"

This reverts commit 50aaabd.
awadell1 · Nov 19, 2021 · a8b25a8 · a8b25a8
1 parent 9a56453
commit a8b25a8
Show file tree

Hide file tree

Showing 5 changed files with 3 additions and 71 deletions.
diff --git a/docs/src/reference.md b/docs/src/reference.md
@@ -4,7 +4,6 @@
 PkgJogger.benchmark_dir
 PkgJogger.locate_benchmarks
 PkgJogger.judge
-PkgJogger.tune!
 ```
 
 ## Internal

diff --git a/src/jogger.jl b/src/jogger.jl
@@ -88,24 +88,15 @@ macro jog(pkg)
                 suite
             end
 
-            # Dispatch calls to tune! here so we can use the jogger variant of load_benchmarks
-            __tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...) = PkgJogger.tune!(group, ref; kwargs...)
-            __tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = PkgJogger.tune!(group, load_benchmarks(ref); kwargs...)
-            __tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = BenchmarkTools.tune!(group; kwargs...)
-
             """
-                benchmark(; ref = nothing, verbose = false)
+                benchmark(; verbose = false)
 
             Warmup, tune and run the benchmarking suite for $($pkg)
-
-            To reuse prior tuning results set `ref` to a BenchmarkGroup or suitable identifier
-            for [`$($modname).load_benchmarks`](@ref). See [`PkgJogger.tune!`](@ref) for
-            more information about re-using tuning results.
             """
-            function benchmark(; ref = nothing, verbose = false)
+            function benchmark(; verbose = false)
                 s = suite()
                 BenchmarkTools.warmup(s; verbose)
-                __tune!(s, ref; verbose = verbose)
+                BenchmarkTools.tune!(s; verbose = verbose)
                 BenchmarkTools.run(s; verbose = verbose)
             end
 

diff --git a/src/utils.jl b/src/utils.jl
@@ -94,37 +94,3 @@ end
 _get_benchmarks(b::BenchmarkTools.BenchmarkGroup) = b
 _get_benchmarks(b::Dict) =  b["benchmarks"]::BenchmarkTools.BenchmarkGroup
 _get_benchmarks(filename::String) = load_benchmarks(filename) |> _get_benchmarks
-
-"""
-    tune!(group::BenchmarkGroup, ref::BenchmarkGroup; verbose::Bool=false)
-
-Tunes a BenchmarkGroup, only tunning benchmarks not found in `ref`, otherwise reuse tuning
-results from the reference BenchmarkGroup, by copying over all benchmark parameters from `ref`.
-
-This can reduce benchmarking runtimes significantly by only tuning new benchmarks. But does
-ignore the following:
-    - Changes to benchmarking parameters (ie. memory_tolerance) between `group` and `ref`
-    - Significant changes in performance, such that re-tunning is warranted
-    - Other changes (ie. changing machines), such that re-tunning is warranted?
-"""
-function tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...)
-    ids = keys(group) |> collect
-    ref_ids = keys(ref) |> collect
-
-    # Tune new benchmarks
-    for id in setdiff(ids, ref_ids)
-        tune!(group[id]; kwargs...)
-    end
-
-    # Reuse tunning from prior benchmarks
-    for id in intersect(ids, ref_ids)
-        tune!(group[id], ref[id]; kwargs...)
-    end
-
-    return group
-end
-tune!(b::BenchmarkTools.Benchmark, ref; kwargs...) = b.params = copy(ref.params)
-tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = tune!(group; kwargs...)
-tune!(group::BenchmarkTools.BenchmarkGroup; kwargs...) = BenchmarkTools.tune!(group; kwargs...)
-tune!(group::BenchmarkTools.BenchmarkGroup, ref::Dict; kwargs...) = tune!(group, ref["benchmarks"]; kwargs...)
-tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = tune!(group, load_benchmarks(ref); kwargs...)
diff --git a/test/smoke.jl b/test/smoke.jl
@@ -46,13 +46,6 @@ include("utils.jl")
         @test_throws AssertionError JogExample.load_benchmarks(UUIDs.uuid4())
     end
 
-    # Test Retuning
-    @testset "Reusing tune! results" begin
-        test_benchmark(JogPkgJogger.benchmark(ref = r), r)
-        test_benchmark(JogPkgJogger.benchmark(ref = get_uuid(file)), r)
-        test_benchmark(JogPkgJogger.benchmark(ref = file), r)
-    end
-
     # Test Judging
     @test_nowarn JogExample.judge(file, file)
 

diff --git a/test/utils.jl b/test/utils.jl
@@ -52,20 +52,3 @@ function get_uuid(filename)
     splitpath(filename)[end] |> x -> split(x, ".")[1]
 end
 
-"""
-    test_benchmark(target::BenchmarkGroup, ref)
-
-Checks that target and ref are from equivalent benchmarking suite
-"""
-function test_benchmark(target, ref::BenchmarkGroup)
-    @test typeof(target) <: BenchmarkGroup
-    @test isempty(setdiff(keys(target), keys(ref)))
-    map(test_benchmark, target, ref)
-end
-test_benchmark(target, ref) = @test typeof(target) <: typeof(ref)
-function test_benchmark(target, ref::BenchmarkTools.Trial)
-    @test typeof(target) <: BenchmarkTools.Trial
-    @test params(target) == params(ref)
-end
-
-