From b063394f6a6024ded53ec092e527556e84670fa6 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Tue, 5 Oct 2021 00:09:49 -0400 Subject: [PATCH 1/6] ci: expand build matrix --- .github/workflows/CI.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f38fced..07f6914 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -11,8 +11,12 @@ jobs: matrix: version: - '1' + - 1.7-nightly + - nightly os: - ubuntu-latest + - windows-latest + - macOS-latest arch: - x64 steps: From a333a9e8b5b656140603b514db1d6f199a92f212 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Tue, 5 Oct 2021 00:33:55 -0400 Subject: [PATCH 2/6] fix: bug in tests for windows --- test/ci.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/ci.jl b/test/ci.jl index 627922a..68d16e8 100644 --- a/test/ci.jl +++ b/test/ci.jl @@ -10,15 +10,17 @@ function run_ci_workflow(pkg_dir) cp(pkg_dir, temp_project; force=true) # Construct CI Command + pkgjogger_path = escape_string(PKG_JOGGER_PATH) cmd = Cmd([ "julia", "--code-coverage=all", "--eval", - "using Pkg; Pkg.develop(path=\"$PKG_JOGGER_PATH\"); using PkgJogger; PkgJogger.ci()" + "using Pkg; Pkg.develop(path=\"$pkgjogger_path\"); using PkgJogger; PkgJogger.ci()" ]) |> ignorestatus # Set Environmental Variables + sep = Sys.iswindows() ? ";" : ":" cmd = setenv(cmd, "JULIA_PROJECT" => temp_project, # Use the temporary project - "JULIA_LOAD_PATH" => "@:@stdlib", # Enable stdlib but ignore user projects + "JULIA_LOAD_PATH" => join(["@", "@stdlib"], sep), # Enable stdlib but ignore user projects "PATH" => Sys.BINDIR, # Add Julia to the PATH ) From 60ffa639916cc08a10f408bfe815859a38b066b3 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Mon, 25 Oct 2021 10:08:38 -0400 Subject: [PATCH 3/6] Fix usage of Base.env_project_file due to change in def Really should have been like this from the beginning (the docs haven't changed), but just check if benchmark directory defines an environment --- src/ci.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ci.jl b/src/ci.jl index aabc5f8..2849b31 100644 --- a/src/ci.jl +++ b/src/ci.jl @@ -28,10 +28,9 @@ function ci() # Look for a benchmark project, and add to LOAD_PATH if it exists # TODO: Use Sub-project https://github.com/JuliaLang/Pkg.jl/issues/1233 bench_dir = benchmark_dir(pkg) - benchmark_project = Base.env_project_file(bench_dir) load_path = String[] - if isfile(benchmark_project) - @info "Found benchmark project: $benchmark_project" + if Base.env_project_file(bench_dir) != false + @info "Found benchmarking environment at $bench_dir" instantiate(bench_dir) push!(load_path, bench_dir) end From f478b0cd9f2a98e006b6cc7562e115f2d2f60c50 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Wed, 27 Oct 2021 15:59:22 -0400 Subject: [PATCH 4/6] feat: build up temp benchmarking env No more mucking around with the LOAD_PATH, just use Pkg.jl's API Changes to resolution - benchmark dependecies get resolved with pkg's (At the same tier) - PkgJogger's dependecies get resolved tiered So PkgJogger can now change resolution, but only if it would otherwise fail Ie. the options are fail to run the benchmarks or modify the resolution As we display the manifest this should be okay --- src/ci.jl | 55 +++++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/ci.jl b/src/ci.jl index 2849b31..17d129a 100644 --- a/src/ci.jl +++ b/src/ci.jl @@ -25,20 +25,10 @@ function ci() path=dirname(project), ) - # Look for a benchmark project, and add to LOAD_PATH if it exists - # TODO: Use Sub-project https://github.com/JuliaLang/Pkg.jl/issues/1233 - bench_dir = benchmark_dir(pkg) - load_path = String[] - if Base.env_project_file(bench_dir) != false - @info "Found benchmarking environment at $bench_dir" - instantiate(bench_dir) - push!(load_path, bench_dir) - end - # Run in sandbox pkgname = Symbol(pkg.name) jogger = Symbol(:Jog, pkg.name) - sandbox(pkg, load_path) do + sandbox(pkg) do @eval Main begin using PkgJogger using $pkgname @@ -50,7 +40,7 @@ function ci() end end -function sandbox(f, pkg, load_path) +function sandbox(f, pkg) # Save current project and load path current_project = Pkg.project() current_load_path = Base.LOAD_PATH @@ -61,25 +51,34 @@ function sandbox(f, pkg, load_path) uuid = JOGGER_PKGS[1].uuid, ) - # Build temporary environment - # Add the project being benchmarked, then PkgJogger restricted to existing - # manifest. Ie. The benchmarked projects drives manifest resolution, not PkgJogger - Pkg.activate(;temp=true) - Pkg.develop(pkg; io=devnull) - Pkg.add(self; preserve=PRESERVE_ALL, io=devnull) - Pkg.instantiate(; io=devnull) + # Locate benchmark project + # TODO: Use Sub-project https://github.com/JuliaLang/Pkg.jl/issues/1233 + bench_dir = benchmark_dir(pkg) + bench_project = joinpath(bench_dir, Base.current_project(bench_dir)) - # Update LOAD_PATH - # Only load code from: Temp Environment or benchmark/Project.toml - empty!(Base.LOAD_PATH) - append!(Base.LOAD_PATH, vcat(["@"], load_path)) + # Create a temporary environment + mktempdir() do temp_env + # Copy benchmarking project over iff it exists + if isfile(bench_project) + cp(bench_project, joinpath(temp_env, basename(bench_project))) + end - # Report current status - Pkg.status(;mode=PKGMODE_MANIFEST) - @debug "LOAD_PATH: $(Base.LOAD_PATH)" + # Build up benchmarked environment + Pkg.activate(temp_env; io=devnull) + Pkg.develop(pkg; preserve=PRESERVE_NONE, io=devnull) + Pkg.add(self; preserve=PRESERVE_TIERED, io=devnull) + Pkg.instantiate(; io=devnull) - # Run function - f() + # Strip LOAD_PATH to the temporary environment + empty!(Base.LOAD_PATH) + push!(Base.LOAD_PATH, "@") + + # Report current status + Pkg.status(;mode=PKGMODE_MANIFEST) + + # Run function + f() + end # Restore environment and load path empty!(Base.LOAD_PATH) From 34dbfee9a7e95ba1db808ecdfd731775ad5313be Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Tue, 2 Nov 2021 20:05:10 -0400 Subject: [PATCH 5/6] Update docs for benchmarking environment --- docs/src/ci.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/src/ci.md b/docs/src/ci.md index fe3190b..16ac329 100644 --- a/docs/src/ci.md +++ b/docs/src/ci.md @@ -34,15 +34,24 @@ jobs: PkgJogger will create a temporary environment with the following: -1) Instantiate the current package -2) If found, instantiate `benchmark/Project.toml` and add to the `LOAD_PATH` -3) Add PkgJogger while preserving the resolved manifest -4) Remove `@stdlib` and `@v#.#` from the `LOAD_PATH` +1) Activate a temporary Julia environment for benchmarking. + - If a Julia project file exists in `benchmark/`, it will be copied to the + temporary environment. Manifest files are currently ignored. + - Otherwise an empty environment is created. +2) Add the current project (via `Pkg.develop`) to the benchmarking environment + and resolve dependencies using + [`PRESEVE_NONE`](https://pkgdocs.julialang.org/v1/api/#Pkg.add). +3) Add `PkgJogger` and resolve dependencies using + [`PRESERVE_TIERED`](https://pkgdocs.julialang.org/v1/api/#Pkg.add). +4) Strip the + [`LOAD_PATH`](https://docs.julialang.org/en/v1/base/constants/#Base.LOAD_PATH) + to the benchmarking environment. The prior `LOAD_PATH` is restored after benchmarking. This results in an isolated environment with the following properties: -- PkgJogger does not dictate package resolution; the benchmarked package does +- Minimizes PkgJogger's impact on dependency resolution. - Packages not explicitly added by `Project.toml` or `benchmark/Project.toml` + are not available in the benchmarking environment. ## Reference From 8124673888baeddbdab22369a2be5c6c40218057 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Tue, 2 Nov 2021 20:05:39 -0400 Subject: [PATCH 6/6] Revamp docs and use README.md for index.md Instead of having two home pages -> Now there's just one! --- README.md | 77 ++++++++++++++++++++++++++++++++++++---------- docs/make.jl | 15 +++++++++ docs/src/index.md | 76 --------------------------------------------- docs/src/io.md | 4 +-- docs/src/jogger.md | 8 ++--- 5 files changed, 81 insertions(+), 99 deletions(-) delete mode 100644 docs/src/index.md diff --git a/README.md b/README.md index f71f3e2..a17aa82 100644 --- a/README.md +++ b/README.md @@ -8,28 +8,71 @@ [![version](https://juliahub.com/docs/PkgJogger/version.svg)](https://juliahub.com/ui/Packages/PkgJogger/AaLEJ) [![pkgeval](https://juliahub.com/docs/PkgJogger/pkgeval.svg)](https://juliahub.com/ui/Packages/PkgJogger/AaLEJ) -PkgJogger is a benchmarking framework for Julia built on -[BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) with the -following features: +PkgJogger provides a framework for running suites of +[BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) benchmarks +without the boilerplate. -- Just write benchmarks files:`benchmark/bench_*.jl` +## Just write benchmarks - PkgJogger will wrap each benchmark file into a separate module, and return a - top-level module with helper methods for running the suite +Create a `benchmark/bench_*.jl` file, define a +[BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) `suite` and +go! - Individual benchmark files only need to define a `suite::BenchmarkGroup` +```julia +using BenchmarkTools +using AwesomePkg +suite = BenchmarkGroup() +suite["fast"] = @benchmarkable fast_code() +``` -- Revise, benchmark, and revise again +PkgJogger will wrap each `benchmark/bench_*.jl` in a module and bundle them into `JogAwesomePkg` - PkgJogger uses [Revise.jl](https://github.com/timholy/Revise.jl) to track - changes to benchmarking files and updates the suite as you edit. No more - waiting for benchmarks to precompile! +```julia +using AwesomePkg +using PkgJogger -- Continuous Benchmarking Baked In! +# Creates the JogAwesomePkg module +@jog AwesomePkg - Setup and isolated environment, run benchmarks and save results with a - one-liner: +# Warmup, tune, and run all of AwesomePkg's benchmarks +JogAwesomePkg.benchmark() +``` - ```shell - julia -e 'using Pkg; Pkg.add("PkgJogger"); using PkgJogger; PkgJogger.ci()' - ``` +## Benchmark, Revise, and Benchmark Again! + +PkgJogger uses [Revise.jl](https://github.com/timholy/Revise.jl) to track +changes to your `benchmark/bench_*.jl` files and reload your suite as you edit. +No more waiting for benchmarks to precompile! + +Tracked Changes: + +- Changing your benchmarked function +- Changing benchmarking parameters (i.e. `seconds` or `samples`) +- Adding new benchmarks + +Current Limitations: + +- New benchmark files are not tracked +- Deleted benchmarks will stick around +- Renamed benchmarks will create a new benchmark and retain the old name + +To get around the above, run `@jog PkgName` to get an updated jogger. + +## Continuous Benchmarking Baked In! + +Install PkgJogger, run benchmarks, and save results to a `*.json.gz` with a +one-line command. + +```shell +julia -e 'using Pkg; Pkg.add("PkgJogger"); using PkgJogger; PkgJogger.ci()' +``` + +What gets done: + +- Constructs a temporary + [benchmarking environment](https://awadell1.github.io/PkgJogger.jl/stable/ci/#Isolated-Benchmarking-Environment) + from `Project.toml` and `benchmark/Project.toml`. +- Creates a [jogger](https://awadell1.github.io/PkgJogger.jl/stable/jogger/) + to run the package's benchmarks. +- Warmup, tune and run all benchmarks. +- Save Benchmarking results and more to a compressed `*.json.gz` file. diff --git a/docs/make.jl b/docs/make.jl index 74c85de..ae7a88a 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -9,6 +9,18 @@ using Example DocMeta.setdocmeta!(PkgJogger, :DocTestSetup, :(using PkgJogger); recursive=true) +# Generate index.md from README.md +index_md = joinpath(@__DIR__, "src", "index.md") +readme_md = joinpath(@__DIR__, "..", "README.md") +open(index_md, "w") do io + write(io, """ + ```@meta + EditURL = "$readme_md" + ``` + """) + write(io, read(readme_md, String)) +end + makedocs(; modules=[PkgJogger, JogExample], authors="Alexius Wadell and contributors", @@ -34,3 +46,6 @@ deploydocs(; repo="github.com/awadell1/PkgJogger.jl", devbranch="main", ) + +# Remove index.md +rm(joinpath(@__DIR__, "src", "index.md")) diff --git a/docs/src/index.md b/docs/src/index.md deleted file mode 100644 index f227d8a..0000000 --- a/docs/src/index.md +++ /dev/null @@ -1,76 +0,0 @@ -```@meta -CurrentModule = PkgJogger -``` - -# PkgJogger - -[![version](https://juliahub.com/docs/PkgJogger/version.svg)](https://juliahub.com/ui/Packages/PkgJogger/AaLEJ) -[![Build Status](https://github.com/awadell1/PkgJogger.jl/workflows/CI/badge.svg)](https://github.com/awadell1/PkgJogger.jl/actions) -[![pkgeval](https://juliahub.com/docs/PkgJogger/pkgeval.svg)](https://juliahub.com/ui/Packages/PkgJogger/AaLEJ) - -PkgJogger provides a framework for running suites of -[BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) benchmarks -without the boilerplate. - -## Just write benchmarks - -Create a `benchmark/bench_*.jl` file, define a -[BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) `suite` and -go! - -```julia -using BenchmarkTools -using AwesomePkg -suite = BenchmarkGroup() -suite["fast"] = @benchmarkable fast_code() -``` - -PkgJogger will wrap each `benchmark/bench_*.jl` in a module and bundle them into `JogAwesomePkg` - -```julia -using AwesomePkg -using PkgJogger - -# Creates the JogAwesomePkg module -@jog AwesomePkg - -# Warmup, tune, and run all of AwesomePkg's benchmarks -JogAwesomePkg.benchmark() -``` - -## Benchmark, Revise, and Benchmark Again! - -PkgJogger uses [Revise.jl](https://github.com/timholy/Revise.jl) to track -changes to your `benchmark/bench_*.jl` files and reload your suite as you edit. -No more waiting for benchmarks to precompile! - -Tracked Changes: - -- Changing your benchmarked function -- Changing benchmarking parameters (i.e. `seconds` or `samples`) -- Adding new benchmarks - -Current Limitations: - -- New benchmark files are not tracked -- Deleted benchmarks will stick around -- Renamed benchmarks will create a new benchmark and retain the old name - -To get around the above, run `@jog PkgName` to get an updated jogger. - -## Continuous Benchmarking Baked In! - -Install PkgJogger, run benchmarks, and save results to a `*.json.gz` with a -one-line command. - -```shell -julia -e 'using Pkg; Pkg.add("PkgJogger"); using PkgJogger; PkgJogger.ci()' -``` - -What gets done: - -- Add the package at `pwd()` to a temporary environment -- If found, instantiate `benchmark/Project.toml` and add to the `LOAD_PATH` -- Add PkgJogger to the environment and build `JogPkgName` for your package -- Warmup, tune and run all benchmarks -- Save Benchmarking results and more to a compressed `*.json.gz` file diff --git a/docs/src/io.md b/docs/src/io.md index 5f0900a..1e23259 100644 --- a/docs/src/io.md +++ b/docs/src/io.md @@ -9,8 +9,8 @@ These methods build on - Additional information such as: - Julia Version, Commit and Build Date - System Information (Essentially everything in `Sys`) - - Timestamp of when the results were saved - - Git Information if saved in a Git Repository + - Timestamp when the results get saved + - Git Information, if run from a Git Repository Overall the resulting files are ~10x smaller, despite capturing additional information. diff --git a/docs/src/jogger.md b/docs/src/jogger.md index 266c084..e202883 100644 --- a/docs/src/jogger.md +++ b/docs/src/jogger.md @@ -1,8 +1,8 @@ # Generated Jogger Modules -At it's core `PkgJogger` uses meta-programming to generate a Jogger module for +At its core, `PkgJogger` uses meta-programming to generate a Jogger module for running a package's benchmarks. For example, calling `@jog` on `Example` gives -a jogger named `JogExample` for running the benchmark suite of `Example`. +a jogger named `JogExample` for running the benchmark suite of `Example`: ```jldoctest julia> using PkgJogger, Example @@ -12,7 +12,7 @@ JogExample ``` -Similarly, `@jog AwesomePkg` would create a module named `JogAwesomePkg`. +Similarly, `@jog AwesomePkg` would create a module named `JogAwesomePkg`: ```@docs PkgJogger.@jog @@ -21,7 +21,7 @@ PkgJogger.@jog ## Jogger Reference Jogger modules provide helper methods for working with their package's -benchmarking suite. For reference, this sections documents the methods for `@jog +benchmarking suite. For reference, this section documents the methods for `@jog Example`. ```@docs