From 893872063f1727e8ba888a58acc05286c610f935 Mon Sep 17 00:00:00 2001 From: Michael Ingold Date: Fri, 25 Oct 2024 08:10:56 -0400 Subject: [PATCH] Implement benchmarking with AirspeedVelocity.jl (#119) * Setup basic benchmarking * Simplify * Simplify * Add LinearAlgebra dep * Simplify scope structure for debugging * Add CI Action * Reorganize * Use product of test settings to construct individual tests * Formatting changes * Remove explicit LinearAlgebra ref * Combine multiple geometries into same loop * Bugfix * Add N field for evals settings * Add differentials tests * Tidying * Formatting fix * Update .github/workflows/AirspeedVelocity.yml * Update .github/workflows/AirspeedVelocity.yml * Update .github/workflows/AirspeedVelocity.yml * Use new versions of dependencies * Update .github/workflows/AirspeedVelocity.yml * Try to use benchmark script from head branch * Replace deprecated `::set-output` * Apply suggestions from code review Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> * Drop Unitful * Split indices --------- Co-authored-by: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> --- .github/workflows/AirspeedVelocity.yml | 78 ++++++++++++++++++++++++++ benchmark/Project.toml | 9 +++ benchmark/benchmarks.jl | 48 ++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 .github/workflows/AirspeedVelocity.yml create mode 100644 benchmark/Project.toml create mode 100644 benchmark/benchmarks.jl diff --git a/.github/workflows/AirspeedVelocity.yml b/.github/workflows/AirspeedVelocity.yml new file mode 100644 index 00000000..81c3af42 --- /dev/null +++ b/.github/workflows/AirspeedVelocity.yml @@ -0,0 +1,78 @@ +name: Benchmark a Pull Request + +on: + pull_request: + branches: + - main + +permissions: + pull-requests: write + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: "1.10" + - uses: julia-actions/cache@v2 + - name: Extract Package Name from Project.toml + id: extract-package-name + run: | + PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/') + echo "package_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT + - name: Build AirspeedVelocity + env: + JULIA_NUM_THREADS: 2 + run: | + # Lightweight build step, as sometimes the runner runs out of memory: + julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")' + julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")' + - name: Add ~/.julia/bin to PATH + run: | + echo "$HOME/.julia/bin" >> $GITHUB_PATH + - name: Run benchmarks + run: | + echo $PATH + ls -l ~/.julia/bin + mkdir results + benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune + - name: Create plots from benchmarks + run: | + mkdir -p plots + benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/ + - name: Upload plot as artifact + uses: actions/upload-artifact@v4 + with: + name: plots + path: plots + - name: Create markdown table from benchmarks + run: | + benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md + echo '### Benchmark Results' > body.md + echo '' >> body.md + echo '' >> body.md + cat table.md >> body.md + echo '' >> body.md + echo '' >> body.md + echo '### Benchmark Plots' >> body.md + echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md + echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md + + - name: Find Comment + uses: peter-evans/find-comment@v3 + id: fcbenchmark + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmark Results + + - name: Comment on PR + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fcbenchmark.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: body.md + edit-mode: replace \ No newline at end of file diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 00000000..7ab23bea --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,9 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Meshes = "eacbb407-ea5a-433e-ab97-5258b1ca43fa" + +[compat] +BenchmarkTools = "1.5" +LinearAlgebra = "1" +Meshes = "0.50, 0.51, 0.52" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl new file mode 100644 index 00000000..c438e3e2 --- /dev/null +++ b/benchmark/benchmarks.jl @@ -0,0 +1,48 @@ +using BenchmarkTools +using LinearAlgebra +using Meshes +using MeshIntegrals + +const SUITE = BenchmarkGroup() + +############################################################################################ +# Integrals +############################################################################################ + +integrands = ( + (name = "Scalar", f = p -> norm(to(p))), + (name = "Vector", f = p -> fill(norm(to(p)), 3)) +) +rules = ( + (name = "GaussLegendre", rule = GaussLegendre(100), N = 100), + (name = "GaussKronrod", rule = GaussKronrod(), N = 100), + (name = "HAdaptiveCubature", rule = HAdaptiveCubature(), N = 500) +) +geometries = ( + (name = "Meshes.Segment", item = Segment(Point(0, 0, 0), Point(1, 1, 1))), + (name = "Meshes.Sphere", item = Sphere(Point(0, 0, 0), 1.0)) +) + +SUITE["Integrals"] = let s = BenchmarkGroup() + for (int, rule, geometry) in Iterators.product(integrands, rules, geometries) + n1, n2, N = geometry.name, "$(int.name) $(rule.name)", rule.N + s[n1][n2] = @benchmarkable integral($int.f, $geometry.item, $rule.rule) evals=N + end + s +end + +############################################################################################ +# Differentials +############################################################################################ + +sphere = geometries[2].item +differential = MeshIntegrals.differential + +SUITE["Differentials"] = let s = BenchmarkGroup() + s["Jacobian"] = @benchmarkable jacobian($sphere, $(0.5, 0.5)) evals=1000 + s["Differential"] = @benchmarkable differential($sphere, $(0.5, 0.5)) evals=1000 + s +end + +#tune!(SUITE) +#run(SUITE, verbose=true)