test: Regression test for #11200 (#11201) #2689
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build time benchmarks | |
| # Do not run this workflow on pull request since this workflow has permission to modify contents. | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - ci-bench/fix-after-ocaml51 | |
| permissions: | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| jobs: | |
| build: | |
| name: Build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| ocaml-compiler: | |
| - 5.1.x | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Use OCaml ${{ matrix.ocaml-compiler }} | |
| uses: ocaml/setup-ocaml@v3 | |
| with: | |
| ocaml-compiler: ${{ matrix.ocaml-compiler }} | |
| opam-depext: false | |
| # dune doesn't have any additional dependencies so we can build it right | |
| # away this makes it possible to see build errors as soon as possible | |
| - run: opam exec -- make _boot/dune.exe | |
| - name: Install deps on Unix | |
| run: | | |
| opam install . --deps-only --with-test | |
| opam exec -- make dev-deps | |
| # Install hyperfine | |
| wget https://github.com/sharkdp/hyperfine/releases/download/v1.14.0/hyperfine_1.14.0_amd64.deb | |
| sudo dpkg -i hyperfine_1.14.0_amd64.deb | |
| - name: Create watch synthetic benchmark | |
| working-directory: bench | |
| run: opam exec -- ../_boot/dune.exe exec ./gen_synthetic_dune_watch.exe -- synthetic-watch | |
| - name: Run synthetic watch benchmark | |
| working-directory: bench/synthetic-watch | |
| run: ../gen-benchmark.sh 'opam exec -- ../run-synthetic-dune-watch.sh ../../_boot/dune.exe' 'opam exec -- ../../_boot/dune.exe build @all' 'synthetic watch build time (warm, ${{ runner.os }})' > synthetic-benchmark-result.json | |
| - name: Print synthetic watch benchmark results | |
| working-directory: bench/synthetic-watch | |
| run: | | |
| cat bench.json | |
| cat synthetic-benchmark-result.json | |
| - name: Store synthetic watch benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: Synthetic Watch Benchmark | |
| tool: "customSmallerIsBetter" | |
| output-file-path: bench/synthetic-watch/synthetic-benchmark-result.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| # Ratio indicating how worse the current benchmark result is. | |
| # 175% means if last build took 40s and current takes >70s, it will trigger an alert | |
| alert-threshold: "225%" | |
| fail-on-alert: true | |
| # Enable alert commit comment | |
| comment-on-alert: true | |
| # Mention @jchavarri in the commit comment | |
| alert-comment-cc-users: '@jchavarri' | |
| - name: Clone pupilfirst fork | |
| run: git clone --depth 1 https://github.com/jchavarri/pupilfirst.git | |
| - name: Install all deps | |
| working-directory: pupilfirst | |
| run: | | |
| opam install -y . --deps-only | |
| - name: Run pupilfirst benchmark | |
| working-directory: pupilfirst | |
| run: ../bench/gen-benchmark.sh 'opam exec -- ../_boot/dune.exe build --root=. @main' 'opam exec -- ../_boot/dune.exe clean --root=.' 'pupilfirst build time (${{ runner.os }})' > melange-benchmark-result.json | |
| - name: Print pupilfirst benchmark results | |
| working-directory: pupilfirst | |
| run: | | |
| cat bench.json | |
| cat melange-benchmark-result.json | |
| - name: Store melange benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: Melange Benchmark | |
| tool: "customSmallerIsBetter" | |
| output-file-path: pupilfirst/melange-benchmark-result.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| # Ratio indicating how worse the current benchmark result is. | |
| # 150% means if last build took 40s and current takes >60s, it will trigger an alert | |
| alert-threshold: "225%" | |
| fail-on-alert: true | |
| # Enable alert commit comment | |
| comment-on-alert: true | |
| # Mention @jchavarri in the commit comment | |
| alert-comment-cc-users: '@jchavarri' | |
| - name: Create synthetic benchmark | |
| working-directory: bench | |
| run: opam exec -- ../_boot/dune.exe exec ./gen_synthetic.exe -- -n 2000 synthetic | |
| - name: Run cold synthetic benchmark | |
| working-directory: bench/synthetic | |
| run: ../gen-benchmark.sh 'opam exec -- ../../_boot/dune.exe build @all' 'opam exec -- ../../_boot/dune.exe clean' 'synthetic build time (cold, ${{ runner.os }})' > synthetic-benchmark-result.json | |
| - name: Print cold synthetic benchmark results | |
| working-directory: bench/synthetic | |
| run: | | |
| cat bench.json | |
| cat synthetic-benchmark-result.json | |
| - name: Store cold synthetic benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: Synthetic Benchmark | |
| tool: "customSmallerIsBetter" | |
| output-file-path: bench/synthetic/synthetic-benchmark-result.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| # Ratio indicating how worse the current benchmark result is. | |
| # 150% means if last build took 40s and current takes >60s, it will trigger an alert | |
| alert-threshold: "225%" | |
| fail-on-alert: true | |
| # Enable alert commit comment | |
| comment-on-alert: true | |
| # Mention @jchavarri in the commit comment | |
| alert-comment-cc-users: '@jchavarri' | |
| - name: Run warm synthetic benchmark | |
| working-directory: bench/synthetic | |
| run: ../gen-benchmark.sh 'opam exec -- ../../_boot/dune.exe build @all' 'true' 'synthetic build time (warm, ${{ runner.os }})' > synthetic-benchmark-result.json | |
| - name: Print warm synthetic benchmark results | |
| working-directory: bench/synthetic | |
| run: | | |
| cat bench.json | |
| cat synthetic-benchmark-result.json | |
| - name: Store warm synthetic benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: Synthetic Benchmark | |
| tool: "customSmallerIsBetter" | |
| output-file-path: bench/synthetic/synthetic-benchmark-result.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| # Ratio indicating how worse the current benchmark result is. | |
| # 150% means if last build took 40s and current takes >60s, it will trigger an alert | |
| alert-threshold: "225%" | |
| fail-on-alert: true | |
| # Enable alert commit comment | |
| comment-on-alert: true | |
| # Mention @jchavarri in the commit comment | |
| alert-comment-cc-users: '@jchavarri' |