[CN-Test-Gen] Add more failing CI tests #7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CN Benchmarks | |
| on: | |
| push: | |
| branches: | |
| - master | |
| env: | |
| CERBERUS_IMAGE_ID: ghcr.io/rems-project/cerberus/cn:release | |
| permissions: | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| # cancel in-progress job when a new push is performed | |
| concurrency: | |
| group: ci-bench-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark: | |
| name: Performance benchmarks | |
| strategy: | |
| matrix: | |
| # version: [4.12.0, 4.14.1] | |
| version: [4.14.1] | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: System dependencies (ubuntu) | |
| run: | | |
| sudo apt install build-essential libgmp-dev z3 opam jq | |
| - name: Restore cached opam | |
| id: cache-opam-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.opam | |
| key: ${{ matrix.version }} | |
| - name: Setup opam | |
| if: steps.cache-opam-restore.outputs.cache-hit != 'true' | |
| run: | | |
| opam init --yes --no-setup --shell=sh --compiler=${{ matrix.version }} | |
| opam install --deps-only --yes ./cerberus-lib.opam | |
| opam switch create with_coq ${{ matrix.version }} | |
| - name: Save cached opam | |
| if: steps.cache-opam-restore.outputs.cache-hit != 'true' | |
| id: cache-opam-save | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ~/.opam | |
| key: ${{ steps.cache-opam-restore.outputs.cache-primary-key }} | |
| - name: Install Cerberus | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| opam pin --yes --no-action add cerberus-lib . | |
| opam pin --yes --no-action add cerberus . | |
| opam install --yes cerberus | |
| - name: Download cvc5 release | |
| uses: robinraju/release-downloader@v1 | |
| with: | |
| repository: cvc5/cvc5 | |
| tag: cvc5-1.2.0 | |
| fileName: cvc5-Linux-x86_64-static.zip | |
| - name: Unzip and install cvc5 | |
| run: | | |
| unzip cvc5-Linux-x86_64-static.zip | |
| chmod +x cvc5-Linux-x86_64-static/bin/cvc5 | |
| sudo cp cvc5-Linux-x86_64-static/bin/cvc5 /usr/local/bin/ | |
| - name: Install CN | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| opam pin --yes --no-action add cn . | |
| opam install --yes cn ocamlformat.0.26.2 | |
| - name: Run benchmarks | |
| run: | | |
| opam switch ${{ matrix.version }} | |
| eval $(opam env --switch=${{ matrix.version }}) | |
| cd tests; SOLVER='z3' ./run-ci-benchmarks.sh; SOLVER='cvc5' ./run-ci-benchmarks.sh | |
| cd .. | |
| - name: Store benchmark result | |
| uses: GaloisInc/github-action-benchmark@47b8b8960c7ed9a55d1db3326ae1ea69aa302380 | |
| with: | |
| name: CN Benchmarks | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: | | |
| { | |
| "z3": "tests/benchmark-data-z3.json", | |
| "cvc5": "tests/benchmark-data-cvc5.json" | |
| } | |
| # Access token to deploy GitHub Pages branch | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Push and deploy GitHub pages branch automatically | |
| auto-push: true |