From 97334e144adc4dcdb34e4ec5fbd6bbcd151f6ff8 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Sat, 12 Feb 2022 14:55:24 -0800 Subject: [PATCH 01/27] added continuous benchmarking workflow file --- .github/workflows/continuous-benchmark.yml | 70 ++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/continuous-benchmark.yml diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml new file mode 100644 index 000000000..936a290c3 --- /dev/null +++ b/.github/workflows/continuous-benchmark.yml @@ -0,0 +1,70 @@ +name: Continuous Benchmarking + + +on: + push: + branches: [continuous-benchmarking] + + +permissions: + contents: write + deployments: write + + +jobs: + benchmark: + name: Run C benchmark and record results + runs-on: ubuntu-latest + + steps: + - name: Setup Java JDK + uses: actions/setup-java@v1.4.3 + with: + java-version: 14 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Checkout Lingua Franca repository + uses: actions/checkout@v2 + with: + repository: LDeng0205/lingua-franca + submodules: true + + - name: Checkout reactor-c repository + uses: actions/checkout@v2 + with: + repository: lf-lang/reactor-c + path: org.lflang/src/lib/c/reactor-c + + - name: Install Python dependencies + run: pip3 install -r benchmark/runner/requirements.txt + + - name: Build lfc + run: | + ./gradlew buildLfc + + # LF BENCHMARK RUN + - name: Set LF_PATH + run: | + echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + - name: Run C micro pingpong benchmark + run: | + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark=savina_micro_pingpong target=lf-c + + # Use continuous benchmark action to store result + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Lingua Franca C target Benchmark + tool: customSmallerIsBetter + output-file-path: ./benchmark/runner/benchmark_result.json + # github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '120%' + comment-on-alert: true + fail-on-alert: true + \ No newline at end of file From 0a98ccc6b92d134a3fcc4ed5748edcd3caaebed2 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Sat, 12 Feb 2022 16:02:58 -0800 Subject: [PATCH 02/27] udpated output file path --- .github/workflows/api-docs.yml | 56 +++++++++++----------- .github/workflows/ci.yml | 2 +- .github/workflows/continuous-benchmark.yml | 2 +- .github/workflows/unit-tests.yml | 48 +++++++++---------- 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/.github/workflows/api-docs.yml b/.github/workflows/api-docs.yml index c63befc80..e784f7dfc 100644 --- a/.github/workflows/api-docs.yml +++ b/.github/workflows/api-docs.yml @@ -1,32 +1,32 @@ -name: API docs +# name: API docs -on: - push: - branches: [ main ] +# on: +# push: +# branches: [ main ] -jobs: - build: +# jobs: +# build: - runs-on: macos-latest +# runs-on: macos-latest - steps: - - name: Requirements - run: brew install doxygen - && brew install sphinx-doc - && pip3 install sphinx-rtd-theme - && pip3 install breathe - && pip3 install sphinx-sitemap - && pip3 install exhale - - name: Checkout repo - uses: actions/checkout@1.0.0 - - name: Build docs - run: cd docs - && make doxygen - && cd _build/html - && touch .nojekyll - - name: Deploy - uses: JamesIves/github-pages-deploy-action@releases/v3 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages # The branch the action should deploy to. - FOLDER: docs/_build/html # The folder the action should deploy. +# steps: +# - name: Requirements +# run: brew install doxygen +# && brew install sphinx-doc +# && pip3 install sphinx-rtd-theme +# && pip3 install breathe +# && pip3 install sphinx-sitemap +# && pip3 install exhale +# - name: Checkout repo +# uses: actions/checkout@1.0.0 +# - name: Build docs +# run: cd docs +# && make doxygen +# && cd _build/html +# && touch .nojekyll +# - name: Deploy +# uses: JamesIves/github-pages-deploy-action@releases/v3 +# with: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# BRANCH: gh-pages # The branch the action should deploy to. +# FOLDER: docs/_build/html # The folder the action should deploy. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88659485a..95686c417 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +# name: CI on: push: diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 936a290c3..b39599789 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -60,7 +60,7 @@ jobs: with: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter - output-file-path: ./benchmark/runner/benchmark_result.json + output-file-path: benchmark/runner/benchmark_result.json # github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true # Show alert with commit comment on detecting possible performance regression diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5f74ef6f9..adce001fd 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,27 +1,27 @@ -name: Unit tests +# name: Unit tests -on: - workflow_call: - inputs: - cmake-args: - description: Arguments to pass to cmake - type: string - required: false -jobs: - run: - strategy: - matrix: - platform: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.platform }} +# on: +# workflow_call: +# inputs: +# cmake-args: +# description: Arguments to pass to cmake +# type: string +# required: false +# jobs: +# run: +# strategy: +# matrix: +# platform: [ubuntu-latest, macos-latest, windows-latest] +# runs-on: ${{ matrix.platform }} - steps: - - name: Check out reactor-c repository - uses: actions/checkout@v2 +# steps: +# - name: Check out reactor-c repository +# uses: actions/checkout@v2 - - name: Build and run unit tests ${{ inputs.cmake-args }} - run: | - mkdir build - cd build - cmake .. ${{ inputs.cmake-args }} - cmake --build . - make test +# - name: Build and run unit tests ${{ inputs.cmake-args }} +# run: | +# mkdir build +# cd build +# cmake .. ${{ inputs.cmake-args }} +# cmake --build . +# make test From 063c9210076ef62289ae6a110289fdcdd3755985 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Sun, 13 Feb 2022 16:57:10 -0800 Subject: [PATCH 03/27] modifiede checkout file path --- .github/workflows/api-docs.yml | 56 +++++++++++----------- .github/workflows/ci.yml | 2 +- .github/workflows/continuous-benchmark.yml | 12 ++--- .github/workflows/unit-tests.yml | 48 +++++++++---------- 4 files changed, 59 insertions(+), 59 deletions(-) diff --git a/.github/workflows/api-docs.yml b/.github/workflows/api-docs.yml index e784f7dfc..c63befc80 100644 --- a/.github/workflows/api-docs.yml +++ b/.github/workflows/api-docs.yml @@ -1,32 +1,32 @@ -# name: API docs +name: API docs -# on: -# push: -# branches: [ main ] +on: + push: + branches: [ main ] -# jobs: -# build: +jobs: + build: -# runs-on: macos-latest + runs-on: macos-latest -# steps: -# - name: Requirements -# run: brew install doxygen -# && brew install sphinx-doc -# && pip3 install sphinx-rtd-theme -# && pip3 install breathe -# && pip3 install sphinx-sitemap -# && pip3 install exhale -# - name: Checkout repo -# uses: actions/checkout@1.0.0 -# - name: Build docs -# run: cd docs -# && make doxygen -# && cd _build/html -# && touch .nojekyll -# - name: Deploy -# uses: JamesIves/github-pages-deploy-action@releases/v3 -# with: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# BRANCH: gh-pages # The branch the action should deploy to. -# FOLDER: docs/_build/html # The folder the action should deploy. + steps: + - name: Requirements + run: brew install doxygen + && brew install sphinx-doc + && pip3 install sphinx-rtd-theme + && pip3 install breathe + && pip3 install sphinx-sitemap + && pip3 install exhale + - name: Checkout repo + uses: actions/checkout@1.0.0 + - name: Build docs + run: cd docs + && make doxygen + && cd _build/html + && touch .nojekyll + - name: Deploy + uses: JamesIves/github-pages-deploy-action@releases/v3 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages # The branch the action should deploy to. + FOLDER: docs/_build/html # The folder the action should deploy. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95686c417..88659485a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -# name: CI +name: CI on: push: diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index b39599789..d6234e317 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -33,11 +33,11 @@ jobs: repository: LDeng0205/lingua-franca submodules: true - - name: Checkout reactor-c repository - uses: actions/checkout@v2 - with: - repository: lf-lang/reactor-c - path: org.lflang/src/lib/c/reactor-c + # - name: Checkout reactor-c repository + # uses: actions/checkout@v2 + # with: + # repository: lf-lang/reactor-c + # path: org.lflang/src/lib/c/reactor-c - name: Install Python dependencies run: pip3 install -r benchmark/runner/requirements.txt @@ -60,7 +60,7 @@ jobs: with: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter - output-file-path: benchmark/runner/benchmark_result.json + output-file-path: /benchmark/runner/benchmark_result.json # github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true # Show alert with commit comment on detecting possible performance regression diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index adce001fd..5f74ef6f9 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,27 +1,27 @@ -# name: Unit tests +name: Unit tests -# on: -# workflow_call: -# inputs: -# cmake-args: -# description: Arguments to pass to cmake -# type: string -# required: false -# jobs: -# run: -# strategy: -# matrix: -# platform: [ubuntu-latest, macos-latest, windows-latest] -# runs-on: ${{ matrix.platform }} +on: + workflow_call: + inputs: + cmake-args: + description: Arguments to pass to cmake + type: string + required: false +jobs: + run: + strategy: + matrix: + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} -# steps: -# - name: Check out reactor-c repository -# uses: actions/checkout@v2 + steps: + - name: Check out reactor-c repository + uses: actions/checkout@v2 -# - name: Build and run unit tests ${{ inputs.cmake-args }} -# run: | -# mkdir build -# cd build -# cmake .. ${{ inputs.cmake-args }} -# cmake --build . -# make test + - name: Build and run unit tests ${{ inputs.cmake-args }} + run: | + mkdir build + cd build + cmake .. ${{ inputs.cmake-args }} + cmake --build . + make test From 0df54bf900039f31914657ec03e037a39a18ca07 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Sun, 13 Feb 2022 23:48:59 -0800 Subject: [PATCH 04/27] modified file path to match updated result generation script --- .github/workflows/continuous-benchmark.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index d6234e317..0afb2ed61 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -50,9 +50,9 @@ jobs: - name: Set LF_PATH run: | echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - name: Run C micro pingpong benchmark + - name: Run C micro pingpong and count benchmark run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark=savina_micro_pingpong target=lf-c + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark=savina_micro_pingpong,savina_micro_count target=lf-c # Use continuous benchmark action to store result - name: Store benchmark result @@ -60,11 +60,11 @@ jobs: with: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter - output-file-path: /benchmark/runner/benchmark_result.json + output-file-path: multirun/benchmark_result.json # github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true + auto-push: false # Show alert with commit comment on detecting possible performance regression alert-threshold: '120%' - comment-on-alert: true - fail-on-alert: true + comment-on-alert: false + fail-on-alert: false \ No newline at end of file From c5211ded8b11d92ef193b9b2a4806c0f4f3b8c59 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Mon, 14 Feb 2022 10:28:03 -0800 Subject: [PATCH 05/27] adjusted workflow settings --- .github/workflows/continuous-benchmark.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 0afb2ed61..c68799d06 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -33,12 +33,6 @@ jobs: repository: LDeng0205/lingua-franca submodules: true - # - name: Checkout reactor-c repository - # uses: actions/checkout@v2 - # with: - # repository: lf-lang/reactor-c - # path: org.lflang/src/lib/c/reactor-c - - name: Install Python dependencies run: pip3 install -r benchmark/runner/requirements.txt @@ -61,8 +55,8 @@ jobs: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter output-file-path: multirun/benchmark_result.json - # github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: false + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true # Show alert with commit comment on detecting possible performance regression alert-threshold: '120%' comment-on-alert: false From 85f7845a14e4604f4a1fa249b93955ebbbd8c896 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Mon, 14 Feb 2022 10:41:15 -0800 Subject: [PATCH 06/27] running all benchmarks --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index c68799d06..5726a8d03 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -46,7 +46,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C micro pingpong and count benchmark run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark=savina_micro_pingpong,savina_micro_count target=lf-c + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark="glob(*)" target=lf-c # Use continuous benchmark action to store result - name: Store benchmark result From adb55e53774d12cbbc5d48b07ba7369b28780c5e Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Mon, 14 Feb 2022 20:18:53 -0800 Subject: [PATCH 07/27] tweaked workflow file --- .github/workflows/continuous-benchmark.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 5726a8d03..58a27e45e 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -13,7 +13,7 @@ permissions: jobs: benchmark: - name: Run C benchmark and record results + name: Run C Benchmarks runs-on: ubuntu-latest steps: @@ -44,12 +44,12 @@ jobs: - name: Set LF_PATH run: | echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - name: Run C micro pingpong and count benchmark + - name: Run C Benchmarks run: | python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark="glob(*)" target=lf-c # Use continuous benchmark action to store result - - name: Store benchmark result + - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: name: Lingua Franca C target Benchmark @@ -57,8 +57,9 @@ jobs: output-file-path: multirun/benchmark_result.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true + # Show alert with commit comment on detecting possible performance regression - alert-threshold: '120%' - comment-on-alert: false + alert-threshold: '150%' + comment-on-alert: true fail-on-alert: false \ No newline at end of file From 7398aa447392d82a36dab7897823450aa97f8f3f Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 15 Feb 2022 10:06:21 -0800 Subject: [PATCH 08/27] test --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 58a27e45e..2e245063e 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -59,7 +59,7 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '150%' - comment-on-alert: true + alert-threshold: '200%' + comment-on-alert: false fail-on-alert: false \ No newline at end of file From 45f337be7259221c0d8185557c8dfe8b5200369a Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 15 Feb 2022 20:41:50 -0800 Subject: [PATCH 09/27] changed iteration number --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 2e245063e..fc435c53f 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -46,7 +46,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark="glob(*)" target=lf-c + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c # Use continuous benchmark action to store result - name: Store Benchmark Result From cd2f1d424eb2c00205b12d574f75cbf292f696d5 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 1 Mar 2022 15:19:37 -0800 Subject: [PATCH 10/27] continue on error --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index fc435c53f..8e5c4a7be 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -46,7 +46,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True # Use continuous benchmark action to store result - name: Store Benchmark Result From b8568b15c09c85763e827b4341411c8003b1fed8 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 8 Mar 2022 11:25:59 -0800 Subject: [PATCH 11/27] changed action and runner to russell --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 8e5c4a7be..0b248a56c 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -14,7 +14,7 @@ permissions: jobs: benchmark: name: Run C Benchmarks - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Setup Java JDK @@ -46,7 +46,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=12 benchmark="glob(*)" target=lf-c continue_on_error=True # Use continuous benchmark action to store result - name: Store Benchmark Result From 4201ce176424835b61b25a2b665945384889dbeb Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 8 Mar 2022 12:47:05 -0800 Subject: [PATCH 12/27] checking out current version of reactor-c --- .github/workflows/continuous-benchmark.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 0b248a56c..a2e2f98c2 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -2,9 +2,12 @@ name: Continuous Benchmarking on: + # pull_request: + # types: + # - opened push: branches: [continuous-benchmarking] - + permissions: contents: write @@ -14,7 +17,7 @@ permissions: jobs: benchmark: name: Run C Benchmarks - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Setup Java JDK @@ -31,7 +34,10 @@ jobs: uses: actions/checkout@v2 with: repository: LDeng0205/lingua-franca - submodules: true + - name: Checkout current branch of reactor-c + uses: actions/checkout@v2 + with: + path: org.lflang/src/lib/c/reactor-c - name: Install Python dependencies run: pip3 install -r benchmark/runner/requirements.txt @@ -59,7 +65,7 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' + alert-threshold: '120%' comment-on-alert: false fail-on-alert: false \ No newline at end of file From c30ac24753e59e549e35850574f8f341d88bedd0 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 8 Mar 2022 12:58:51 -0800 Subject: [PATCH 13/27] checking out lf from lf-lang --- .github/workflows/continuous-benchmark.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index a2e2f98c2..b906d1cce 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -2,9 +2,9 @@ name: Continuous Benchmarking on: - # pull_request: - # types: - # - opened + pull_request: + types: + - opened push: branches: [continuous-benchmarking] @@ -33,7 +33,8 @@ jobs: - name: Checkout Lingua Franca repository uses: actions/checkout@v2 with: - repository: LDeng0205/lingua-franca + repository: lf-lang/lingua-franca + ref: continuous-benchmark - name: Checkout current branch of reactor-c uses: actions/checkout@v2 with: @@ -52,7 +53,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=12 benchmark="glob(*)" target=lf-c continue_on_error=True + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True # Use continuous benchmark action to store result - name: Store Benchmark Result @@ -65,7 +66,7 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '120%' - comment-on-alert: false + alert-threshold: '200%' + comment-on-alert: true fail-on-alert: false \ No newline at end of file From 3d4e91e72f68b7e1e9a8bdee93870577c765e7a1 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 8 Mar 2022 13:08:22 -0800 Subject: [PATCH 14/27] fixed file name --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index b906d1cce..07b67eb85 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -53,7 +53,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True + python3 benchmark/runner/run_benchmark.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True json=True # Use continuous benchmark action to store result - name: Store Benchmark Result From 87c2811779022e0a18c257a466539e3e910be381 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Thu, 10 Mar 2022 18:16:25 -0800 Subject: [PATCH 15/27] modified to use benchmark repository --- .github/workflows/continuous-benchmark.yml | 37 ++++++++++++++-------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 07b67eb85..0372884d1 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -14,46 +14,57 @@ permissions: deployments: write -jobs: + +jobs: benchmark: name: Run C Benchmarks - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Setup Java JDK uses: actions/setup-java@v1.4.3 with: java-version: 14 - + - name: Setup Python uses: actions/setup-python@v2 with: python-version: 3.8 + + - name: Checkout benchmark repository + uses: actions/checkout@v2 + with: + repository: lf-lang/benchmarks-lingua-franca + ref: continuous-benchmark + - name: Checkout Lingua Franca repository uses: actions/checkout@v2 with: repository: lf-lang/lingua-franca - ref: continuous-benchmark - - name: Checkout current branch of reactor-c + path: lf + + - name: Checkout current version of reactor-c uses: actions/checkout@v2 with: - path: org.lflang/src/lib/c/reactor-c - + path: lf/org.lflang/src/lib/c/reactor-c + - name: Install Python dependencies - run: pip3 install -r benchmark/runner/requirements.txt + run: pip3 install -r runner/requirements.txt - name: Build lfc run: | + cd lf ./gradlew buildLfc - # LF BENCHMARK RUN - - name: Set LF_PATH + - name: Set LF_PATH and LF_BENCHMARKS_PATH environmental variable run: | - echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV + echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True json=True + python3 runner/run_benchmark.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c json=True # Use continuous benchmark action to store result - name: Store Benchmark Result @@ -67,6 +78,6 @@ jobs: # Show alert with commit comment on detecting possible performance regression alert-threshold: '200%' - comment-on-alert: true + comment-on-alert: false fail-on-alert: false \ No newline at end of file From aa71a558fcc5ce7189c0e4670dfedcc395587452 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Thu, 10 Mar 2022 18:17:45 -0800 Subject: [PATCH 16/27] testing on github machine --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 0372884d1..8a14988ff 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -18,7 +18,7 @@ permissions: jobs: benchmark: name: Run C Benchmarks - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Setup Java JDK From 9105e37c52fd5b40e13221db73ff7f9380f45af6 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Thu, 10 Mar 2022 21:18:24 -0800 Subject: [PATCH 17/27] added comments --- .github/workflows/continuous-benchmark.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 8a14988ff..9d0eb6c46 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -18,7 +18,7 @@ permissions: jobs: benchmark: name: Run C Benchmarks - runs-on: ubuntu-latest + runs-on: ubuntu-latest #FIXME: change to self-hosted after russel is set up. steps: - name: Setup Java JDK @@ -36,7 +36,7 @@ jobs: uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: continuous-benchmark + ref: continuous-benchmark # FIXME: delete this line after merge - name: Checkout Lingua Franca repository uses: actions/checkout@v2 @@ -64,7 +64,7 @@ jobs: - name: Run C Benchmarks run: | - python3 runner/run_benchmark.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c json=True + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 benchmark="glob(*)" target=lf-c json=True # Use continuous benchmark action to store result - name: Store Benchmark Result @@ -77,7 +77,7 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true comment-on-alert: false fail-on-alert: false \ No newline at end of file From d740d3378501a84fd74d5efbc4150045ccaf9fdf Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Sat, 9 Jul 2022 16:51:50 -0700 Subject: [PATCH 18/27] [benchmark] Update cb workflow. --- .github/workflows/continuous-benchmark.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 9d0eb6c46..3148284d7 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -2,11 +2,8 @@ name: Continuous Benchmarking on: - pull_request: - types: - - opened - push: - branches: [continuous-benchmarking] + pull_request: + workflow_dispatch: permissions: From ceab5942ba49cc75552b20ed37930a1377af2395 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Sat, 9 Jul 2022 19:15:34 -0700 Subject: [PATCH 19/27] [benchmarks] Update workflow. --- .github/workflows/continuous-benchmark.yml | 32 ++++++++++------------ 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 3148284d7..d4d497206 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -4,36 +4,29 @@ name: Continuous Benchmarking on: pull_request: workflow_dispatch: - - + + permissions: contents: write deployments: write - -jobs: +jobs: benchmark: name: Run C Benchmarks runs-on: ubuntu-latest #FIXME: change to self-hosted after russel is set up. steps: - - name: Setup Java JDK - uses: actions/setup-java@v1.4.3 - with: - java-version: 14 - - name: Setup Python uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Checkout benchmark repository uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: continuous-benchmark # FIXME: delete this line after merge + ref: cb # FIXME: delete this line after merge - name: Checkout Lingua Franca repository uses: actions/checkout@v2 @@ -41,13 +34,16 @@ jobs: repository: lf-lang/lingua-franca path: lf + - name: Prepare LF build environment + uses: ./lf/.github/actions/prepare-build-env + - name: Checkout current version of reactor-c uses: actions/checkout@v2 with: path: lf/org.lflang/src/lib/c/reactor-c - name: Install Python dependencies - run: pip3 install -r runner/requirements.txt + run: pip3 install -r runner/requirements.txt - name: Build lfc run: | @@ -58,18 +54,21 @@ jobs: run: | echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - + - name: Run C Benchmarks run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 benchmark="glob(*)" target=lf-c json=True - + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 benchmark="glob(*)" target=lf-c + + - name: Collect results + run: python3 runner/collect_results.py latest continuous_benchmarking_results.json + # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter - output-file-path: multirun/benchmark_result.json + output-file-path: continuous_benchmarking_results.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true @@ -77,4 +76,3 @@ jobs: alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true comment-on-alert: false fail-on-alert: false - \ No newline at end of file From 3ac8ecf1711f8610615a70400f52f4ac8c8b01bc Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 13 Jul 2022 14:13:28 -0700 Subject: [PATCH 20/27] REVERT ME --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index d4d497206..858d79096 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -73,6 +73,6 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true - comment-on-alert: false + alert-threshold: '100%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + comment-on-alert: true fail-on-alert: false From 4ca49de79e2ecf57a836ebb67e36f932996ab44e Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 13 Jul 2022 15:09:59 -0700 Subject: [PATCH 21/27] [benchmarks] Benchmark different runtime versions. --- .github/workflows/continuous-benchmark.yml | 28 ++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 858d79096..c2a2ce125 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -55,9 +55,33 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - name: Run C Benchmarks + - name: Run C Benchmarks (multithreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 benchmark="glob(*)" target=lf-c + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 + + - name: Collect results + run: python3 runner/collect_results.py latest continuous_benchmarking_results.json + + # Use continuous benchmark action to store result + - name: Store Benchmark Result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Lingua Franca C target Benchmark + tool: customSmallerIsBetter + output-file-path: continuous_benchmarking_results.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '100%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + comment-on-alert: true + fail-on-alert: false + + - name: Run C Benchmarks (unthreaded) + run: | + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + benchmark="glob(*)" target=lf-c-unthreaded - name: Collect results run: python3 runner/collect_results.py latest continuous_benchmarking_results.json From dd98802221b55602b1cd06a202368da2b755b242 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 14 Jul 2022 15:17:51 -0700 Subject: [PATCH 22/27] [benchmarks] Use size=fast. --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index c2a2ce125..6f759dac4 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -57,7 +57,7 @@ jobs: - name: Run C Benchmarks (multithreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 size=fast \ benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 - name: Collect results @@ -80,7 +80,7 @@ jobs: - name: Run C Benchmarks (unthreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 size=fast \ benchmark="glob(*)" target=lf-c-unthreaded - name: Collect results From b9f341af731bd5502b6a6a0fb2174dba2bd224db Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 10:43:21 -0700 Subject: [PATCH 23/27] Update continuous-benchmark.yml. --- .github/workflows/continuous-benchmark.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 6f759dac4..879798e72 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -61,15 +61,15 @@ jobs: benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 - name: Collect results - run: python3 runner/collect_results.py latest continuous_benchmarking_results.json + run: python3 runner/collect_results.py latest continuous-benchmarking-results-multi-threaded.json # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: - name: Lingua Franca C target Benchmark + name: Lingua Franca C Benchmark -- Multithreaded tool: customSmallerIsBetter - output-file-path: continuous_benchmarking_results.json + output-file-path: continuous-benchmarking-results-multi-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true @@ -84,15 +84,15 @@ jobs: benchmark="glob(*)" target=lf-c-unthreaded - name: Collect results - run: python3 runner/collect_results.py latest continuous_benchmarking_results.json + run: python3 runner/collect_results.py latest continuous-benchmarking-results-single-threaded.json # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: - name: Lingua Franca C target Benchmark + name: Lingua Franca C Benchmark -- Single-Threaded tool: customSmallerIsBetter - output-file-path: continuous_benchmarking_results.json + output-file-path: continuous-benchmarking-results-single-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true From 5f89f934587511b73edbfa4d2a8122684c51e2c3 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 16:25:56 -0700 Subject: [PATCH 24/27] [benchmarks] Adjust threshold for warning. --- .github/workflows/continuous-benchmark.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 879798e72..c1fa4c46f 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -72,9 +72,7 @@ jobs: output-file-path: continuous-benchmarking-results-multi-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '100%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true comment-on-alert: true fail-on-alert: false @@ -97,6 +95,6 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '100%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true comment-on-alert: true fail-on-alert: false From 3812932d03996bb127f55445189072dfca20ac56 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 16:45:17 -0700 Subject: [PATCH 25/27] [benchmarks] Rename a step; update comments. --- .github/workflows/continuous-benchmark.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index c1fa4c46f..654241696 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest #FIXME: change to self-hosted after russel is set up. steps: - - name: Setup Python + - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.8 @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: cb # FIXME: delete this line after merge + ref: cb # FIXME: delete this line after `cb` is merged. - name: Checkout Lingua Franca repository uses: actions/checkout@v2 @@ -63,7 +63,6 @@ jobs: - name: Collect results run: python3 runner/collect_results.py latest continuous-benchmarking-results-multi-threaded.json - # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: @@ -72,7 +71,7 @@ jobs: output-file-path: continuous-benchmarking-results-multi-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold comment-on-alert: true fail-on-alert: false @@ -84,7 +83,6 @@ jobs: - name: Collect results run: python3 runner/collect_results.py latest continuous-benchmarking-results-single-threaded.json - # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: @@ -93,8 +91,6 @@ jobs: output-file-path: continuous-benchmarking-results-single-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold comment-on-alert: true fail-on-alert: false From 72edc94e505b396bbd6ef6ce21345b25edf234ec Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 20 Jul 2022 16:54:15 -0700 Subject: [PATCH 26/27] [benchmarks] Update to match collect_results.py. --- .github/workflows/continuous-benchmark.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 654241696..87cc4b36a 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -57,11 +57,11 @@ jobs: - name: Run C Benchmarks (multithreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 size=fast \ + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 problem_size=small \ benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 - name: Collect results - run: python3 runner/collect_results.py latest continuous-benchmarking-results-multi-threaded.json + run: python3 runner/collect_results.py continuous-benchmarking-results-multi-threaded.json - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 @@ -77,11 +77,11 @@ jobs: - name: Run C Benchmarks (unthreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 size=fast \ + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 problem_size=small \ benchmark="glob(*)" target=lf-c-unthreaded - name: Collect results - run: python3 runner/collect_results.py latest continuous-benchmarking-results-single-threaded.json + run: python3 runner/collect_results.py continuous-benchmarking-results-single-threaded.json - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 From 62c26925b27c32189fbba37f542c5a6f9f01389d Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 28 Jul 2022 16:12:22 -0700 Subject: [PATCH 27/27] [benchmarks] Update ref. --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 87cc4b36a..837872ed5 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: cb # FIXME: delete this line after `cb` is merged. + ref: automated-full-benchmark # FIXME: delete this line - name: Checkout Lingua Franca repository uses: actions/checkout@v2