diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000000..33d2697271a --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,74 @@ +name: Performance benchmarks + +on: + pull_request_target: + types: [opened, ready_for_review] + branches: + - main + issue_comment: + types: [created] + +permissions: + issues: write + pull-requests: write + +jobs: + run-benchmarks: + if: > + github.event_name == 'pull_request_target' || + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/rerun-benchmarks')) + runs-on: ubuntu-latest + steps: + - name: Checkout main branch + uses: actions/checkout@v4 + with: + ref: main + repository: EwoutH/mesa + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Add project directory to PYTHONPATH + run: echo "PYTHONPATH=$PYTHONPATH:$(pwd)" >> $GITHUB_ENV + - name: Install dependencies + run: pip install numpy pandas tqdm tabulate + - name: Run benchmarks on main branch + working-directory: benchmarks + run: python global_benchmark.py + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: timings-main + path: benchmarks/timings_1.pickle + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ secrets.GITHUB_TOKEN }} + clean: false + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: timings-main + path: benchmarks + - name: Run benchmarks on PR branch + working-directory: benchmarks + run: python global_benchmark.py + - name: Run compare timings and encode output + working-directory: benchmarks + run: | + TIMING_COMPARISON=$(python compare_timings.py | base64 -w 0) # Base64 encode the output + echo "TIMING_COMPARISON=$TIMING_COMPARISON" >> $GITHUB_ENV + - name: Comment PR + uses: actions/github-script@v7 + with: + script: | + const output = Buffer.from(process.env.TIMING_COMPARISON, 'base64').toString('utf-8'); + const issue_number = context.issue.number; + github.rest.issues.createComment({ + issue_number: issue_number, + owner: context.repo.owner, + repo: context.repo.repo, + body: 'Benchmark Comparison:\n\n' + output + }); diff --git a/benchmarks/global_benchmark.py b/benchmarks/global_benchmark.py index 9547d94a674..f1cf938eab0 100644 --- a/benchmarks/global_benchmark.py +++ b/benchmarks/global_benchmark.py @@ -12,13 +12,13 @@ def run_model(model_class, seed, parameters): start_init = timeit.default_timer() model = model_class(seed=seed, **parameters) - # time.sleep(0.001) + time.sleep(0.001) end_init_start_run = timeit.default_timer() for _ in range(config["steps"]): model.step() - # time.sleep(0.0001) + time.sleep(0.0001) end_run = timeit.default_timer() return (end_init_start_run - start_init), (end_run - end_init_start_run)