-
Notifications
You must be signed in to change notification settings - Fork 413
173 lines (149 loc) · 6.72 KB
/
bench.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
name: Build time benchmarks
# Do not run this workflow on pull request since this workflow has permission to modify contents.
on:
push:
branches:
- main
- ci-bench/fix-after-ocaml51
permissions:
# deployments permission to deploy GitHub pages website
deployments: write
# contents permission to update benchmark contents in gh-pages branch
contents: write
jobs:
build:
name: Build
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
ocaml-compiler:
- 5.1.x
runs-on: ${{ matrix.os }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Use OCaml ${{ matrix.ocaml-compiler }}
uses: ocaml/setup-ocaml@v3
with:
ocaml-compiler: ${{ matrix.ocaml-compiler }}
opam-depext: false
# dune doesn't have any additional dependencies so we can build it right
# away this makes it possible to see build errors as soon as possible
- run: opam exec -- make _boot/dune.exe
- name: Install deps on Unix
run: |
opam install . --deps-only --with-test
opam exec -- make dev-deps
# Install hyperfine
wget https://github.com/sharkdp/hyperfine/releases/download/v1.14.0/hyperfine_1.14.0_amd64.deb
sudo dpkg -i hyperfine_1.14.0_amd64.deb
- name: Create watch synthetic benchmark
working-directory: bench
run: opam exec -- ../_boot/dune.exe exec ./gen_synthetic_dune_watch.exe -- synthetic-watch
- name: Run synthetic watch benchmark
working-directory: bench/synthetic-watch
run: ../gen-benchmark.sh 'opam exec -- ../run-synthetic-dune-watch.sh ../../_boot/dune.exe' 'opam exec -- ../../_boot/dune.exe build @all' 'synthetic watch build time (warm, ${{ runner.os }})' > synthetic-benchmark-result.json
- name: Print synthetic watch benchmark results
working-directory: bench/synthetic-watch
run: |
cat bench.json
cat synthetic-benchmark-result.json
- name: Store synthetic watch benchmark result
uses: benchmark-action/github-action-benchmark@v1
with:
name: Synthetic Watch Benchmark
tool: "customSmallerIsBetter"
output-file-path: bench/synthetic-watch/synthetic-benchmark-result.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
# Ratio indicating how worse the current benchmark result is.
# 175% means if last build took 40s and current takes >70s, it will trigger an alert
alert-threshold: "225%"
fail-on-alert: true
# Enable alert commit comment
comment-on-alert: true
# Mention @jchavarri in the commit comment
alert-comment-cc-users: '@jchavarri'
- name: Clone pupilfirst fork
run: git clone --depth 1 https://github.com/jchavarri/pupilfirst.git
- name: Install all deps
working-directory: pupilfirst
run: |
opam install -y . --deps-only
- name: Run pupilfirst benchmark
working-directory: pupilfirst
run: ../bench/gen-benchmark.sh 'opam exec -- ../_boot/dune.exe build --root=. @main' 'opam exec -- ../_boot/dune.exe clean --root=.' 'pupilfirst build time (${{ runner.os }})' > melange-benchmark-result.json
- name: Print pupilfirst benchmark results
working-directory: pupilfirst
run: |
cat bench.json
cat melange-benchmark-result.json
- name: Store melange benchmark result
uses: benchmark-action/github-action-benchmark@v1
with:
name: Melange Benchmark
tool: "customSmallerIsBetter"
output-file-path: pupilfirst/melange-benchmark-result.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
# Ratio indicating how worse the current benchmark result is.
# 150% means if last build took 40s and current takes >60s, it will trigger an alert
alert-threshold: "225%"
fail-on-alert: true
# Enable alert commit comment
comment-on-alert: true
# Mention @jchavarri in the commit comment
alert-comment-cc-users: '@jchavarri'
- name: Create synthetic benchmark
working-directory: bench
run: opam exec -- ../_boot/dune.exe exec ./gen_synthetic.exe -- -n 2000 synthetic
- name: Run cold synthetic benchmark
working-directory: bench/synthetic
run: ../gen-benchmark.sh 'opam exec -- ../../_boot/dune.exe build @all' 'opam exec -- ../../_boot/dune.exe clean' 'synthetic build time (cold, ${{ runner.os }})' > synthetic-benchmark-result.json
- name: Print cold synthetic benchmark results
working-directory: bench/synthetic
run: |
cat bench.json
cat synthetic-benchmark-result.json
- name: Store cold synthetic benchmark result
uses: benchmark-action/github-action-benchmark@v1
with:
name: Synthetic Benchmark
tool: "customSmallerIsBetter"
output-file-path: bench/synthetic/synthetic-benchmark-result.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
# Ratio indicating how worse the current benchmark result is.
# 150% means if last build took 40s and current takes >60s, it will trigger an alert
alert-threshold: "225%"
fail-on-alert: true
# Enable alert commit comment
comment-on-alert: true
# Mention @jchavarri in the commit comment
alert-comment-cc-users: '@jchavarri'
- name: Run warm synthetic benchmark
working-directory: bench/synthetic
run: ../gen-benchmark.sh 'opam exec -- ../../_boot/dune.exe build @all' 'true' 'synthetic build time (warm, ${{ runner.os }})' > synthetic-benchmark-result.json
- name: Print warm synthetic benchmark results
working-directory: bench/synthetic
run: |
cat bench.json
cat synthetic-benchmark-result.json
- name: Store warm synthetic benchmark result
uses: benchmark-action/github-action-benchmark@v1
with:
name: Synthetic Benchmark
tool: "customSmallerIsBetter"
output-file-path: bench/synthetic/synthetic-benchmark-result.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
# Ratio indicating how worse the current benchmark result is.
# 150% means if last build took 40s and current takes >60s, it will trigger an alert
alert-threshold: "225%"
fail-on-alert: true
# Enable alert commit comment
comment-on-alert: true
# Mention @jchavarri in the commit comment
alert-comment-cc-users: '@jchavarri'