-
Notifications
You must be signed in to change notification settings - Fork 346
149 lines (123 loc) · 4.95 KB
/
benchmark_execution_time.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
name: ⏱️ Benchmark execution time comparison with the main branch
on:
issue_comment:
types: [created, edited, deleted]
jobs:
building-pr-branch:
if: (github.event.issue.pull_request != null) && github.event.comment.body == '!github easy-benchmark'
runs-on: ubuntu-20.04
timeout-minutes: 15
steps:
- name: Checkout to PR branch
uses: actions/checkout@v4
- name: Install requirements
run: sudo ./.github/scripts/dependency.sh
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/[email protected]
- name: Install Just
uses: taiki-e/install-action@just
- name: Building PR branch
run: just youki-release
- name: Uploading PR build to artifact
uses: actions/upload-artifact@v4
with:
name: pr-youki
path: ./youki
building-main-branch:
if: (github.event.issue.pull_request != null) && github.event.comment.body == '!github easy-benchmark'
runs-on: ubuntu-20.04
timeout-minutes: 15
steps:
- name: Checkout to main branch
uses: actions/checkout@v4
with:
ref: main
- name: Install requirements
run: sudo ./.github/scripts/dependency.sh
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/[email protected]
- name: Install just
uses: taiki-e/install-action@just
- name: Building main branch
run: just youki-release
- name: Uploading main build to artifact
uses: actions/upload-artifact@v4
with:
name: main-youki
path: ./youki
benchmark-exec:
if: (github.event.issue.pull_request != null) && github.event.comment.body == '!github easy-benchmark'
needs:
- building-pr-branch
- building-main-branch
runs-on: ubuntu-20.04
timeout-minutes: 15
steps:
- name: Setup Linux env
run: |
sudo apt -y update
sudo apt install jq podman
- name: Checkout to PR branch
uses: actions/checkout@v4
- name: Downloading PR build from artifact
uses: actions/download-artifact@v4
with:
name: pr-youki
path: ./pr_youki
- name: Downloading main build from artifact
uses: actions/download-artifact@v4
with:
name: main-youki
path: ./main_youki
- name: Preparing binaries
run: |
mv ./main_youki/youki ./youki_main
mv ./pr_youki/youki ./youki_pr
chmod +x ./youki_main
chmod +x ./youki_pr
- name: Preparing benchmark bundle
run: |
mkdir -p ./benchmark_exec_bundle/rootfs
cd benchmark_exec_bundle
sudo bash -c 'podman export $(podman create busybox) | tar -C rootfs -xvf -'
../youki_main spec
sed -i 's/"sh"/"sh", "-c", "echo Hi, my PID is $$; echo Bye Bye"/' config.json
sed -i 's/"terminal": true/"terminal": false/' config.json
- name: Installing hyperfine
run: |
curl -s https://api.github.com/repos/sharkdp/hyperfine/releases/latest \
| jq -r '.assets[] | select(.name? | match("hyperfine-musl_.*_amd64.deb")) | .browser_download_url' \
| wget -qi - -O hyperfine.deb
sudo dpkg -i hyperfine.deb
- name: Running benchmark
run: |
hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' \
--export-markdown bench_exec_result.md \
--warmup 10 \
--min-runs 100 \
--command-name 'main youki build' \
'sudo ./youki_main run -b benchmark_exec_bundle main_container && sudo ./youki_main delete -f main_container' \
--command-name 'pr youki build' \
'sudo ./youki_pr run -b benchmark_exec_bundle pr_container && sudo ./youki_pr delete -f pr_container'
- name: Adding info into result
run: |
sed -i '1i > commit (${{ github.event.pull_request.head.sha }})\n' bench_exec_result.md
# since the GITHUB_TOKEN is needed to let the bot commit messages in the PR
# but right now it is controlled by the organization.
# TODO: change back to use this when the permission granted
# - name: Writing report to PR comment
# uses: marocchino/sticky-pull-request-comment@v2
# with:
# append: true
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# path: ./bench_exec_result.md
# temp solution, print the benchmark result in the terminal.
# glow is the prettiest markdown rendering tool I know.
# TODO: remove this step when GITHUB_TOKEN permission granted
- name: Printing report in the terminal
run: |
curl -s https://api.github.com/repos/charmbracelet/glow/releases/latest \
| jq -r '.assets[] | select(.name? | match("glow_.*_linux_amd64.deb")) | .browser_download_url' \
| wget -qi - -O glow.deb
sudo dpkg -i glow.deb
glow bench_exec_result.md