-
Notifications
You must be signed in to change notification settings - Fork 1.3k
174 lines (144 loc) · 7.1 KB
/
benchmark.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
name: Benchmark
on:
pull_request:
branches: ['*']
paths:
- .github/workflows/benchmark.yml
schedule:
- cron: '0 4 * * 1-5' # Mon-Fri 4AM UTC
workflow_dispatch: {}
concurrency: 'benchmarks' # Ensure only one of this runs at a time
jobs:
run-benchmarks:
name: Clickhouse queries
runs-on: ubuntu-20.04
environment: clickhouse-benchmarks
# Benchmarks are expensive to run so we only run them (periodically) against master branch and for PRs labeled `performance`
if: ${{ github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'performance') }}
env:
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
REDIS_URL: 'redis://localhost'
DEBUG: '1'
CLICKHOUSE_DATABASE: posthog
CLICKHOUSE_HOST: ${{ secrets.BENCHMARKS_CLICKHOUSE_HOST }}
CLICKHOUSE_USER: ${{ secrets.BENCHMARKS_CLICKHOUSE_USER }}
CLICKHOUSE_PASSWORD: ${{ secrets.BENCHMARKS_CLICKHOUSE_PASSWORD }}
CLICKHOUSE_SECURE: 'false'
CLICKHOUSE_VERIFY: 'false'
SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
BENCHMARK: '1'
steps:
- uses: actions/checkout@v3
with:
# Checkout repo with full history
fetch-depth: 0
- name: Check out PostHog/benchmarks-results repo
uses: actions/checkout@v3
with:
path: ee/benchmarks/results
repository: PostHog/benchmark-results
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- name: Stop/Start stack with Docker Compose
run: |
docker compose -f docker-compose.dev.yml down
docker compose -f docker-compose.dev.yml up -d
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.10.10
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
- uses: syphar/restore-virtualenv@v1
id: cache-benchmark-tests
- uses: syphar/restore-pip-download-cache@v1
if: steps.cache-benchmark-tests.outputs.cache-hit != 'true'
- name: Install SAML (python3-saml) dependencies
shell: bash
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
- name: Install python dependencies
if: steps.cache-benchmark-tests.outputs.cache-hit != 'true'
run: |
python -m pip install -r requirements-dev.txt
python -m pip install -r requirements.txt
- name: Install asv
run: python -m pip install asv==0.5.1 virtualenv
- name: Set up PostHog
run: |
python manage.py migrate & wait
python manage.py setup_dev --no-data
- name: Configure benchmarks
run: asv machine --config ee/benchmarks/asv.conf.json --yes --machine ci-benchmarks
- name: Run benchmarks
run: asv run --config ee/benchmarks/asv.conf.json --show-stderr --strict
- name: Compare results
run: |
asv compare $(cat ee/benchmarks/results/last-master-commit) HEAD --config ee/benchmarks/asv.conf.json --factor 1.2 | tee pr_vs_master.txt
asv compare $(cat ee/benchmarks/results/last-master-commit) HEAD --config ee/benchmarks/asv.conf.json --factor 1.2 --only-changed | tee pr_vs_master_changed.txt
- name: Save last benchmarked commit
if: ${{ github.ref == 'refs/heads/master' }}
run: echo "${{ github.sha }}" | tee ee/benchmarks/results/last-master-commit
- name: Generate HTML report of results
if: ${{ github.ref == 'refs/heads/master' }}
run: asv publish --config ee/benchmarks/asv.conf.json
- name: Commit update for benchmark results
if: ${{ github.repository == 'PostHog/posthog' && github.ref == 'refs/heads/master' }}
uses: stefanzweifel/git-auto-commit-action@v4
with:
repository: ee/benchmarks/results
branch: master
commit_message: 'Save benchmark results'
commit_user_name: PostHog Bot
commit_user_email: [email protected]
commit_author: PostHog Bot <[email protected]>
- name: Upload results as artifacts
uses: actions/upload-artifact@v3
with:
name: benchmarks
path: |
pr_vs_master.txt
pr_vs_master_changed.txt
- name: Read benchmark output
if: ${{ github.event_name == 'pull_request' }}
id: pr_vs_master_changed
uses: juliangruber/[email protected]
with:
path: pr_vs_master_changed.txt
- name: Read benchmark output (full)
if: ${{ github.event_name == 'pull_request' }}
id: pr_vs_master
uses: juliangruber/[email protected]
with:
path: pr_vs_master.txt
- name: Find Comment
if: ${{ github.event_name == 'pull_request' }}
uses: peter-evans/find-comment@v1
id: fc
with:
issue-number: ${{ github.event.number }}
comment-author: 'github-actions[bot]'
body-includes: ClickHouse query benchmark results from GitHub Actions
- name: Create or update comment
if: ${{ github.event_name == 'pull_request' }}
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.number }}
body: |
ClickHouse query benchmark results from GitHub Actions
Lower numbers are good, higher numbers are bad. A ratio less than 1
means a speed up and greater than 1 means a slowdown. Green lines
beginning with `+` are slowdowns (the PR is slower then master or
master is slower than the previous release). Red lines beginning
with `-` are speedups. Blank means no changes.
Significantly changed benchmark results (PR vs master)
```diff
${{ steps.pr_vs_master_changed.outputs.content }}
```
<details>
<summary>Click to view full benchmark results</summary>
```diff
${{ steps.pr_vs_master.outputs.content }}
```
</details>
edit-mode: replace