-
Notifications
You must be signed in to change notification settings - Fork 978
313 lines (266 loc) · 11 KB
/
ci-taptests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
name: CI-taptests
on:
push:
branches: [ "v2.x" ]
paths-ignore:
- '.github/**'
- '**.md'
pull_request:
branches: [ "v2.x" ]
paths-ignore:
- '.github/**'
- '**.md'
# schedule:
# - cron: '15 13 * * 3'
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-22.04
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Install dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli
sudo pip3 install fastcov
wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb
sudo dpkg -i orchestrator-client_3.2.6_amd64.deb
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
wget https://repo.mysql.com/mysql-apt-config_0.8.24-1_all.deb
sudo dpkg -i mysql-apt-config_0.8.24-1_all.deb
sudo apt-get update -y
sudo apt-cache policy mysql-shell
sudo apt-get install -y mysql-shell
sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list
sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list
sudo apt-get update -y
sudo apt-cache policy libmysqlclient-dev
sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7*
- name: Checkout proxysql
uses: actions/checkout@v3
with:
repository: 'sysown/proxysql'
# ref: 'v2.x'
fetch-depth: 0
path: 'proxysql'
- name: Set GIT_VERSION
run: |
cd proxysql/
git fetch --tags --force
echo "GIT_VERSION=$(git describe --long --abbrev=7)" >> $GITHUB_ENV
- name: Checkout jenkins_build_scripts
uses: actions/checkout@v3
with:
repository: 'proxysql/jenkins-build-scripts'
ref: 'kubernetes'
# fetch-depth: 0
path: 'jenkins-build-scripts'
token: ${{ secrets.GH_TOKEN }}
submodules: 'false'
- name: Patch TAP-tests
run: |
# apply patches
for PATCH in $(cd jenkins-build-scripts/test-scripts/patches; find . -type f); do
if [[ $PATCH =~ \.patch$ ]]; then
patch proxysql/${PATCH%.patch} jenkins-build-scripts/test-scripts/patches/${PATCH} || true
elif [[ ! -f jenkins-build-scripts/test-scripts/patches/${PATCH#./}.patch ]]; then
cp -v jenkins-build-scripts/test-scripts/patches/${PATCH#./} proxysql/${PATCH#./} || true
fi
done
# patch multi port listening
cd proxysql/
sed -i '61 s/"0.0.0.0:6032"/ current.c_str()/' test/tap/tests/admin-listen_on_unix-t.cpp
sed -i 's/:6035/:6036/' test/tap/tests/test_sqlite3_server-t.cpp
- name: Build
run: |
cd proxysql/
# fix compile issues
#grep -rl "<curl/curl.h>" test/ | xargs sed -i 's|<curl/curl.h>|"curl/curl.h"|'
#sed -i 's|-I/usr/include/mysql |-I/usr/include/mysql -I$(CURL_IDIR) |' test/tap/tests/Makefile
# build proxysql
sed -i "/command/c \ command: bash -l -c 'cd /opt/proxysql && make debug_clickhouse WITHGCOV=1'" docker-compose.yml
#cat docker-compose.yml
make ubuntu22-dbg
# build tap tests
sed -i "/command/c \ command: bash -l -c 'cd /opt/proxysql && make build_tap_test_debug WITHGCOV=1'" docker-compose.yml
#cat docker-compose.yml
make ubuntu22-dbg
# remove irrelevant tests (missing infra)
cd test/tap/tests
rm -f *clickhouse*-t
#rm -f *cluster*-t
rm -f *binlog*-t
rm -f *mariadb*-t
rm -f reg_test_3992_fast_forward_malformed_packet*-t
rm -f test_cluster_sync-t
rm -f test_cluster_sync_mysql_servers-t
rm -f test_read_only_actions_offline_hard_servers-t
# remove always failing tests
rm -f test_rw_binary_data-t
rm -f reg_test_3603-stmt_metadata-t
rm -f test_log_last_insert_id-t
rm -f test_session_status_flags-t
rm -f set_testing-240-t
rm -f reg_test_fast_forward_split_packet-t
rm -f test_com_reset_connection_com_change_user-t
rm -f test_cluster1-t
rm -f test_server_sess_status-t
# remove occasionaly failing tests
rm -f max_connections_ff-t
rm -f test_backend_conn_ping-t
- name: Prepare tests
id: set-matrix
if: ${{ always() }}
run: |
find proxysql/src -type f -executable -print > proxysql/executable.txt
find proxysql/test -type f -executable -print >> proxysql/executable.txt
MATRIX="[ "$(find proxysql/test/tap/tests/ -type f -name '*-t' -executable -printf "'%f', ")" ]"
#echo "matrix=${MATRIX}"
echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
- name: Archive artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v3
with:
name: ci-taptests-${{ env.GIT_VERSION }}-run#${{ github.run_number }}
path: |
proxysql/src/
proxysql/test/
proxysql/deps/libssl/openssl-*/include/openssl
proxysql/executable.txt
###############################################################################################################
test:
runs-on: ubuntu-22.04
needs: [ build ]
strategy:
fail-fast: false
matrix:
# testgroup: [ '.*admin.*-t|.*basic.*-t|.*charset.*-t|.*firewall.*-t|.*kill.*-t|.*max.*-t', 'multiple.*-t', '.*mysql.*-t', '.*prepare.*-t', 'reg_test.*-t', 'save.*-t', '.*set.*-t', '.*sqlite.*-t', '.*ssl.*-t', 'test_.*-t' ]
# testgroup: [ '' ]
# testgroup: [ 'kill_connection3-t', 'reg_test_3847_admin_lock-t', 'set_testing-240-t', 'max_connections_ff-t', 'test_server_sess_status-t' ]
testgroup: ${{ fromJson(needs.build.outputs.matrix) }}
steps:
- name: Install dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli
sudo pip3 install fastcov
wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb
sudo dpkg -i orchestrator-client_3.2.6_amd64.deb
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
wget https://repo.mysql.com/mysql-apt-config_0.8.24-1_all.deb
sudo dpkg -i mysql-apt-config_0.8.24-1_all.deb
sudo apt-get update -y
sudo apt-cache policy mysql-shell
sudo apt-get install -y mysql-shell
sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list
sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list
sudo apt-get update -y
sudo apt-cache policy libmysqlclient-dev
sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7*
- name: Checkout proxysql
uses: actions/checkout@v3
with:
repository: 'sysown/proxysql'
# ref: 'v2.x'
fetch-depth: 0
path: 'proxysql'
- name: Set GIT_VERSION
run: |
cd proxysql/
git fetch --tags --force
echo "GIT_VERSION=$(git describe --long --abbrev=7)" >> $GITHUB_ENV
- name: Download build
uses: actions/download-artifact@v3
with:
name: ci-taptests-${{ env.GIT_VERSION }}-run#${{ github.run_number }}
path: proxysql
- name: Fix build
run: |
cat proxysql/executable.txt | xargs -n1 chmod +x
sudo ln -s ${{ github.workspace }}/proxysql /opt/proxysql
- name: Checkout jenkins_build_scripts
uses: actions/checkout@v3
with:
repository: 'proxysql/jenkins-build-scripts'
ref: 'kubernetes'
# fetch-depth: 0
path: 'jenkins-build-scripts'
token: ${{ secrets.GH_TOKEN }}
submodules: 'false'
- name: Configure env.sh
run: |
cd jenkins-build-scripts
# configure paths
sed -i "s|JENKINS_SCRIPTS_PATH=.*|JENKINS_SCRIPTS_PATH=${{ github.workspace }}/jenkins-build-scripts|" env.sh
sed -i "s|WORKSPACE=.*|WORKSPACE=${{ github.workspace }}/proxysql|" env.sh
- name: Configure test/env.sh
run: |
cd proxysql
# select tests
echo "export TEST_PY_INTERNAL=0" >> test/env.sh
echo "export TEST_PY_BENCHMARK=0" >> test/env.sh
echo "export TEST_PY_CHUSER=0" >> test/env.sh
echo "export TEST_PY_STATS=0" >> test/env.sh
echo "export TEST_PY_TAP=1" >> test/env.sh
echo "export TEST_PY_TAPINT=0" >> test/env.sh
echo "export TEST_PY_FAILOVER=0" >> test/env.sh
echo 'export TEST_PY_TAP_INCL="${{ matrix.testgroup }}"' >> test/env.sh
#echo 'export TEST_PY_TAP_EXCL=""' >> test/env.sh
echo 'export MALLOC_CONF="retain:false"' >> test/env.sh
echo 'export' >> test/env.sh
cat test/env.sh
- name: Docker-hoster
run: |
cd ${{ github.workspace }}/jenkins-build-scripts/infra-docker-hoster
docker-compose up -d
- name: Run proxysql
run: |
set -x
set +e
cd ${{ github.workspace }}/jenkins-build-scripts
source ./env.sh
./cluster_start.bash
sleep 10
cd ${{ github.workspace }}/proxysql
mkdir -p ci_infra_logs/regular_infra/proxysql
cd src
mkdir coverage_reports
(./proxysql --clickhouse-server --sqlite3-server --idle-threads -f -c "$DOCKER_SCRIPT_PATH/conf/proxysql/proxysql.cnf" -D $REGULAR_INFRA_DATADIR &> $REGULAR_INFRA_DATADIR/proxysql.log) &
sleep 10
mysql -uadmin -padmin -h127.0.0.1 -P6032 -e "SELECT version();"
cd ${{ github.workspace }}/jenkins-build-scripts
./cluster_init.bash
sleep 10
- name: Infra
run: |
cd ${{ github.workspace }}/jenkins-build-scripts
source ./env.sh
cd ${{ github.workspace }}/jenkins-build-scripts/infra-mysql57
sed -i "s/\${INFRA}-\${CONTAINER}-1/\${INFRA}_\${CONTAINER}_1/" docker-compose-init.bash
./docker-compose-init.bash
docker ps -a
cat /etc/hosts
- name: TAP-tests
run: |
set +e
cd ${{ github.workspace }}/jenkins-build-scripts
source ./env.sh
env | sort
sudo -E ./test-scripts/bin/proxysql-tester.py
RC=$?
exit $RC
- name: Fix premissions
if: ${{ failure() }}
run: |
sudo chmod -R 777 ${{ github.workspace }}/*
echo "${{ matrix.testgroup }}" | tr -d '*|' | xargs -0 printf 'TESTGROUP=%s' >> $GITHUB_ENV
- name: Archive artifacts
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: ci-selftests-${{ env.GIT_VERSION }}-run#${{ github.run_number }}-${{ env.TESTGROUP }}
path: |
proxysql/ci_gcov_logs/
proxysql/ci_infra_logs/
proxysql/ci_tests_logs/