-
-
Notifications
You must be signed in to change notification settings - Fork 23
442 lines (420 loc) · 19.6 KB
/
ci-sage.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
name: Run Sage CI for Linux/Cygwin/macOS
## This GitHub Actions workflow provides:
##
## - portability testing, by building and testing this project on many platforms
## (Linux variants and Cygwin), each with two configurations (installed packages),
##
## - continuous integration, by building and testing other software
## that depends on this project.
##
## It runs on every pull request and push of a tag to the GitHub repository.
##
## The testing can be monitored in the "Actions" tab of the GitHub repository.
##
## After all jobs have finished (or are canceled) and a short delay,
## tar files of all logs are made available as "build artifacts".
##
## This GitHub Actions workflow uses the portability testing framework
## of SageMath (https://www.sagemath.org/). For more information, see
## https://doc.sagemath.org/html/en/developer/portability_testing.html
## The workflow consists of two jobs:
##
## - First, it builds a source distribution of the project
## and generates a script "update-pkgs.sh". It uploads them
## as a build artifact named upstream.
##
## - Second, it checks out a copy of the SageMath source tree.
## It downloads the upstream artifact and replaces the project's
## package in the SageMath distribution by the newly packaged one
## from the upstream artifact, by running the script "update-pkgs.sh".
## Then it builds a small portion of the Sage distribution.
##
## Many copies of the second step are run in parallel for each of the tested
## systems/configurations.
#on: [push, pull_request]
on:
pull_request:
types: [opened, synchronize]
push:
tags:
- '*'
workflow_dispatch:
# Allow to run manually
env:
# Ubuntu packages to install so that the project's "setup.py sdist" can succeed
DIST_PREREQ: python3-setuptools autoconf
# Name of this project in the Sage distribution
SPKG: cysignals
# Sage distribution packages to build
TARGETS_PRE: build/make/Makefile
TARGETS: SAGE_CHECK=no SAGE_CHECK_PACKAGES="cysignals,cypari" cysignals cypari
TARGETS_OPTIONAL: build/make/Makefile
# Standard setting: Test the current beta release of Sage:
SAGE_REPO: sagemath/sage
SAGE_REF: develop
# Temporarily test with the branch from sage ticket
# (this is a no-op after that ticket is merged)
SAGE_TRAC_GIT: https://github.com/sagemath/sagetrac-mirror.git
SAGE_TICKET: 32579
REMOVE_PATCHES: "*"
jobs:
dist:
runs-on: ubuntu-latest
steps:
- name: Check out ${{ env.SPKG }}
uses: actions/checkout@v2
with:
path: build/pkgs/${{ env.SPKG }}/src
- name: Install prerequisites
run: |
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install $DIST_PREREQ
- name: Run make dist, prepare upstream artifact
run: |
(cd build/pkgs/${{ env.SPKG }}/src && autoreconf -fi && python3 setup.py sdist) \
&& mkdir -p upstream && cp build/pkgs/${{ env.SPKG }}/src/dist/*.tar.gz upstream/${{ env.SPKG }}-git.tar.gz \
&& echo "sage-package create ${{ env.SPKG }} --version git --tarball ${{ env.SPKG }}-git.tar.gz --type=standard" > upstream/update-pkgs.sh \
&& if [ -n "${{ env.REMOVE_PATCHES }}" ]; then echo "(cd ../build/pkgs/${{ env.SPKG }}/patches && rm -f ${{ env.REMOVE_PATCHES }}; :)" >> upstream/update-pkgs.sh; fi \
&& ls -l upstream/
- uses: actions/upload-artifact@v2
with:
path: upstream
name: upstream
cygwin-without-sage:
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
python-version-start: [python-3]
python-version: [7, 8, 9]
steps:
- run: |
git config --global core.autocrlf false
git config --global core.symlinks true
- name: Set up the repository
uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
- name: install cygwin and minimal prerequisites with choco
shell: bash {0}
run: |
choco --version
choco install make autoconf gcc-core gcc-g++ python3${{ matrix.python-version }}-devel --source cygwin
- name: install dependencies
run: |
C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && python3.${{ matrix.python-version }} -m pip install -U setuptools cysignals Cython Sphinx flake8'
- name: install
run: |
C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && python3.${{ matrix.python-version }} setup.py build_ext -i'
- name: test
run: |
C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make configure && ./configure }}'
C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make install PYTHON=python3.${{ matrix.python-version }}'
C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make check-all PYTHON=python3.${{ matrix.python-version }}'
C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make distcheck PYTHON=python3.${{ matrix.python-version }}'
cygwin:
env:
STAGE: i-a
LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
MAKE: make -j8
SAGE_NUM_THREADS: 3
SAGE_CHECK: yes
SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
CYGWIN: winsymlinks:native
CONFIGURE_ARGS: --enable-experimental-packages --enable-download-from-upstream-url
SAGE_FAT_BINARY: yes
SAGE_LOCAL: /opt/sage-cysignals-${{ github.sha }}
runs-on: windows-latest
needs: [dist, cygwin-without-sage]
strategy:
fail-fast: false
matrix:
pkgs: [minimal, standard]
steps:
- run: |
git config --global core.autocrlf false
git config --global core.symlinks true
- name: install cygwin with choco
shell: bash {0}
run: |
choco --version
choco install git python3 --source cygwin
- name: Check out SageMath
uses: actions/checkout@v2
with:
repository: ${{ env.SAGE_REPO }}
ref: ${{ env.SAGE_REF }}
fetch-depth: 2000
if: env.SAGE_REPO != ''
- name: Check out git-trac-command
uses: actions/checkout@v2
with:
repository: sagemath/git-trac-command
path: git-trac-command
if: env.SAGE_TRAC_GIT != ''
- name: Check out SageMath from trac.sagemath.org
shell: bash {0}
# Random sleep and retry to limit the load on trac.sagemath.org
run: |
git config --global user.email "[email protected]"
git config --global user.name "ci-sage workflow"
if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch $SAGE_TICKET; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
if: env.SAGE_TRAC_GIT != ''
- uses: actions/download-artifact@v2
with:
path: upstream
name: upstream
- name: install minimal prerequisites with choco
shell: bash {0}
run: |
choco --version
PACKAGES="python38 python38-pip"
choco install $PACKAGES --source cygwin
- name: Update Sage packages from upstream artifact
run: |
C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && ls -l upstream/ && export PATH="$(pwd)/build/bin:$PATH:/usr/local/bin:/usr/bin" && (cd upstream && bash -x update-pkgs.sh) && git diff'
- name: tox
run: |
C:\\tools\\cygwin\\bin\\bash -l -x -c 'python3.8 -m pip install tox'
C:\\tools\\cygwin\\bin\\bash -l -x -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && PREFIX="${{ env.SAGE_LOCAL }}" tox -e local-cygwin-choco-${{ matrix.pkgs }} -- $TARGETS'
- name: Prepare logs artifact
shell: bash
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; for a in local/var/tmp/sage/build/*; do if [ -d $a ]; then tar -c --remove-files -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a).tar" $a || tar -c --ignore-failed-read -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a)-save.tar" $a ; fi; done; cp -r logs/* "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- uses: actions/upload-artifact@v2
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
if: always()
- name: Print out logs for immediate inspection
# The markup in the output is a GitHub Actions logging command
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions
shell: bash
run: |
find "artifacts/$LOGS_ARTIFACT_NAME" -type f -name "*.log" -exec sh -c 'if tail -20 "{}" 2>/dev/null | grep "^Error" >/dev/null; then echo :":"error file={}:":" ==== LOG FILE {} CONTAINS AN ERROR ====; cat {} ; fi' \;
if: always()
- name: Prepare sage-local artifact
# We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
# We remove the $SAGE_LOCAL/lib64 link, which will be recreated by the next stage.
run: |
C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && rm -f "${{ env.SAGE_LOCAL }}"/lib64; tar -cf /tmp/sage-local-${{ env.STAGE }}.tar --remove-files "${{ env.SAGE_LOCAL }}"'
if: always()
- uses: actions/upload-artifact@v2
# upload-artifact@v2 does not support whitespace in file names.
# so we tar up the directory ourselves
with:
path: C:\\tools\\cygwin\\tmp\\sage-local-${{ env.STAGE }}.tar
name: ${{ env.LOCAL_ARTIFACT_NAME }}
if: always()
ubuntu-without-sage:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
steps:
- name: Set up the repository
uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -U setuptools Cython Sphinx flake8
- name: Freeze pip
run: |
pip freeze
- name: Local build
run: |
make configure
./configure
make -j4 install
make -j4 check-all
make -j4 distcheck
docker:
runs-on: ubuntu-latest
needs: [dist, ubuntu-without-sage]
strategy:
fail-fast: false
max-parallel: 32
matrix:
tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, ubuntu-impish, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, centos-7, centos-8, gentoo, gentoo-python3.7, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386, raspbian-buster-armhf]
tox_packages_factor: [minimal, standard]
env:
TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
DOCKER_TARGETS: configured with-targets with-targets-optional
steps:
- name: Check out SageMath
uses: actions/checkout@v2
with:
repository: ${{ env.SAGE_REPO }}
ref: ${{ env.SAGE_REF }}
fetch-depth: 2000
if: env.SAGE_REPO != ''
- name: Check out git-trac-command
uses: actions/checkout@v2
with:
repository: sagemath/git-trac-command
path: git-trac-command
if: env.SAGE_TRAC_GIT != ''
- name: Check out SageMath from trac.sagemath.org
shell: bash {0}
run: |
git config --global user.email "[email protected]"
git config --global user.name "ci-sage workflow"
if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch $SAGE_TICKET; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
if: env.SAGE_TRAC_GIT != ''
- uses: actions/download-artifact@v2
with:
path: upstream
name: upstream
- name: Install test prerequisites
run: |
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install tox python3-setuptools
- name: Update Sage packages from upstream artifact
run: |
(export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff)
- name: Configure and build Sage distribution within a Docker container
run: |
set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
- name: Copy logs from the Docker image or build container
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
if: always()
- uses: actions/upload-artifact@v2
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
if: always()
- name: Print out logs for immediate inspection
# and markup the output with GitHub Actions logging commands
run: |
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- name: Push Docker images
run: |
if [ -f .tox/$TOX_ENV/Dockertags ]; then
TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
if [ -z "$TOKEN" ]; then
TOKEN="${{ secrets.GITHUB_TOKEN }}"
fi
echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
for a in $(cat .tox/$TOX_ENV/Dockertags); do
FULL_TAG=docker.pkg.github.com/$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')/$a
docker tag $a $FULL_TAG
echo Pushing $FULL_TAG
docker push $FULL_TAG
done || echo "(Ignoring errors)"
fi
if: always()
macos-without-sage:
runs-on: macos-latest
strategy:
fail-fast: false
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
steps:
- name: Set up the repository
uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -U setuptools Cython Sphinx flake8
- name: Freeze pip
run: |
pip freeze
- name: Local build
run: |
make configure
./configure
make -j4 install
make -j4 check-all
make -j4 distcheck
macos:
runs-on: macos-latest
strategy:
fail-fast: false
max-parallel: 4
matrix:
os: [ macos-10.15, macos-11.0 ]
tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, conda-forge-macos]
tox_packages_factor: [minimal, standard]
xcode_version_factor: [11.7, default, 12.3]
needs: [dist, macos-without-sage]
env:
TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
DOCKER_TARGETS: configured with-targets with-targets-optional
steps:
- name: Select Xcode version
run: |
if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
- name: Check out SageMath
uses: actions/checkout@v2
with:
repository: ${{ env.SAGE_REPO }}
ref: ${{ env.SAGE_REF }}
fetch-depth: 2000
if: env.SAGE_REPO != ''
- name: Check out git-trac-command
uses: actions/checkout@v2
with:
repository: sagemath/git-trac-command
path: git-trac-command
if: env.SAGE_TRAC_GIT != ''
- name: Check out SageMath from trac.sagemath.org
shell: bash {0}
run: |
git config --global user.email "[email protected]"
git config --global user.name "ci-sage workflow"
if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch $SAGE_TICKET; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
if: env.SAGE_TRAC_GIT != ''
- uses: actions/download-artifact@v2
with:
path: upstream
name: upstream
- name: Update Sage packages from upstream artifact
run: |
(export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && git diff)
- name: Install test prerequisites
run: |
brew install tox
- name: Build and test with tox
# We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
# For doctesting, we use a lower parallelization to avoid timeouts.
run: |
MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
- name: Prepare logs artifact
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- uses: actions/upload-artifact@v1
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
if: always()
- name: Print out logs for immediate inspection
# and markup the output with GitHub Actions logging commands
run: |
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
if: always()