From 096414b4a78e8f040174286aefc456d5a6129b3e Mon Sep 17 00:00:00 2001 From: Laurent Rineau Date: Thu, 2 Jan 2020 18:08:08 +0100 Subject: [PATCH 1/6] Use 2to3 to update to Python3 --- build_images.py | 6 +++--- cgal_docker.py | 20 ++++++++++---------- cgal_release.py | 18 +++++++++--------- resubmit-results.py | 12 ++++++------ test_cgal.py | 18 +++++++++--------- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/build_images.py b/build_images.py index d8ce009..7a63eb0 100755 --- a/build_images.py +++ b/build_images.py @@ -38,14 +38,14 @@ def main(): fullnames = (os.path.join(os.getcwd(), name) for name in os.listdir(os.getcwd())) args.dirs = [name for name in fullnames if os.path.isfile(os.path.join(name, 'Dockerfile'))] - print args.dirs + print(args.dirs) for d in args.dirs: tag = 'cgal-testsuite/' + os.path.basename(d).lower() - print 'Building directory {} with tag {}'.format(d, tag) + print('Building directory {} with tag {}'.format(d, tag)) response = client.build(path=d, rm=True, tag=tag, stream=False) for l in response: - print l + print(l) if __name__ == "__main__": main() diff --git a/cgal_docker.py b/cgal_docker.py index 9896ae5..1c826e4 100644 --- a/cgal_docker.py +++ b/cgal_docker.py @@ -2,7 +2,7 @@ import logging import docker import re -import StringIO +import io class TestsuiteException(Exception): pass @@ -24,7 +24,7 @@ def __str__(self): def container_by_id(docker_client, Id): """Returns a container given an `Id`. Raises `TestsuiteError` if the `Id` cannot be found.""" - contlist = [cont for cont in docker_client.containers(all=True) if Id == cont[u'Id']] + contlist = [cont for cont in docker_client.containers(all=True) if Id == cont['Id']] if len(contlist) != 1: raise TestsuiteError('Requested Container Id ' + Id + 'does not exist') return contlist[0] @@ -45,7 +45,7 @@ def images(docker_client, images): def _default_images(docker_client): images = [] for img in docker_client.images(): - tag = next((x for x in img[u'RepoTags'] if x.startswith(u'cgal-testsuite/')), None) + tag = next((x for x in img['RepoTags'] if x.startswith('cgal-testsuite/')), None) if tag: images.append(tag) return images @@ -109,7 +109,7 @@ def run(self, image, cpuset): container_id = self._create_container(image, cpuset) cont = container_by_id(self.docker_client, container_id) logging.info('Created container: {0} with id {1[Id]} from image {1[Image]} on cpus {2}' - .format(', '.join(cont[u'Names']), cont, cpuset)) + .format(', '.join(cont['Names']), cont, cpuset)) self.docker_client.start(container_id) return container_id @@ -124,10 +124,10 @@ def _create_container(self, img, cpuset): else: chosen_name = 'CGAL-{}-testsuite'.format(res.group(2)) - existing = [cont for cont in self.docker_client.containers(all=True) if '/' + chosen_name in cont[u'Names']] + existing = [cont for cont in self.docker_client.containers(all=True) if '/' + chosen_name in cont['Names']] assert len(existing) == 0 or len(existing) == 1, 'Length of existing containers is odd' - if len(existing) != 0 and u'Exited' in existing[0][u'Status']: + if len(existing) != 0 and 'Exited' in existing[0]['Status']: logging.info('An Exited container with name {} already exists. Removing.'.format(chosen_name)) self.docker_client.remove_container(container=chosen_name) elif len(existing) != 0 and self.force_rm: @@ -152,10 +152,10 @@ def _create_container(self, img, cpuset): mac_address=self.mac_address ) - if container[u'Warnings']: - logging.warning('Container of image {} got created with warnings: {}'.format(img, container[u'Warnings'])) + if container['Warnings']: + logging.warning('Container of image {} got created with warnings: {}'.format(img, container['Warnings'])) - return container[u'Id'] + return container['Id'] class ContainerScheduler: def __init__(self, runner, images, cpusets): @@ -165,7 +165,7 @@ def __init__(self, runner, images, cpusets): self.running_containers = {} # error handling self.errors_encountered = False - self.error_buffer=StringIO.StringIO() + self.error_buffer=io.StringIO() self.error_handler=logging.StreamHandler(self.error_buffer) self.error_handler.setFormatter( logging.Formatter('%(levelname)s: %(message)s') ) self.error_logger = logging.getLogger("Error_logger") diff --git a/cgal_release.py b/cgal_release.py index 46b5592..0e5893d 100644 --- a/cgal_release.py +++ b/cgal_release.py @@ -1,4 +1,4 @@ -import urllib2 +import urllib.request, urllib.error, urllib.parse import os import tarfile import shutil @@ -16,11 +16,11 @@ class Release: def __init__(self, testsuite, use_local, user, passwd): if not use_local and user and passwd: logging.info('Setting up user and password for download.') - password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, Release._release_url, user, passwd) - handler = urllib2.HTTPBasicAuthHandler(password_mgr) - opener = urllib2.build_opener(handler) - urllib2.install_opener(opener) + handler = urllib.request.HTTPBasicAuthHandler(password_mgr) + opener = urllib.request.build_opener(handler) + urllib.request.install_opener(opener) if use_local: logging.info('Using local CGAL release at {}'.format(testsuite)) @@ -35,7 +35,7 @@ def __init__(self, testsuite, use_local, user, passwd): self.path = Release._extract_release(path_to_tar) logging.info('Removing {}'.format(path_to_tar)) os.remove(path_to_tar) - except urllib2.URLError as e: + except urllib.error.URLError as e: if hasattr(e, 'code') and e.code == 401: logging.warning('URLError 401: Did you forget to provide --user and --passwd?') raise @@ -49,8 +49,8 @@ def __str__(self): @staticmethod def _get_latest(): - response = urllib2.urlopen(Release._latest_url) - return response.read().strip() + response = urllib.request.urlopen(Release._latest_url) + return response.read().strip().decode(encoding='UTF-8') @staticmethod def _get_cgal(latest, testsuite): @@ -62,7 +62,7 @@ def _get_cgal(latest, testsuite): logging.warning('Path {} already exists, reusing it.'.format(download_to)) return download_to - response = urllib2.urlopen(download_from) + response = urllib.request.urlopen(download_from) with open(download_to, "wb") as local_file: local_file.write(response.read()) return download_to diff --git a/resubmit-results.py b/resubmit-results.py index fd72fde..51a92ca 100755 --- a/resubmit-results.py +++ b/resubmit-results.py @@ -17,18 +17,18 @@ for i in client.images(name='docker.io/cgal/testsuite-docker'): args.images.append(i['RepoTags'][0]) -print 'Using images {}'.format(', '.join(args.images)) +print('Using images {}'.format(', '.join(args.images))) release = cgal_release.Release(args.testsuite, args.use_local, args.user, args.passwd) -print 'Using release ' + release.version +print('Using release ' + release.version) for c in client.containers(filters={ 'status': 'exited' }): - if not c[u'Image'] in args.images: + if not c['Image'] in args.images: continue - id = c[u'Id'] + id = c['Id'] platform = platform_from_container(client, id) - print 'Resubmit results for platform ' + platform + print('Resubmit results for platform ' + platform) try: handle_results(client, id, args.upload_results, args.testresults, release, args.tester) except TestsuiteError as e: - print ' ERROR: ' + str(e) + print(' ERROR: ' + str(e)) diff --git a/test_cgal.py b/test_cgal.py index 7ab4908..893b8b4 100755 --- a/test_cgal.py +++ b/test_cgal.py @@ -15,7 +15,7 @@ # # Author(s) : Philipp Moeller -from __future__ import division + import logging from cgal_docker import * @@ -87,7 +87,7 @@ def upload_results(local_path): def platform_from_container(client, cont_id): platform_regex = re.compile('^CGAL_TEST_PLATFORM=(.*)$') - for e in client.inspect_container(container=cont_id)[u'Config'][u'Env']: + for e in client.inspect_container(container=cont_id)['Config']['Env']: res = platform_regex.search(e) if res: return res.group(1) @@ -178,7 +178,7 @@ def main(): existing = [cont for cont in client.containers(filters = { 'status' : 'running'})] generic_name_regex = re.compile('CGAL-.+-testsuite') for cont in existing: - for name in cont[u'Names']: + for name in cont['Names']: if generic_name_regex.match(name): info.error('Testsuite Container {} of previous suite still running. Aborting. NOTE: This could also be a name clash.') sys.exit(0) @@ -243,15 +243,15 @@ def main(): try: for ev in client.events(since=before_start, decode=True): assert isinstance(ev, dict) - if ev[u'Type'] != u'container': + if ev['Type'] != 'container': continue; - event_id = ev[u'id'] + event_id = ev['id'] if scheduler.is_ours(event_id): # we care - if ev[u'status'] == u'die': - if ev[u'Actor'][u'Attributes'][u'exitCode']!='0': + if ev['status'] == 'die': + if ev['Actor']['Attributes']['exitCode']!='0': logging.warning('Could not parse exit status: {}. Assuming dirty death of the container.' - .format(ev[u'Actor'][u'Attributes'][u'exitCode'])) + .format(ev['Actor']['Attributes']['exitCode'])) else: logging.info('Container died cleanly, handling results.') try: @@ -280,7 +280,7 @@ def main(): remove_pidfile() if scheduler.errors_encountered: - print (scheduler.error_buffer.getvalue()) + print((scheduler.error_buffer.getvalue())) exit(33) if __name__ == "__main__": From 495936fe370c58ab469263d90d3706b0226fb2e6 Mon Sep 17 00:00:00 2001 From: Laurent Rineau Date: Fri, 21 Feb 2020 09:31:41 +0100 Subject: [PATCH 2/6] Refresh one more script --- resubmit-results.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resubmit-results.py b/resubmit-results.py index 51a92ca..b5f916a 100755 --- a/resubmit-results.py +++ b/resubmit-results.py @@ -10,25 +10,25 @@ parser = cgal_docker_args.parser() args = parser.parse_args() -client=docker.Client(base_url=args.docker_url) +client=docker.APIClient(base_url=args.docker_url) if not args.images: args.images = [] for i in client.images(name='docker.io/cgal/testsuite-docker'): args.images.append(i['RepoTags'][0]) -print('Using images {}'.format(', '.join(args.images))) +print(('Using images {}'.format(', '.join(args.images)))) release = cgal_release.Release(args.testsuite, args.use_local, args.user, args.passwd) -print('Using release ' + release.version) +print(('Using release ' + release.version)) for c in client.containers(filters={ 'status': 'exited' }): if not c['Image'] in args.images: continue id = c['Id'] platform = platform_from_container(client, id) - print('Resubmit results for platform ' + platform) + print(('Resubmit results for platform ' + platform)) try: handle_results(client, id, args.upload_results, args.testresults, release, args.tester) except TestsuiteError as e: - print(' ERROR: ' + str(e)) + print((' ERROR: ' + str(e))) From 4c8e5071554b786087ec2a40f83bae508ae2a4e4 Mon Sep 17 00:00:00 2001 From: Laurent Rineau Date: Wed, 20 May 2020 12:09:22 +0200 Subject: [PATCH 3/6] Add coreutils, for the `timeout` utility --- ArchLinux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ArchLinux/Dockerfile b/ArchLinux/Dockerfile index 21b00cc..f325bad 100644 --- a/ArchLinux/Dockerfile +++ b/ArchLinux/Dockerfile @@ -8,6 +8,7 @@ MAINTAINER Alexander Kobel , Laurent Rineau Date: Tue, 21 Jun 2022 10:29:24 +0200 Subject: [PATCH 4/6] Install libasan --- Fedora-Release/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Fedora-Release/Dockerfile b/Fedora-Release/Dockerfile index b5d4288..2fdef95 100644 --- a/Fedora-Release/Dockerfile +++ b/Fedora-Release/Dockerfile @@ -2,6 +2,8 @@ FROM cgal/testsuite-docker:fedora ARG dockerfile_url ENV DOCKERFILE_URL=$dockerfile_url +RUN dnf -y install libasan && dnf clean all + ENV CGAL_TEST_PLATFORM="Fedora-Release" ENV CGAL_CMAKE_FLAGS="('-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_CXX_FLAGS_RELEASE=-msse3 -O3 -DCGAL_NDEBUG' '-DWITH_GMPXX=ON')" ENV CGAL_USE_ASAN=1 From 2a1040179cdbf21dad068ebe5ed15bb91263d060 Mon Sep 17 00:00:00 2001 From: Laurent Rineau Date: Tue, 19 Sep 2023 15:14:26 +0200 Subject: [PATCH 5/6] allow back the possibility to run from a branch It seems recently that script was only tested from a release tarball. --- run-testsuite.sh | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/run-testsuite.sh b/run-testsuite.sh index c269413..7b2906a 100755 --- a/run-testsuite.sh +++ b/run-testsuite.sh @@ -1,9 +1,14 @@ #!/bin/bash set -e +if [ -z "${CGAL_TESTER}"]; then + export CGAL_TESTER=$(whoami) + echo 'CGAL_TESTER not set. Using `whoami`:'" $CGAL_TESTER" +fi + if [ -z "${CGAL_TEST_PLATFORM}" ]; then export CGAL_TEST_PLATFORM="${HOSTNAME}" - echo "CGAL_TEST_PLATFORM not set. Using HOSTNAME:${HOSTNAME}" + echo "CGAL_TEST_PLATFORM not set. Using HOSTNAME: ${HOSTNAME}" fi # HACK: We depend on this line to easily extract the platform name @@ -28,6 +33,16 @@ CGAL_SRC_DIR="${CGAL_RELEASE_DIR}src/" CGAL_TEST_DIR="${CGAL_RELEASE_DIR}test/" # Directory where CGAL data are stored. CGAL_DATA_DIR="${CGAL_RELEASE_DIR}data/" +# Directory where include/CGAL/version.h can be found. +CGAL_VERSION_DIR="${CGAL_RELEASE_DIR}" +if ! [ -f "${CGAL_VERSION_DIR}include/CGAL/version.h" ]; then + CGAL_VERSION_DIR="${CGAL_RELEASE_DIR}Installation/" +fi +# Directory Testsuite (if branch build), or CGAL_RELEASE_DIR +CGAL_TESTSUITE_DIR="${CGAL_RELEASE_DIR}Testsuite/" +if ! [ -d "${CGAL_TESTSUITE_DIR}" ]; then + CGAL_TESTSUITE_DIR="${CGAL_RELEASE_DIR}" +fi # The directory where testresults are stored. CGAL_TESTRESULTS="/mnt/testresults/" @@ -101,7 +116,7 @@ rm -f "$RESULT_FILE" touch "$RESULT_FILE" sed -n '/The CXX compiler/s/-- The CXX compiler identification is/COMPILER_VERSION =/p' < "${CGAL_TESTRESULTS}installation.log" |sed -E "s/ = (.*)/\ = '\1\'/">> "$RESULT_FILE" -sed -n '/CGAL_VERSION /s/#define //p' < "${CGAL_RELEASE_DIR}include/CGAL/version.h" >> "$RESULT_FILE" +sed -n '/CGAL_VERSION /s/#define //p' < "${CGAL_VERSION_DIR}include/CGAL/version.h" >> "$RESULT_FILE" echo "TESTER ${CGAL_TESTER}" >> "$RESULT_FILE" echo "TESTER_NAME ${CGAL_TESTER_NAME}" >> "$RESULT_FILE" echo "TESTER_ADDRESS ${CGAL_TESTER_ADDRESS}" >> "$RESULT_FILE" @@ -111,8 +126,10 @@ grep -e "^-- USING " "${CGAL_TESTRESULTS}installation.log"|sort -u >> $RESULT_FI sed -i -E 's/(^-- USING )(DEBUG|RELEASE) (CXXFLAGS)/\1\3/' $RESULT_FILE echo "------------" >> "$RESULT_FILE" touch ../../../../../.scm-branch -cat /mnt/testsuite/.scm-branch >> ../../../../../.scm-branch -python3 ${CGAL_RELEASE_DIR}test/parse-ctest-dashboard-xml.py $CGAL_TESTER $PLATFORM +if [ -f /mnt/testsuite/.scm-branch ]; then + cat /mnt/testsuite/.scm-branch >> ../../../../../.scm-branch +fi +python3 ${CGAL_TESTSUITE_DIR}test/parse-ctest-dashboard-xml.py $CGAL_TESTER $PLATFORM for file in $(ls|grep _Tests); do mv $file "$(echo "$file" | sed 's/_Tests//g')" @@ -124,7 +141,7 @@ chmod 777 Installation cat "${CGAL_TESTRESULTS}package_installation.log" >> "Installation/${TEST_REPORT}" #call the python script to complete the results report. -python3 ${CGAL_RELEASE_DIR}test/post_process_ctest_results.py Installation/${TEST_REPORT} ${TEST_REPORT} results_${CGAL_TESTER}_${PLATFORM}.txt +python3 ${CGAL_TESTSUITE_DIR}test/post_process_ctest_results.py Installation/${TEST_REPORT} ${TEST_REPORT} results_${CGAL_TESTER}_${PLATFORM}.txt rm -f $OUTPUT_FILE $OUTPUT_FILE.gz rm ../../../../../.scm-branch tar cf $OUTPUT_FILE results_${CGAL_TESTER}_${PLATFORM}.txt */"$TEST_REPORT" From 54ee7ece910ccf2559ab945bd6ec8e9c1c531f35 Mon Sep 17 00:00:00 2001 From: Laurent Rineau Date: Fri, 9 Aug 2024 16:10:41 +0200 Subject: [PATCH 6/6] python3 ! --- test_cgal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_cgal.py b/test_cgal.py index eaf14e7..09e8793 100755 --- a/test_cgal.py +++ b/test_cgal.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (c) 2015 GeometryFactory (France). All rights reserved. # All rights reserved. #