From fb04fefaae66a16fee76dee781be463a5719e6ee Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 14 Mar 2016 22:31:35 -0400 Subject: [PATCH 1/4] Fix JsonSchema in Py3 --- pyresttest/ext/validator_jsonschema.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyresttest/ext/validator_jsonschema.py b/pyresttest/ext/validator_jsonschema.py index d74b7da4..386e8088 100644 --- a/pyresttest/ext/validator_jsonschema.py +++ b/pyresttest/ext/validator_jsonschema.py @@ -1,9 +1,12 @@ import traceback import json +from sys import version_info import yaml import jsonschema +PYTHON_MAJOR_VERSION = version_info[0] + try: # First try to load pyresttest from global namespace from pyresttest import validators from pyresttest import binding @@ -27,7 +30,10 @@ def validate(self, body=None, headers=None, context=None): try: # TODO try draft3/draft4 iter_errors - # https://python-jsonschema.readthedocs.org/en/latest/validate/#jsonschema.IValidator.iter_errors - jsonschema.validate(json.loads(body), schema) + parsed_body = body + if PYTHON_MAJOR_VERSION > 2 and isinstance(body, bytes): + parsed_body = str(body, 'utf-8') + jsonschema.validate(json.loads(parsed_body), schema) return True except jsonschema.exceptions.ValidationError as ve: trace = traceback.format_exc() From be11cfef3d62483c7ffdc4058d8ba2b72d032509 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 14 Mar 2016 22:43:30 -0400 Subject: [PATCH 2/4] Add a quick test that JSONSchema extension works correctly --- docker/build.sh | 6 +++--- docker/centos6-py26/Dockerfile | 2 +- docker/python3/Dockerfile | 2 +- docker/ubuntu14-py27/Dockerfile | 2 +- jenkins/jenkins-build-images.groovy | 6 +++--- pyresttest/functionaltest.py | 19 ++++++++++++++++++- pyresttest/miniapp-schema.json | 21 +++++++++++++++++++++ pyresttest/testapp/schema_test.yaml | 5 +++++ 8 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 pyresttest/miniapp-schema.json create mode 100644 pyresttest/testapp/schema_test.yaml diff --git a/docker/build.sh b/docker/build.sh index 1931aee4..51667cda 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -5,9 +5,9 @@ set -x DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $DIR -UBUNTU_14_VERSION=0.5 -CENTOS6_VERSION=0.5 -PYTHON3_VERSION=0.6 +UBUNTU_14_VERSION=0.6 +CENTOS6_VERSION=0.6 +PYTHON3_VERSION=0.7 docker build -t pyresttest-build-ubuntu-14:$UBUNTU_14_VERSION-SNAPSHOT ./ubuntu14-py27 docker build -t pyresttest-build-centos6:$CENTOS6_VERSION-SNAPSHOT ./centos6-py26 diff --git a/docker/centos6-py26/Dockerfile b/docker/centos6-py26/Dockerfile index d88bd070..f7f59096 100644 --- a/docker/centos6-py26/Dockerfile +++ b/docker/centos6-py26/Dockerfile @@ -6,7 +6,7 @@ RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noar # pycurl is part of yum, and so is python, so we don't install RUN yum install -y python-pip git-core python-mock rpm-build tar && yum clean all \ - && pip install discover jmespath pyyaml django==1.6.5 django-tastypie==0.12.1 + && pip install discover jmespath jsonpath pyyaml django==1.6.5 django-tastypie==0.12.1 COPY verify_image.py /tmp/verify_image.py RUN chmod a+rwx /tmp/verify_image.py \ No newline at end of file diff --git a/docker/python3/Dockerfile b/docker/python3/Dockerfile index 1c089d43..03c68edd 100644 --- a/docker/python3/Dockerfile +++ b/docker/python3/Dockerfile @@ -5,7 +5,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y git-core tar \ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Python 2 and 3 dependencies, future is just for python 3 compat, sigh -RUN pip3 install pycurl jmespath pyyaml django==1.6.5 django-tastypie==0.12.1 future +RUN pip3 install pycurl jmespath jsonpath pyyaml django==1.6.5 django-tastypie==0.12.1 future COPY verify_image.py /tmp/verify_image.py RUN chmod a+rwx /tmp/verify_image.py \ No newline at end of file diff --git a/docker/ubuntu14-py27/Dockerfile b/docker/ubuntu14-py27/Dockerfile index 70e5ffcf..9bfa518e 100644 --- a/docker/ubuntu14-py27/Dockerfile +++ b/docker/ubuntu14-py27/Dockerfile @@ -5,7 +5,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python python-p && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Python 2 and 3 dependencies -RUN pip install mock jmespath pyyaml django==1.6.5 django-tastypie==0.12.1 +RUN pip install mock jmespath jsonpath pyyaml django==1.6.5 django-tastypie==0.12.1 COPY verify_image.py /tmp/verify_image.py RUN chmod a+rwx /tmp/verify_image.py \ No newline at end of file diff --git a/jenkins/jenkins-build-images.groovy b/jenkins/jenkins-build-images.groovy index c6c07289..cb233880 100644 --- a/jenkins/jenkins-build-images.groovy +++ b/jenkins/jenkins-build-images.groovy @@ -25,11 +25,11 @@ node { def python3 = docker.build("pyresttest-build-python3:test", 'docker/python3') stage name:'test/tag', concurrency: 1 - run_test(ubuntu14_py27, 'python', '0.5') - run_test(centos6_py26, 'python', '0.5') + run_test(ubuntu14_py27, 'python', '0.6') + run_test(centos6_py26, 'python', '0.6') // For some inexplicable reasons, the test script here is more brittle than the others // Direct docker run works, it hit issues with docker.build, so... okay? - run_test(python3, 'python3', '0.6') + run_test(python3, 'python3', '0.7') } diff --git a/pyresttest/functionaltest.py b/pyresttest/functionaltest.py index d05899cf..c634787c 100644 --- a/pyresttest/functionaltest.py +++ b/pyresttest/functionaltest.py @@ -317,7 +317,24 @@ def test_benchmark_get(self): self.assertTrue(benchmark_config.benchmark_runs, len( benchmark_result.results['total_time'])) - def test_get_validators_jmespath_fail(self): + def test_use_validator_ext_jsonschema(self): + try: + import jsonschema + except ImportError: + print("Skipping jsonschema import test because library absent") + raise unittest.SkipTest("JSONSchema module absent") + # Serious hack dumping the schema in local directory to allow executing it this way + # But otherwise kind of painful + path = os.path.join(os.path.dirname( + os.path.realpath(__file__)), 'testapp/schema_test.yaml') + print(path) + tests = resttest.parse_testsets('http://localhost:8000', resttest.read_test_file( + path), working_directory=os.path.dirname(os.path.realpath(__file__))) + failures = resttest.run_testsets(tests) + self.assertTrue( + failures == 0, 'Simple tests failed where success expected') + + def test_use_validators_jmespath_fail(self): try: import jmespath except ImportError: diff --git a/pyresttest/miniapp-schema.json b/pyresttest/miniapp-schema.json new file mode 100644 index 00000000..0cc868dd --- /dev/null +++ b/pyresttest/miniapp-schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Person", + "description": "A person from the miniapp", + "type": "object", + "properties": { + "id": { + "type": "integer", + "description": "Unique person ID" + }, + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "login": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/pyresttest/testapp/schema_test.yaml b/pyresttest/testapp/schema_test.yaml new file mode 100644 index 00000000..a5f86ae1 --- /dev/null +++ b/pyresttest/testapp/schema_test.yaml @@ -0,0 +1,5 @@ +--- +- test: + - url: /api/person/1/ + - validators: + - json_schema: {schema: {file: 'miniapp-schema.json'}} From 7d07ba153061f2705233df9c0270ca3433d81d50 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 14 Mar 2016 22:50:17 -0400 Subject: [PATCH 3/4] Fix docker builds failing if image did not exist --- docker/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index 51667cda..6af8dd20 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -22,7 +22,7 @@ if [ $? -ne 0 ]; then # Test failed, remove the built image and exit with error fi docker tag -f pyresttest-build-ubuntu-14:$UBUNTU_14_VERSION-SNAPSHOT pyresttest-build-ubuntu-14:$UBUNTU_14_VERSION docker tag -f pyresttest-build-ubuntu-14:$UBUNTU_14_VERSION-SNAPSHOT pyresttest-build-ubuntu-14:latest -docker rmi pyresttest-build-ubuntu-14:$UBUNTU_14_VERSION-SNAPSHOT +docker rmi pyresttest-build-ubuntu-14:$UBUNTU_14_VERSION-SNAPSHOT || true docker run -t --rm pyresttest-build-centos6:$CENTOS6_VERSION-SNAPSHOT python /tmp/verify_image.py if [ $? -ne 0 ]; then # Test failed, remove the built image and exit with error @@ -32,7 +32,7 @@ if [ $? -ne 0 ]; then # Test failed, remove the built image and exit with error fi docker tag -f pyresttest-build-centos6:$CENTOS6_VERSION-SNAPSHOT pyresttest-build-centos6:$CENTOS6_VERSION docker tag -f pyresttest-build-centos6:$CENTOS6_VERSION-SNAPSHOT pyresttest-build-centos6:latest -docker rmi pyresttest-build-centos6:$CENTOS6_VERSION-SNAPSHOT +docker rmi pyresttest-build-centos6:$CENTOS6_VERSION-SNAPSHOT || true docker run -t --rm pyresttest-build-python3:$PYTHON3_VERSION-SNAPSHOT python3 /tmp/verify_image.py if [ $? -ne 0 ]; then # Test failed, remove the built image and exit with error From 43981cdb4a975e063457f61478780e41094efe2e Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 14 Mar 2016 23:02:22 -0400 Subject: [PATCH 4/4] Sigh. --- docker/centos6-py26/Dockerfile | 2 +- docker/python3/Dockerfile | 2 +- docker/ubuntu14-py27/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/centos6-py26/Dockerfile b/docker/centos6-py26/Dockerfile index f7f59096..8d066b32 100644 --- a/docker/centos6-py26/Dockerfile +++ b/docker/centos6-py26/Dockerfile @@ -6,7 +6,7 @@ RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noar # pycurl is part of yum, and so is python, so we don't install RUN yum install -y python-pip git-core python-mock rpm-build tar && yum clean all \ - && pip install discover jmespath jsonpath pyyaml django==1.6.5 django-tastypie==0.12.1 + && pip install discover jmespath jsonschema pyyaml django==1.6.5 django-tastypie==0.12.1 COPY verify_image.py /tmp/verify_image.py RUN chmod a+rwx /tmp/verify_image.py \ No newline at end of file diff --git a/docker/python3/Dockerfile b/docker/python3/Dockerfile index 03c68edd..1b8ec77b 100644 --- a/docker/python3/Dockerfile +++ b/docker/python3/Dockerfile @@ -5,7 +5,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y git-core tar \ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Python 2 and 3 dependencies, future is just for python 3 compat, sigh -RUN pip3 install pycurl jmespath jsonpath pyyaml django==1.6.5 django-tastypie==0.12.1 future +RUN pip3 install pycurl jmespath jsonschema pyyaml django==1.6.5 django-tastypie==0.12.1 future COPY verify_image.py /tmp/verify_image.py RUN chmod a+rwx /tmp/verify_image.py \ No newline at end of file diff --git a/docker/ubuntu14-py27/Dockerfile b/docker/ubuntu14-py27/Dockerfile index 9bfa518e..68c65806 100644 --- a/docker/ubuntu14-py27/Dockerfile +++ b/docker/ubuntu14-py27/Dockerfile @@ -5,7 +5,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python python-p && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Python 2 and 3 dependencies -RUN pip install mock jmespath jsonpath pyyaml django==1.6.5 django-tastypie==0.12.1 +RUN pip install mock jmespath jsonschema pyyaml django==1.6.5 django-tastypie==0.12.1 COPY verify_image.py /tmp/verify_image.py RUN chmod a+rwx /tmp/verify_image.py \ No newline at end of file