Add API schema tests using schemathesis (#3444)

* Add schemathesis tests Signed-off-by: Olga Bulat <[email protected]> * Add `not_a_server_error` check Signed-off-by: Olga Bulat <[email protected]> * Fix the related 500 error Signed-off-by: Olga Bulat <[email protected]> * Fix oembed endpoint Signed-off-by: Olga Bulat <[email protected]> * Get schemathesis running in docker with failing tests * Run schemathesis in pytest suite with failing cases * Fix schema for all endpoints except auth * Remove unnecessary method overrides and fix various path issues * Update the lock file Signed-off-by: Olga Bulat <[email protected]> * Lint Signed-off-by: Olga Bulat <[email protected]> * Fix test errors Signed-off-by: Olga Bulat <[email protected]> * Remove unnecessary CI test Signed-off-by: Olga Bulat <[email protected]> * Catch both DRF and Django Validation errors Signed-off-by: Olga Bulat <[email protected]> * Handle invalid credentials for /token view Signed-off-by: Olga Bulat <[email protected]> * Add Invalid credentials error Signed-off-by: Olga Bulat <[email protected]> * Add `alt_files` serializer Signed-off-by: Olga Bulat <[email protected]> * Create a new section for gitignore Signed-off-by: Olga Bulat <[email protected]> * Pin dependencies up to minor version Signed-off-by: Olga Bulat <[email protected]> --------- Signed-off-by: Olga Bulat <[email protected]> Co-authored-by: sarayourfriend <[email protected]>
WordPress · Feb 29, 2024 · 2747615 · 2747615
1 parent 8438dd8
commit 2747615
Show file tree

Hide file tree

Showing 18 changed files with 1,228 additions and 658 deletions.
diff --git a/api/.gitignore b/api/.gitignore
@@ -11,3 +11,6 @@ docs/_build/
 # OpenAPI
 openapi.yaml
 openapi.json
+
+# Schemathesis
+.hypothesis
diff --git a/api/Pipfile b/api/Pipfile
@@ -5,17 +5,18 @@ verify_ssl = true
 
 [dev-packages]
 factory-boy = "~=3.2"
-fakeredis = "==2.19.0"
+fakeredis = "==2.19"
+freezegun = "~=1.2"
 ipython = "~=8.17"
+pgcli = "~=3.5"
+pook = {ref = "master", git = "git+https://github.com/h2non/pook.git"}
 pycodestyle = "~=2.10"
 pytest = "~=7.4"
 pytest-django = "~=4.6"
 pytest-raises = "~=0.11"
-remote-pdb = "~=2.1"
-pgcli = "~=3.5"
-freezegun = "~=1.2.2"
 pytest-sugar = "~=0.9"
-pook = {ref = "master", git = "git+https://github.com/h2non/pook.git"}
+remote-pdb = "~=2.1"
+schemathesis = "~=3.23"
 
 [packages]
 adrf = "~=0.1.2"
@@ -33,11 +34,11 @@ django-tqdm = "~=1.3"
 django-uuslug = "~=2.0"
 djangorestframework = "~=3.14"
 drf-spectacular = "*"
-elasticsearch = "==8.12.0"
+elasticsearch = "==8.12"
 elasticsearch-dsl = "~=8.9"
 future = "~=0.18"
 limit = "~=0.2"
-Pillow = "~=10.2.0"
+Pillow = "~=10.2"
 psycopg = "~=3.1"
 python-decouple = "~=3.8"
 python-xmp-toolkit = "~=2.0"

diff --git a/api/Pipfile.lock b/api/Pipfile.lock
diff --git a/api/api/docs/audio_docs.py b/api/api/docs/audio_docs.py
@@ -1,3 +1,9 @@
+from rest_framework.exceptions import (
+    NotAuthenticated,
+    NotFound,
+    ValidationError,
+)
+
 from drf_spectacular.utils import OpenApiResponse, extend_schema
 
 from api.docs.base_docs import collection_schema, custom_extend_schema, fields_to_md
@@ -25,10 +31,6 @@
     AudioSerializer,
     AudioWaveformSerializer,
 )
-from api.serializers.error_serializers import (
-    InputErrorSerializer,
-    NotFoundErrorSerializer,
-)
 from api.serializers.media_serializers import MediaThumbnailRequestSerializer
 from api.serializers.provider_serializers import ProviderSerializer
 
@@ -53,7 +55,8 @@
     params=AudioSearchRequestSerializer,
     res={
         200: (AudioSerializer, audio_search_200_example),
-        400: (InputErrorSerializer, audio_search_400_example),
+        400: (ValidationError, audio_search_400_example),
+        401: (NotAuthenticated, None),
     },
     eg=[audio_search_list_curl],
     external_docs={
@@ -69,7 +72,7 @@
 
         By using this endpoint, you can obtain info about content providers such
         as {fields_to_md(ProviderSerializer.Meta.fields)}.""",
-    res={200: (ProviderSerializer, audio_stats_200_example)},
+    res={200: (ProviderSerializer(many=True), audio_stats_200_example)},
     eg=[audio_stats_curl],
 )
 
@@ -81,7 +84,7 @@
         {fields_to_md(AudioSerializer.Meta.fields)}""",
     res={
         200: (AudioSerializer, audio_detail_200_example),
-        404: (NotFoundErrorSerializer, audio_detail_404_example),
+        404: (NotFound, audio_detail_404_example),
     },
     eg=[audio_detail_curl],
 )
@@ -94,13 +97,16 @@
         {fields_to_md(AudioSerializer.Meta.fields)}.""",
     res={
         200: (AudioSerializer(many=True), audio_related_200_example),
-        404: (NotFoundErrorSerializer, audio_related_404_example),
+        404: (NotFound, audio_related_404_example),
     },
     eg=[audio_related_curl],
 )
 
 report = custom_extend_schema(
-    res={201: (AudioReportRequestSerializer, audio_complain_201_example)},
+    res={
+        201: (AudioReportRequestSerializer, audio_complain_201_example),
+        400: (ValidationError, None),
+    },
     eg=[audio_complain_curl],
 )
 
@@ -112,7 +118,7 @@
 waveform = custom_extend_schema(
     res={
         200: (AudioWaveformSerializer, audio_waveform_200_example),
-        404: (NotFoundErrorSerializer, audio_waveform_404_example),
+        404: (NotFound, audio_waveform_404_example),
     },
     eg=[audio_waveform_curl],
 )

diff --git a/api/api/docs/base_docs.py b/api/api/docs/base_docs.py
@@ -2,6 +2,12 @@
 from textwrap import dedent
 from typing import Literal
 
+from rest_framework.exceptions import (
+    NotAuthenticated,
+    NotFound,
+    ValidationError,
+)
+
 from drf_spectacular.openapi import AutoSchema
 from drf_spectacular.utils import (
     OpenApiExample,
@@ -15,7 +21,6 @@
     AudioCollectionRequestSerializer,
     AudioSerializer,
 )
-from api.serializers.error_serializers import NotFoundErrorSerializer
 from api.serializers.image_serializers import ImageSerializer
 from api.serializers.media_serializers import PaginatedRequestSerializer
 
@@ -96,7 +101,7 @@ def get_operation_id(self) -> str:
 
 source_404_message = "Invalid source 'name'. Valid sources are ..."
 source_404_response = OpenApiResponse(
-    NotFoundErrorSerializer,
+    NotFound,
     examples=[
         OpenApiExample(
             name="404",
@@ -114,22 +119,31 @@ def build_source_path_parameter(media_type: MediaType):
 
     return OpenApiParameter(
         name="source",
-        type=str,
+        type={
+            "type": "string",
+            "pattern": "^[^/.]+?$",
+        },
         location=OpenApiParameter.PATH,
         description=f"The source of {media_type}. {valid_description}",
     )
 
 
 creator_path_parameter = OpenApiParameter(
     name="creator",
-    type=str,
+    type={
+        "type": "string",
+        "pattern": "^.+$",
+    },
     location=OpenApiParameter.PATH,
     description="The name of the media creator. This parameter "
     "is case-sensitive, and matches exactly.",
 )
 tag_path_parameter = OpenApiParameter(
     name="tag",
-    type=str,
+    type={
+        "type": "string",
+        "pattern": "^[^/.]+?$",
+    },
     location=OpenApiParameter.PATH,
     description="The tag of the media. Not case-sensitive, matches exactly.",
 )
@@ -210,10 +224,20 @@ def collection_schema(
         serializer = AudioSerializer
 
     if collection == "tag":
-        responses = {200: serializer(many=True)}
+        responses = {
+            200: serializer(many=True),
+            404: NotFound,
+            400: ValidationError,
+            401: (NotAuthenticated, None),
+        }
         path_parameters = [tag_path_parameter]
     else:
-        responses = {200: serializer(many=True), 404: source_404_response}
+        responses = {
+            200: serializer(many=True),
+            404: source_404_response,
+            400: ValidationError,
+            401: (NotAuthenticated, None),
+        }
         path_parameters = [build_source_path_parameter(media_type)]
         if collection == "creator":
             path_parameters.append(creator_path_parameter)
@@ -225,5 +249,8 @@ def collection_schema(
         auth=[],
         description=description,
         responses=responses,
-        parameters=[request_serializer, *path_parameters],
+        parameters=[
+            request_serializer,
+            *path_parameters,
+        ],
     )
diff --git a/api/api/docs/image_docs.py b/api/api/docs/image_docs.py
@@ -1,3 +1,9 @@
+from rest_framework.exceptions import (
+    NotAuthenticated,
+    NotFound,
+    ValidationError,
+)
+
 from drf_spectacular.utils import OpenApiResponse, extend_schema
 
 from api.docs.base_docs import collection_schema, custom_extend_schema, fields_to_md
@@ -19,10 +25,7 @@
     image_stats_200_example,
     image_stats_curl,
 )
-from api.serializers.error_serializers import (
-    InputErrorSerializer,
-    NotFoundErrorSerializer,
-)
+from api.examples.image_responses import image_oembed_400_example
 from api.serializers.image_serializers import (
     ImageReportRequestSerializer,
     ImageSearchRequestSerializer,
@@ -54,7 +57,8 @@
     params=ImageSearchRequestSerializer,
     res={
         200: (ImageSerializer, image_search_200_example),
-        400: (InputErrorSerializer, image_search_400_example),
+        400: (ValidationError, image_search_400_example),
+        401: (NotAuthenticated, None),
     },
     eg=[image_search_list_curl],
     external_docs={
@@ -70,7 +74,7 @@
 
         By using this endpoint, you can obtain info about content providers such
         as {fields_to_md(ProviderSerializer.Meta.fields)}.""",
-    res={200: (ProviderSerializer, image_stats_200_example)},
+    res={200: (ProviderSerializer(many=True), image_stats_200_example)},
     eg=[image_stats_curl],
 )
 
@@ -82,7 +86,7 @@
         {fields_to_md(ImageSerializer.Meta.fields)}""",
     res={
         200: (ImageSerializer, image_detail_200_example),
-        404: (NotFoundErrorSerializer, image_detail_404_example),
+        404: (NotFound, image_detail_404_example),
     },
     eg=[image_detail_curl],
 )
@@ -95,33 +99,35 @@
         {fields_to_md(ImageSerializer.Meta.fields)}.""",
     res={
         200: (ImageSerializer, image_related_200_example),
-        404: (NotFoundErrorSerializer, image_related_404_example),
+        404: (NotFound, image_related_404_example),
     },
     eg=[image_related_curl],
 )
 
 report = custom_extend_schema(
-    res={201: (ImageReportRequestSerializer, image_complain_201_example)},
+    res={
+        201: (ImageReportRequestSerializer, image_complain_201_example),
+        400: (ValidationError, None),
+    },
     eg=[image_complain_curl],
 )
 
 thumbnail = extend_schema(
     parameters=[MediaThumbnailRequestSerializer],
-    responses={200: OpenApiResponse(description="Thumbnail image")},
+    responses={200: OpenApiResponse(description="Thumbnail image"), 404: NotFound},
 )
 
 oembed = custom_extend_schema(
     params=OembedRequestSerializer,
     res={
         200: (OembedSerializer, image_oembed_200_example),
-        404: (NotFoundErrorSerializer, image_oembed_404_example),
+        404: (NotFound, image_oembed_404_example),
+        400: (ValidationError, image_oembed_400_example),
     },
     eg=[image_oembed_curl],
 )
 
-watermark = custom_extend_schema(
-    deprecated=True,
-)
+watermark = extend_schema(deprecated=True, responses={404: NotFound})
 
 source_collection = collection_schema(
     media_type="images",

diff --git a/api/api/docs/oauth2_docs.py b/api/api/docs/oauth2_docs.py
@@ -1,3 +1,10 @@
+from rest_framework.exceptions import (
+    APIException,
+    NotAuthenticated,
+    PermissionDenied,
+    ValidationError,
+)
+
 from api.docs.base_docs import custom_extend_schema
 from api.examples import (
     auth_key_info_200_example,
@@ -8,11 +15,6 @@
     auth_token_200_example,
     auth_token_curl,
 )
-from api.serializers.error_serializers import (
-    ForbiddenErrorSerializer,
-    InputErrorSerializer,
-    InternalServerErrorSerializer,
-)
 from api.serializers.oauth2_serializers import (
     OAuth2ApplicationSerializer,
     OAuth2KeyInfoSerializer,
@@ -27,7 +29,11 @@
     request=OAuth2RegistrationSerializer,
     res={
         201: (OAuth2ApplicationSerializer, auth_register_201_example),
-        400: (InputErrorSerializer, None),
+        400: (ValidationError, None),
+        429: (
+            APIException("Request was throttled. Expected available in 1 second.", 429),
+            None,
+        ),
     },
     eg=[auth_register_curl],
 )
@@ -36,8 +42,12 @@
     operation_id="key_info",
     res={
         200: (OAuth2KeyInfoSerializer, auth_key_info_200_example),
-        403: (ForbiddenErrorSerializer, auth_key_info_403_example),
-        500: (InternalServerErrorSerializer, None),
+        403: (PermissionDenied, auth_key_info_403_example),
+        429: (
+            APIException("Request was throttled. Expected available in 1 second.", 429),
+            None,
+        ),
+        500: (APIException, None),
     },
     eg=[auth_key_info_curl],
 )
@@ -47,6 +57,8 @@
     request={"application/x-www-form-urlencoded": OAuth2TokenRequestSerializer},
     res={
         200: (OAuth2TokenSerializer, auth_token_200_example),
+        401: (NotAuthenticated, None),
+        400: (APIException("Invalid credentials", 400), None),
     },
     eg=[auth_token_curl],
 )
diff --git a/api/api/examples/image_responses.py b/api/api/examples/image_responses.py
@@ -154,6 +154,9 @@
 image_oembed_404_example = {
     "application/json": {"detail": "An internal server error occurred."}
 }
+image_oembed_400_example = {
+    "application/json": {"detail": {"url": ["Could not parse identifier from URL."]}}
+}
 
 image_complain_201_example = {
     "application/json": {