Switch to scrapy_poet.AnnotatedResult, improve the test.

scrapy-plugins · Oct 13, 2023 · ab34b93 · ab34b93
1 parent 2cabbe0
commit ab34b93
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 4 deletions.
diff --git a/scrapy_zyte_api/providers.py b/scrapy_zyte_api/providers.py
@@ -6,7 +6,7 @@
 from scrapy import Request
 from scrapy.crawler import Crawler
 from scrapy.utils.defer import maybe_deferred_to_future
-from scrapy_poet import PageObjectInputProvider
+from scrapy_poet import AnnotatedResult, PageObjectInputProvider
 from web_poet import BrowserHtml, BrowserResponse
 from zyte_common_items import (
     Article,
@@ -135,7 +135,7 @@ async def __call__(
                 continue
             item = cls_stripped.from_dict(api_response.raw_api_response[kw])
             if is_typing_annotated(cls):
-                item.__metadata__ = cls.__metadata__
+                item = AnnotatedResult(item, cls.__metadata__)
             results.append(item)
             self.update_cache(request, {cls_stripped: item})
         return results
diff --git a/tests/mockserver.py b/tests/mockserver.py
@@ -17,6 +17,7 @@
 from twisted.web.resource import Resource
 from twisted.web.server import NOT_DONE_YET, Site
 
+from scrapy_zyte_api.providers import ExtractFrom
 from scrapy_zyte_api.responses import _API_RESPONSE
 
 from . import SETTINGS, make_handler
@@ -113,6 +114,9 @@ def render_POST(self, request):
                 "price": "10",
                 "currency": "USD",
             }
+            extract_from = request_data.get("productOptions", {}).get("extractFrom")
+            if extract_from == ExtractFrom.httpResponseBody:
+                response_data["product"]["name"] += " (from httpResponseBody)"
 
         return json.dumps(response_data).encode()
 

diff --git a/tests/test_providers.py b/tests/test_providers.py
@@ -191,7 +191,9 @@ class AnnotatedProductPage(BasePage):
 
     class AnnotatedZyteAPISpider(ZyteAPISpider):
         def parse_(self, response: DummyResponse, page: AnnotatedProductPage):
-            return super().parse_(response, page)
+            yield {
+                "product": page.product,
+            }
 
     settings = create_scrapy_settings(None)
     settings.update(SETTINGS)
@@ -204,7 +206,7 @@ def parse_(self, response: DummyResponse, page: AnnotatedProductPage):
     assert item["product"] == Product.from_dict(
         dict(
             url=url,
-            name="Product name",
+            name="Product name (from httpResponseBody)",
             price="10",
             currency="USD",
         )