fix tests and linters

zytedata · Feb 2, 2024 · 63f408f · 63f408f
1 parent ab5aebd
commit 63f408f
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 8 deletions.
diff --git a/docs/templates/e-commerce.rst b/docs/templates/e-commerce.rst
@@ -19,6 +19,6 @@ Parameters
 
 .. autoenum:: zyte_spider_templates.spiders.ecommerce.EcommerceCrawlStrategy
 
-.. autoenum:: zyte_spider_templates.spiders.ecommerce.ExtractFrom
+.. autoenum:: zyte_spider_templates.spiders.base.ExtractFrom
 
 .. autoenum:: zyte_spider_templates.spiders.base.Geolocation
diff --git a/tests/pages/test_product_navigation_heuristics.py b/tests/pages/test_product_navigation_heuristics.py
@@ -1,6 +1,6 @@
 import pytest
 from pytest_twisted import ensureDeferred
-from web_poet import HttpResponse, PageParams, RequestUrl
+from web_poet import AnyResponse, HttpResponse, PageParams, RequestUrl
 from zyte_common_items import ProbabilityRequest, ProductNavigation
 
 from zyte_spider_templates.pages.product_navigation_heuristics import (
@@ -38,7 +38,7 @@ async def test_unknown_product_page():
         </body>
         </html>
     """
-    response = HttpResponse("https://example.com", body)
+    response = AnyResponse(HttpResponse("https://example.com", body))
     navigation = ProductNavigation.from_dict(
         {
             "url": "https://example.com",
@@ -118,7 +118,7 @@ async def test_crawl_nofollow_links():
             </html>
         """
     url = "https://example.com"
-    response = HttpResponse(url, body)
+    response = AnyResponse(HttpResponse(url, body))
     request_url = RequestUrl(response.url)
     navigation = ProductNavigation(url=url)
 

diff --git a/tests/test_ecommerce.py b/tests/test_ecommerce.py
@@ -286,23 +286,36 @@ def test_arguments():
             "ZYTE_API_PROVIDER_PARAMS",
             None,
             "getdict",
-            {"geolocation": "DE"},
+            {
+                "geolocation": "DE",
+                "productOptions": {"extractFrom": "browserHtml"},
+                "productNavigationOptions": {"extractFrom": "browserHtml"},
+            },
         ),
         (
             "geolocation",
             "DE",
             "ZYTE_API_PROVIDER_PARAMS",
             '{"browserHtml": true}',
             "getdict",
-            {"browserHtml": True, "geolocation": "DE"},
+            {
+                "browserHtml": True,
+                "geolocation": "DE",
+                "productOptions": {"extractFrom": "browserHtml"},
+                "productNavigationOptions": {"extractFrom": "browserHtml"},
+            },
         ),
         (
             "geolocation",
             "DE",
             "ZYTE_API_PROVIDER_PARAMS",
             '{"geolocation": "IE"}',
             "getdict",
-            {"geolocation": "DE"},
+            {
+                "geolocation": "DE",
+                "productOptions": {"extractFrom": "browserHtml"},
+                "productNavigationOptions": {"extractFrom": "browserHtml"},
+            },
         ),
         (
             "extract_from",

diff --git a/zyte_spider_templates/pages/product_navigation_heuristics.py b/zyte_spider_templates/pages/product_navigation_heuristics.py
@@ -50,7 +50,9 @@ def _probably_category_links(self) -> List[ProbabilityRequest]:
         ignore_urls = set(self._urls_for_category())
 
         links = []
-        response = TextResponse(url=str(self.response.url), body=self.response.text.encode())
+        response = TextResponse(
+            url=str(self.response.url), body=self.response.text.encode()
+        )
         for link in link_extractor.extract_links(response):
             if link.url in ignore_urls:
                 continue