From c80e3f442343096487d352b14f0903a9b88e786f Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Fri, 27 Oct 2023 21:36:02 +0800 Subject: [PATCH] use new .get_probability() from zyte-common-items==0.12.0 --- setup.py | 2 +- zyte_spider_templates/spiders/base.py | 5 +---- zyte_spider_templates/spiders/ecommerce.py | 4 +--- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 33da81a..6abd9dc 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ "scrapy-poet>=0.16.0", "scrapy-spider-metadata>=0.1.2", "scrapy-zyte-api[provider]>=0.12.2", - "zyte-common-items>=0.8.0", + "zyte-common-items>=0.12.0", ], classifiers=[ "Development Status :: 3 - Alpha", diff --git a/zyte_spider_templates/spiders/base.py b/zyte_spider_templates/spiders/base.py index 4b2fe5b..41f7a21 100644 --- a/zyte_spider_templates/spiders/base.py +++ b/zyte_spider_templates/spiders/base.py @@ -112,10 +112,7 @@ def get_parse_navigation_request( ) def get_parse_product_request_priority(self, request: ProbabilityRequest) -> int: - # TODO: Simplify when https://github.com/zytedata/zyte-common-items/pull/64 is released - probability = 0 - if metadata := getattr(request, "metadata", None): - probability = metadata.probability + probability = request.get_probability() or 0 return int(100 * probability) + self._NEXT_PAGE_PRIORITY def get_parse_product_request( diff --git a/zyte_spider_templates/spiders/ecommerce.py b/zyte_spider_templates/spiders/ecommerce.py index b7c7e28..bf5120e 100644 --- a/zyte_spider_templates/spiders/ecommerce.py +++ b/zyte_spider_templates/spiders/ecommerce.py @@ -156,9 +156,7 @@ def parse_navigation( def parse_product( self, response: DummyResponse, product: Product ) -> Iterable[Product]: - probability = None - if metadata := getattr(product, "metadata", None): - probability = metadata.probability + probability = product.get_probability() # TODO: convert to a configurable parameter later on after the launch if probability is None or probability >= 0.1: