From de9926e883fafecc29a686df277fa27972d73e00 Mon Sep 17 00:00:00 2001 From: nmolivo Date: Mon, 22 Feb 2021 21:26:12 -0500 Subject: [PATCH] Issue #3 --- city_scrapers/spiders/cle_library.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/city_scrapers/spiders/cle_library.py b/city_scrapers/spiders/cle_library.py index d08ef89..244d606 100644 --- a/city_scrapers/spiders/cle_library.py +++ b/city_scrapers/spiders/cle_library.py @@ -30,26 +30,10 @@ def parse(self, response): self.minutes_map = self._parse_minutes(response) yield scrapy.Request( "https://cpl.org/category/meeting/", - callback=self._parse_meetings_page, + callback=self._parse_meetings, dont_filter=True, ) - def _parse_meetings_page(self, response): - """Parse meetings from results pages""" - yield from self._parse_meetings(response) - - page_num = response.url.split("/")[-2] - if page_num.isdigit(): - page_num = int(page_num) + 1 - else: - page_num = 2 - if page_num <= 3: - yield scrapy.Request( - "https://cpl.org/category/meeting/page/{}/".format(page_num), - callback=self._parse_meetings_page, - dont_filter=True, - ) - def _parse_meetings(self, response): for item in response.css("article"): title = self._parse_title(item)