diff --git a/city_scrapers/spiders/cle_design_review.py b/city_scrapers/spiders/cle_design_review.py index cd419ff..e03da92 100644 --- a/city_scrapers/spiders/cle_design_review.py +++ b/city_scrapers/spiders/cle_design_review.py @@ -1,10 +1,12 @@ import re -from datetime import datetime +import time +from datetime import datetime, timedelta from city_scrapers_core.constants import ADVISORY_COMMITTEE from city_scrapers_core.items import Meeting from city_scrapers_core.spiders import CityScrapersSpider -from scrapy import Selector + +from city_scrapers.utils import calculate_upcoming_meeting_days class CleDesignReviewSpider(CityScrapersSpider): @@ -12,92 +14,173 @@ class CleDesignReviewSpider(CityScrapersSpider): agency = "Cleveland Design Review Advisory Committees" timezone = "America/Detroit" start_urls = [ - "http://clevelandohio.gov/CityofCleveland/Home/Government/CityAgencies/CityPlanningCommission/MeetingSchedules" # noqa + "https://planning.clevelandohio.gov/designreview/schedule.php" # noqa ] + description = "Due to Covid meetings are being held on WebEx rather than in person. For more information contact " # noqa + calculated_description = "This is an upcoming meeting - please verify it with staff if you want attend. Due to Covid meetings are being held on WebEx rather than in person. For more information contact " # noqa def parse(self, response): """ - `parse` should always `yield` Meeting items. + There's no element that wraps both the committee name/time and + the dropdown containing the agendas. As such we want to grab + each committee name/times and then use the following dropdown + to get the agendas. Luckily all of the committee name/times are + (and are the only thing in) divs with the class '.mt-3' so we can + grab all the divs with those classes and then look for the next sibling + div with the ".dropdown" class to get the links to all the agendas. + + Note that the city planning meeting is handled by a different scraper so + we do look at it here. Luckily the name/times for the city planning + meeting are not currently wrapped in a div, so the list of nodes described + above won't include it. + + There are three other points to keep in mind for this scraper: + + 1. The way the data is presented doesn't make it easy to know whether or + not a meeting occurred but doesn't have an agenda, or whether a meeting + is going to happen on a normal meeting date. The strategy I'm using is + to treat the agenda links as authoritative for past (and if listed + upcoming) meetings. So previous meetings are just read off of the agenda + links. For future meetings we take the date of the most recent agenda + and then calculate meetings for 60 days from that date. As dates + progress and agendas are added, those tentative meetings will either be + confirmed to exist or disappear based on the ways the agendas are + updated. For calculated meetings we add a line to the description + encouraging users to verify the meeting with staff before attempting to + attend. + + 2. There is no mention of the year anywhere in the text of the site. We + can extract it from the agenda link - at least for now. But it will + be important to keep an eye on how the site is changed in January. - Change the `_parse_title`, `_parse_start`, etc methods to fit your scraping - needs. + 3. Meetings are currently not being held in person but over webex. We've + included this information in the meeting description. """ - page_content = response.css("#content .field-items .field-item")[0] - bold_text = " ".join(page_content.css("strong *::text").extract()) - year_match = re.search(r"\d{4}(?= Agenda)", bold_text) - if year_match: - year_str = year_match.group() - else: - year_str = str(datetime.now().year) - design_review_committees = re.split(r"\", page_content.extract())[1:] - for committee in design_review_committees: - committee_item = Selector(text=committee) - title = self._parse_title(committee_item) + committee_metas = response.css( + "div.mt-3" + ) # this skips city planning since it is handled by a separate scraper + committee_agendas = response.css("div.mt-3 + div.dropdown") + if len(committee_metas) != len(committee_agendas): + # we haven't sucessfully extracted matched metas and agendas so we + # can't safely iterate over them together. + raise ValueError("Cannot match committee agandas to committee metadata") + committee_items = zip(committee_metas, committee_agendas) + + for committee_meta, commitee_agenda_list in committee_items: + title = self._parse_title(committee_meta) if not title: continue - location = self._parse_location(committee_item) - time_str = self._parse_time_str(committee_item) - for row in committee_item.css(".report tr"): - month_str = ( - row.css("td:first-child::text").extract_first().replace(".", "") + location = self._parse_location(committee_meta) + time_str = self._parse_time_str(committee_meta) + email_contact = self._parse_email_contact(committee_meta) + weekday, chosen_ordinals, is_downtown = self._parse_meeting_schedule_info( + committee_meta + ) + most_recent_start = datetime.today() + + # Start by looking through the agendas for existing meetings + for agenda in commitee_agenda_list.css("div.dropdown-menu a.dropdown-item"): + month_str, day_str = ( + agenda.css("*::text").extract_first().strip().split(" ") + ) + year_str = self._parse_year_from_agenda_link(agenda) + + start = self._parse_start(year_str, month_str, day_str, time_str) + # most_recent_start will be used to calculate upcoming meetings + # with no agenda + most_recent_start = start + if not start: + continue + meeting = Meeting( + title=title, + description=self.description + email_contact, + classification=ADVISORY_COMMITTEE, + start=start, + end=None, + all_day=False, + time_notes="", + location=location, + links=self._parse_links(agenda, response), + source=response.url, ) - for date_cell in row.css("td:not(:first-child)"): - start = self._parse_start(date_cell, year_str, month_str, time_str) - if not start: - continue - meeting = Meeting( - title=title, - description="", - classification=ADVISORY_COMMITTEE, - start=start, - end=None, - all_day=False, - time_notes="", - location=location, - links=self._parse_links(date_cell, response), - source=response.url, - ) - - meeting["status"] = self._get_status(meeting) - meeting["id"] = self._get_id(meeting) - - yield meeting + + meeting["status"] = self._get_status(meeting) + meeting["id"] = self._get_id(meeting) + + yield meeting + + # next we calculate upcoming meeting dates for 60 days after the + # last agenda date + calc_start = most_recent_start + timedelta(days=1) + # since downtown meetings are calculated based on the city planning + # meeting one day ahead, we need to add an extra day to avoid + if is_downtown: + calc_start = calc_start + timedelta(days=1) + + calc_end = calc_start + timedelta(days=60) + + upcoming_meetings = calculate_upcoming_meeting_days( + weekday, chosen_ordinals, calc_start.date(), calc_end.date() + ) + if is_downtown: # downtown meetings are a day before the one calculated + upcoming_meetings = [ + day + timedelta(days=-1) for day in upcoming_meetings + ] + + for day in upcoming_meetings: + start = self._parse_calculated_start(day, time_str) + meeting = Meeting( + title=title, + description=self.calculated_description + email_contact, + classification=ADVISORY_COMMITTEE, + start=start, + end=None, + all_day=False, + time_notes="", + location=location, + links=[], + source=response.url, + ) + + meeting["status"] = self._get_status(meeting) + meeting["id"] = self._get_id(meeting) + + yield meeting def _parse_title(self, item): """Parse or generate meeting title.""" committee_strs = [ c.strip() - for c in item.css("p > strong::text").extract() + for c in item.css("h4::text").extract() if c.strip().upper().endswith("DESIGN REVIEW COMMITTEE") ] if len(committee_strs): return committee_strs[0].title() def _parse_time_str(self, item): - desc_text = " ".join(item.css("p *::text").extract()) + """Parse out the time as a string in the format hh:mm:am/pm""" + desc_text = " ".join(item.css("p.mb-1::text").extract()) time_match = re.search(r"\d{1,2}:\d{2}\s*[apm]{2}", desc_text) if time_match: return time_match.group().replace(" ", "") return "12:00am" - def _parse_start(self, item, year_str, month_str, time_str): + def _parse_start(self, year_str, month_str, day_str, time_str): """Parse start datetime as a naive datetime object.""" - cell_text = " ".join(item.css("* ::text").extract()) - date_text = re.sub(r"\D", "", cell_text) - if not date_text or "No meeting" in cell_text: - return - date_str = " ".join([year_str, month_str, date_text, time_str]) - return datetime.strptime(date_str, "%Y %b %d %I:%M%p") + date_str = " ".join([year_str, month_str, day_str, time_str]) + return datetime.strptime(date_str, "%Y %B %d %I:%M%p") + + def _parse_calculated_start(self, day, time_str): + """Parse start datetime from python date and a string with the time.""" + date_str = " ".join([day.strftime("%Y %B %d"), time_str]) + return datetime.strptime(date_str, "%Y %B %d %I:%M%p") def _parse_location(self, item): """Parse or generate location.""" - desc_str = " ".join(item.css("p[id] *::text").extract()) - # Override for first committee - if "CITYWIDE" in desc_str: - desc_str = " ".join( - [l for l in item.css("p *::text").extract() if "days" in l] - ) + desc_str = " ".join(item.css("p.mb-1::text").extract()) loc_str = re.sub(r"\s+", " ", re.split(r"(\sin\s|\sat\s)", desc_str)[-1]) + # The downtown/flats commission doesn't give the full address - it just says + # city hall so we need a special case to add the street address if "City Hall" in loc_str: loc_name = "City Hall" room_match = re.search(r"(?<=Room )\d+", loc_str) @@ -111,6 +194,7 @@ def _parse_location(self, item): split_loc = loc_str.split("-") loc_name = "-".join(split_loc[:-1]) loc_addr = split_loc[-1] + # We need to make sure that the address ends with the city and state if "Cleveland" not in loc_addr: loc_addr = loc_addr.strip() + " Cleveland, OH" return { @@ -119,12 +203,51 @@ def _parse_location(self, item): } def _parse_links(self, item, response): + """Parse out the links for the meeting""" links = [] - for link in item.css("a"): - links.append( - { - "title": " ".join(link.css("*::text").extract()).strip(), - "href": response.urljoin(link.attrib["href"]), - } - ) + links.append({"title": "Agenda", "href": response.urljoin(item.attrib["href"])}) return links + + def _parse_year_from_agenda_link(self, item): + """Parse the year as a string from a link containing the agenda""" + link = item.attrib["href"] + year_match = re.search(r"\/(20\d{2})\/", link) + if year_match: + return year_match.group(1) + return "2021" + + def _parse_email_contact(self, item): + """Parses the email for a committee's contact""" + email_str = item.css("p.mt-1::text").extract()[2] + return email_str.replace(": ", "") + + def _parse_meeting_schedule_info(self, committee_meta): + """Parses out the weekday, and frequency of the meeting for calculating + future dates""" + # Add special case for downtown downtown meetings are the day before city + # planning, so we calculate using the city planning schedule (1, and 3rd + # Friday) and set a flag so we can subtract a day from the results + committee_str = " ".join(committee_meta.css("p.mb-1::text").extract()) + is_downtown = "prior to the City Planning Commission" in committee_str + + if is_downtown: + weekday = 4 + chosen_ordinals = [0, 2] + else: + weekday_str = committee_meta.css("p.mb-1 strong::text").extract_first() + weekday = self._parse_weekday(weekday_str) + raw_weeks = re.findall(r"1st|2nd|3rd|4th", committee_str) + # ordinals here just refer to the 1st, 2nd etc... + chosen_ordinals = [self._parse_ordinal(ordinal) for ordinal in raw_weeks] + return weekday, chosen_ordinals, is_downtown + + def _parse_weekday(self, weekday): + """Parses weekday strings as their integer equivalent""" + # we cut off the last char of weekday, because it comes through with + # an 's' i.e. 'Tuesdays' + return time.strptime(weekday[:-1], "%A").tm_wday + + def _parse_ordinal(self, ordinal_str): + """Parses ordinals as their integer equivalent beginning from 0""" + ordinal_lookup = {"1st": 0, "2nd": 1, "3rd": 2, "4th": 3} + return ordinal_lookup[ordinal_str.lower()] diff --git a/city_scrapers/utils/__init__.py b/city_scrapers/utils/__init__.py new file mode 100644 index 0000000..67e146f --- /dev/null +++ b/city_scrapers/utils/__init__.py @@ -0,0 +1 @@ +from .meeting_date_calculator import calculate_upcoming_meeting_days # noqa diff --git a/city_scrapers/utils/meeting_date_calculator.py b/city_scrapers/utils/meeting_date_calculator.py new file mode 100644 index 0000000..463358a --- /dev/null +++ b/city_scrapers/utils/meeting_date_calculator.py @@ -0,0 +1,96 @@ +import calendar +from datetime import date + + +def calculate_upcoming_meeting_days(chosen_weekday, chosen_ordinals, start, end): + """ + Lots of city meeting websites describe their upcoming meetings by saying + things like: "this committee meets the 1st and 3rd Tuesday of every month ". + This calculator is intended to help parse dates from such a description. It + doesn't handle parsing the actual language, since that might differ from page + to page, but given a weekday, and a list of the oridnals you care about (like + 1st, 3rd), a start date and an end date, it will return all the meeting dates + that match the weekday and ordinals. + + Parameters: + chosen_weekday (int): the weekday that you're looking for. Monday is 0, + so in the examples above this would be 2 + chosen_ordinals (int[]): the particular days you're looking for - like 1st + and 3rd. These days should be passed though starting the count from 0, + i.e [0, 2] for first and third + start (date): the first day to begin calculating meetings from + end (date): the final day to be considered as a potential meeting date + + Returns: + []date: an array of dates that match the given conditions + """ + current_month = start.month + current_year = start.year + + raw_dates = [] + while not (current_month == end.month and current_year == end.year): + current_month_days = _calculate_meeting_days_per_month( + chosen_weekday, chosen_ordinals, current_year, current_month + ) + raw_dates = raw_dates + [ + date(current_year, current_month, day) for day in current_month_days + ] + + # we can't easily use % arithmetic here since we're starting at 1, so + # it's a bit easier to read this way + current_month = current_month + 1 if current_month != 12 else 1 + if current_month == 1: + current_year = current_year + 1 + + # add the days for the final month since they're missed by the loop + current_month_days = _calculate_meeting_days_per_month( + chosen_weekday, chosen_ordinals, current_year, current_month + ) + raw_dates = raw_dates + [ + date(current_year, current_month, day) for day in current_month_days + ] + # we now have all the relevant dates for the given months but we need to + # filter out days before and after start and end + return [ + current_date for current_date in raw_dates if (start <= current_date <= end) + ] + + +def _calculate_meeting_days_per_month(chosen_weekday, chosen_ordinals, year, month): + """ + Lots of city meeting websites describe their upcoming meetings by saying + things like: "this committee meets the 1st and 3rd Tuesday of every month". + This calculator is intended to help parse dates from such a description. It + doesn't handle parsing the actual language, since that might differ from page + to page, but given a weekday, and a list of the oridnals you care about (like + 1st, 3rd) and a month it will return all the days in the month that match the + given conditions. + + Parameters: + chosen_weekday (int): the weekday that you're looking for. Monday is 0, so + in the examples above this would be 2 + chosen_ordinals (int[]): the particular days you're looking for - like 1st and + 3rd. These days should be passed though starting the count from 0, + i.e [0, 2] for first and third + year (int): the year as an integer + month (int): the month as an integer + + Returns: + []int: an array of the days of the month that matched the given conditions. + """ + + days_of_the_month = calendar.Calendar().itermonthdays2(year, month) + # we create a list of all days in the month that are the proper weekday - + # day is 0 if it is outside the month but present to make complete first or + # last weeks + potential_days = [ + day + for day, weekday in days_of_the_month + if day != 0 and weekday == chosen_weekday + ] + # we then see if the resulting number is in the chosen_weeks array + chosen_days = [ + day for i, day in enumerate(potential_days) if (i) in chosen_ordinals + ] + + return chosen_days diff --git a/tests/files/cle_design_review.html b/tests/files/cle_design_review.html index a48c7bd..be133f3 100644 --- a/tests/files/cle_design_review.html +++ b/tests/files/cle_design_review.html @@ -1,608 +1,429 @@ - - - - - - - - - - Design Review Meeting Schedules | City of Cleveland - - - - - - - - - - - - - - - - - - -
- + +

CITY PLANNING COMMISSION

+

The City Planning Commission meets at 9am, every 1st & 3rd Friday of the month in Room 514, City Hall

+

Contact: Michael Bosak Phone: 216.664.3802 Email: mbosak@clevelandohio.gov +

+
+
+ View/Download Agendas Below
+(PDF) +
+ +
+ January 15 + February 5 + February 19 + March 5 + March 19 + April 16 + May 7 + May 21 + June 4 + June 18 + July 2 + July 16 + July 20 + August 6 + August 20 + Sept 3 + Sept 14 + Sept 17 + October 1 + October 15 + November 5 + November 16 + November 19 +
+
+
+
+ View/Download Presentations Below
+(PDF) +
+ +
+ February 19 + February 19 + March 5 + March 19 + April 16 + May 7 + May 21 + June 4 + June 18 + July 2 + July 16 + August 6 + August 20 + Sept. 3 + Sept. 17 + October 1 + October 15 + November 5 +
+
+
+
+
+
+ +
+

DOWNTOWN/FLATS DESIGN REVIEW COMMITTEE

+ +

The Committee meets at 9:00 am, Thursday prior to the City Planning Commission meeting on Fridays in Room 514, City Hall

+ +

Contact: Anthony Santora Phone: 216.664.3815 Email: asantora@clevelandohio.gov

+
+ +
+ January 14 + March 18 +April 15 + May 6 + May 20 + July 15 + September 16 + September 30 + November 18 +
+
+ +
+ +
+

EAST DESIGN REVIEW COMMITTEE +

+ +

The Committee meets on the 2nd & 4th Tuesdays @ 8:30 am in Cornucopia Place, 7201 Kinsman Road, Suite 103B

+ +

Contact: Nickol Calhoun Phone: 216.664.3817 Email: ncalhoun@clevelandohio.gov

+
+ +
+ January 12 + January 26 + February 9 + March 9 + May 25 + June 8 + June 22 + September 28 + October 12 + October 26 + November 9 + November 23 +
+
+ +
+ +
+

EUCLID CORRIDOR DESIGN REVIEW COMMITTEE

+

The Committee meets on the 1st & 3rd Thursdays @ 8:00 am in + The Agora Building- 5000 Euclid Ave

+

Contact: Kim Scott Phone: 216.664.3803 Email: kscott@clevelandohio.gov

+
+ +
+ March 4 + March 18 + May 6 + May 20 + June 17 + July 1 + August 5 + September 2 + October 14 + November 4 + November 18 +
+
+ +
+ +
+

FAR WEST DESIGN REVIEW COMMITTEE

+

The Committee meets on the 1st & 3rd Wednesdays @ 8:00 am at + St. Mel's Catholic Church - 14436 Triskett

+

Contact: Adam Davenport Phone: 216.664.3800 Email: adavenport@clevelandohio.gov

+
+ +
+ February 17 + March 3 + March 17 + April 7 + April 21 + May 19 + June 2 + June 16 + July 7 + July 21 + August 4 + September 1 + September 15 + October 6 + October 20 + December 1 +
+
+ +
+ +
+

NEAR WEST DESIGN REVIEW COMMITTEE

+

The Committee meets on the 2nd & 4th Wednesdays @ 8:30 am at + South Branch Library, 3096 Scranton Rd.

+

Contact: Matt Moss Phone: 216.664.3807 Email: mmoss@clevelandohio.gov

+
+ +
+ January 13 + January 27 + February 10 + February 24 + March 10 + March 24 + April 14 + April 28 + May 12 + May 26 + June 9 + June 23 + July 14 + August 25 + September 22 + October 27 + November 10 + November 24 +
+
+ +
+ +
+

NORTHEAST DESIGN REVIEW COMMITTEE

+

The Committee meets on the 1st & 3rd Tuesdays @ 8:00 am at CPL Memorial-Nottingham Branch -17109 Lakeshore Blvd.

+

Sharonda Whatley Phone: 216.664.3806 Email: swhatley@clevelandohio.gov

+
+ +
+ March 2 + March 16 + April 6 + April 20 + May 4 + June 15 + July 6 + August 3 + August 17 +September 21 + October 5 + October 19 + November 2 + November 16 +
+
+ +
+ +
+

SOUTHEAST DESIGN REVIEW COMMITTEE

+

The Committee meets on the 2nd & 4th Wednesdays @ 5:00 pm at York-Rite Mason Temple -13512 Kinsman Road

+

Contact: Marka Fields Phone: 216.664.3465 Email: mfields@clevelandohio.gov

+
+ +
+ January 13 + January 27 + February 10 + February 24 + March 24 + April 14 + April 28 + May 12 + May 26 + June 9 + June 23 + August 25 + September 22 + October 13 + October 27 + November 10 +
+
+ +

 

+ +
+ +
+ + \ No newline at end of file diff --git a/tests/test_cle_design_review.py b/tests/test_cle_design_review.py index b87197b..49b2501 100644 --- a/tests/test_cle_design_review.py +++ b/tests/test_cle_design_review.py @@ -2,7 +2,7 @@ from os.path import dirname, join import pytest # noqa -from city_scrapers_core.constants import ADVISORY_COMMITTEE, PASSED +from city_scrapers_core.constants import ADVISORY_COMMITTEE, PASSED, TENTATIVE from city_scrapers_core.utils import file_response from freezegun import freeze_time @@ -10,13 +10,11 @@ test_response = file_response( join(dirname(__file__), "files", "cle_design_review.html"), - url=( - "http://clevelandohio.gov/CityofCleveland/Home/Government/CityAgencies/CityPlanningCommission/MeetingSchedules" # noqa - ), + url=("https://planning.clevelandohio.gov/designreview/schedule.php"), # noqa ) spider = CleDesignReviewSpider() -freezer = freeze_time("2020-05-19") +freezer = freeze_time("2021-12-01") freezer.start() parsed_items = [item for item in spider.parse(test_response)] @@ -25,7 +23,7 @@ def test_count(): - assert len(parsed_items) == 165 + assert len(parsed_items) == 117 def test_title(): @@ -33,11 +31,14 @@ def test_title(): def test_description(): - assert parsed_items[0]["description"] == "" + assert ( + parsed_items[0]["description"] + == "Due to Covid meetings are being held on WebEx rather than in person. For more information contact asantora@clevelandohio.gov" # noqa + ) def test_start(): - assert parsed_items[0]["start"] == datetime(2020, 1, 2, 9, 0) + assert parsed_items[0]["start"] == datetime(2021, 1, 14, 9, 0) def test_end(): @@ -51,7 +52,7 @@ def test_time_notes(): def test_id(): assert ( parsed_items[0]["id"] - == "cle_design_review/202001020900/x/downtown_flats_design_review_committee" + == "cle_design_review/202101140900/x/downtown_flats_design_review_committee" ) @@ -64,24 +65,19 @@ def test_location(): "name": "City Hall", "address": "601 Lakeside Ave, Room 514, Cleveland OH 44114", } - assert parsed_items[-1]["location"] == { - "address": "13512 Kinsman Road Cleveland, OH", - "name": "York-Rite Mason Temple", - } def test_source(): assert ( parsed_items[0]["source"] - == "http://clevelandohio.gov/CityofCleveland/Home/Government/CityAgencies/CityPlanningCommission/MeetingSchedules" # noqa + == "https://planning.clevelandohio.gov/designreview/schedule.php" # noqa ) def test_links(): - assert parsed_items[0]["links"] == [] - assert parsed_items[1]["links"] == [ + assert parsed_items[0]["links"] == [ { - "href": "http://clevelandohio.gov/sites/default/files/planning/drc/agenda/2020/DF-DRAC-agenda-1-16-20.pdf", # noqa + "href": "https://planning.clevelandohio.gov/designreview/drcagenda/2021/PDF/CPC-Agenda-WebEx-meeting-011421.pdf", # noqa "title": "Agenda", } ] @@ -93,3 +89,67 @@ def test_classification(): def test_all_day(): assert parsed_items[0]["all_day"] is False + + +""" There's a second set of tests to make sure that we're correctly parsing +out details for meetings based on calculated times""" + + +def test_future_meeting_title(): + assert parsed_items[-1]["title"] == "Southeast Design Review Committee" + + +def test_future_meeting_description(): + assert ( + parsed_items[-1]["description"] + == "This is an upcoming meeting - please verify it with staff if you want attend. Due to Covid meetings are being held on WebEx rather than in person. For more information contact mfields@clevelandohio.gov" # noqa + ) + + +def test_future_meeting_start(): + assert parsed_items[-1]["start"] == datetime(2021, 12, 22, 17, 0) + + +def test_future_meeting_end(): + assert parsed_items[-1]["end"] is None + + +def test_future_meeting_time_notes(): + assert parsed_items[-1]["time_notes"] == "" + + +def test_future_meeting_id(): + assert ( + parsed_items[-1]["id"] + == "cle_design_review/202112221700/x/southeast_design_review_committee" + ) + + +def test_future_meeting_status(): + assert parsed_items[-1]["status"] == TENTATIVE + + +def test_future_meeting_location(): + assert parsed_items[-1]["location"] == { + "name": "York-Rite Mason Temple", + "address": "13512 Kinsman Road Cleveland, OH", + } + + +def test_future_meeting_source(): + assert ( + parsed_items[-1]["source"] + == "https://planning.clevelandohio.gov/designreview/schedule.php" # noqa + ) + + +def test_future_meeting_links(): + assert len(parsed_items[-1]["links"]) == 0 + + +def test_future_meeting_classification(): + assert parsed_items[-1]["classification"] == ADVISORY_COMMITTEE + + +def test_future_meeting_all_day(): + assert parsed_items[-1]["all_day"] is False diff --git a/tests/test_meeting_date_calculator.py b/tests/test_meeting_date_calculator.py new file mode 100644 index 0000000..14d3c9f --- /dev/null +++ b/tests/test_meeting_date_calculator.py @@ -0,0 +1,88 @@ +from datetime import date + +import pytest # noqa + +from city_scrapers.utils import calculate_upcoming_meeting_days + +start = date(2021, 12, 1) +end = date(2022, 1, 1) + + +# test a random input - 1 and 3rd tuesday (1) +def test_happy_path(): + expected = [date(2021, 12, 7), date(2021, 12, 21)] + + out = calculate_upcoming_meeting_days(1, [0, 2], start, end) + assert out == expected + + +def test_single_day(): + expected = [date(2021, 12, 14)] + + out = calculate_upcoming_meeting_days(1, [1], start, end) + assert out == expected + + +def test_multiple_months(): + end = date(2022, 2, 1) + expected = [ + date(2021, 12, 14), + date(2021, 12, 28), + date(2022, 1, 11), + date(2022, 1, 25), + ] + + out = calculate_upcoming_meeting_days(1, [1, 3], start, end) + assert out == expected + + +def test_start(): + start = date(2021, 12, 8) + expected = [date(2021, 12, 21)] + + out = calculate_upcoming_meeting_days(1, [0, 2], start, end) + assert out == expected + + +def test_start_is_inclusive(): + start = date(2021, 12, 7) + expected = [date(2021, 12, 7), date(2021, 12, 21)] + + out = calculate_upcoming_meeting_days(1, [0, 2], start, end) + assert out == expected + + +def test_end(): + end = date(2021, 12, 20) + expected = [date(2021, 12, 7)] + + out = calculate_upcoming_meeting_days(1, [0, 2], start, end) + assert out == expected + + +def test_end_is_inclusive(): + end = date(2021, 12, 21) + expected = [date(2021, 12, 7), date(2021, 12, 21)] + + out = calculate_upcoming_meeting_days(1, [0, 2], start, end) + assert out == expected + + +def test_all_5(): + expected = [ + date(2021, 12, 1), + date(2021, 12, 8), + date(2021, 12, 15), + date(2021, 12, 22), + date(2021, 12, 29), + ] + + out = calculate_upcoming_meeting_days(2, [0, 1, 2, 3, 4], start, end) + assert out == expected + + +def test_ordinals_over_4_are_ignored(): + expected = [] + + out = calculate_upcoming_meeting_days(1, [5, 6, 7, 8], start, end) + assert out == expected