From 66c9baebdd97610771651479c0b49dbb8f9ad066 Mon Sep 17 00:00:00 2001 From: Darrell Malone Jr Date: Mon, 21 Oct 2024 19:18:06 -0500 Subject: [PATCH] Include Agency ID in employment data Remove "Unknown" option from ethnicity - Returns None is a matching ENUM isn't found --- scrapers/fifty_a/fifty_a/items.py | 3 +++ scrapers/fifty_a/fifty_a/spiders/officer.py | 14 +++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/scrapers/fifty_a/fifty_a/items.py b/scrapers/fifty_a/fifty_a/items.py index 0630ca0..18613e6 100644 --- a/scrapers/fifty_a/fifty_a/items.py +++ b/scrapers/fifty_a/fifty_a/items.py @@ -3,9 +3,12 @@ from scrapers.common.base_item import BaseItem +AGENCY_UID = "54485f3cbe3e49229e3d091f0d12e882" + @dataclass class CommandItem(BaseItem): + agency: str = AGENCY_UID pass diff --git a/scrapers/fifty_a/fifty_a/spiders/officer.py b/scrapers/fifty_a/fifty_a/spiders/officer.py index 7e9bb0e..9c5fa38 100644 --- a/scrapers/fifty_a/fifty_a/spiders/officer.py +++ b/scrapers/fifty_a/fifty_a/spiders/officer.py @@ -10,7 +10,7 @@ from models.enums import Ethnicity from models.officers import CreateOfficer, StateId from scrapers.common.parse import parse_string_to_number -from scrapers.fifty_a.fifty_a.items import OfficerItem +from scrapers.fifty_a.fifty_a.items import OfficerItem, AGENCY_UID class OfficerSpider(CrawlSpider): @@ -93,12 +93,16 @@ def parse_officer(self, response): "badge_number": response.css("span.badge::text").get(), "highest_rank": rank, "unit_uid": response.css("div.command a.command::attr(href)").get(), + "agency_uid": AGENCY_UID, } ) prev_employment = response.css("div.commandhistory a::attr(href)").getall() for emp in prev_employment: - employment_history.append({"unit_uid": emp}) + employment_history.append({ + "unit_uid": emp, + "agency_uid": AGENCY_UID + }) try: officer = CreateOfficer(**officer_data) @@ -142,14 +146,14 @@ def parse_description(description): @staticmethod def map_ethnicity(ethnicity): if not ethnicity: - return Ethnicity.UNKNOWN + return None ethnicity_mapping = { "black": Ethnicity.BLACK_AFRICAN_AMERICAN, "white": Ethnicity.WHITE, "asian": Ethnicity.ASIAN, "hispanic": Ethnicity.HISPANIC_LATINO, - "american indian": Ethnicity.AMERICAN_INDIAN_ALASKA_NATIVE, + "native american": Ethnicity.AMERICAN_INDIAN_ALASKA_NATIVE, "native hawaiian": Ethnicity.NATIVE_HAWAIIAN_PACIFIC_ISLANDER, } @@ -157,7 +161,7 @@ def map_ethnicity(ethnicity): if key in ethnicity.lower(): return value - return Ethnicity.UNKNOWN + return None @staticmethod def parse_complaints(response) -> List[Optional[Dict[str, Any]]]: