Include Agency ID in employment data

Remove "Unknown" option from ethnicity - Returns None is a matching ENUM isn't found
National-Police-Data-Coalition · Oct 22, 2024 · 66c9bae · 66c9bae
1 parent b8381ca
commit 66c9bae
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 5 deletions.
diff --git a/scrapers/fifty_a/fifty_a/items.py b/scrapers/fifty_a/fifty_a/items.py
@@ -3,9 +3,12 @@
 
 from scrapers.common.base_item import BaseItem
 
+AGENCY_UID = "54485f3cbe3e49229e3d091f0d12e882"
+
 
 @dataclass
 class CommandItem(BaseItem):
+    agency: str = AGENCY_UID
     pass
 
 

diff --git a/scrapers/fifty_a/fifty_a/spiders/officer.py b/scrapers/fifty_a/fifty_a/spiders/officer.py
@@ -10,7 +10,7 @@
 from models.enums import Ethnicity
 from models.officers import CreateOfficer, StateId
 from scrapers.common.parse import parse_string_to_number
-from scrapers.fifty_a.fifty_a.items import OfficerItem
+from scrapers.fifty_a.fifty_a.items import OfficerItem, AGENCY_UID
 
 
 class OfficerSpider(CrawlSpider):
@@ -93,12 +93,16 @@ def parse_officer(self, response):
                 "badge_number": response.css("span.badge::text").get(),
                 "highest_rank": rank,
                 "unit_uid": response.css("div.command a.command::attr(href)").get(),
+                "agency_uid": AGENCY_UID,
             }
         )
 
         prev_employment = response.css("div.commandhistory a::attr(href)").getall()
         for emp in prev_employment:
-            employment_history.append({"unit_uid": emp})
+            employment_history.append({
+                "unit_uid": emp,
+                "agency_uid": AGENCY_UID
+            })
 
         try:
             officer = CreateOfficer(**officer_data)
@@ -142,22 +146,22 @@ def parse_description(description):
     @staticmethod
     def map_ethnicity(ethnicity):
         if not ethnicity:
-            return Ethnicity.UNKNOWN
+            return None
 
         ethnicity_mapping = {
             "black": Ethnicity.BLACK_AFRICAN_AMERICAN,
             "white": Ethnicity.WHITE,
             "asian": Ethnicity.ASIAN,
             "hispanic": Ethnicity.HISPANIC_LATINO,
-            "american indian": Ethnicity.AMERICAN_INDIAN_ALASKA_NATIVE,
+            "native american": Ethnicity.AMERICAN_INDIAN_ALASKA_NATIVE,
             "native hawaiian": Ethnicity.NATIVE_HAWAIIAN_PACIFIC_ISLANDER,
         }
 
         for key, value in ethnicity_mapping.items():
             if key in ethnicity.lower():
                 return value
 
-        return Ethnicity.UNKNOWN
+        return None
 
     @staticmethod
     def parse_complaints(response) -> List[Optional[Dict[str, Any]]]: