additional_data: codelist_code: Add some more codelist data

Adds some of the more obscure codelists for easier human reading: * locationScope * beneficiary geoCodeType * recipient_organization geoCodeType * funding_organization geoCodeType Related: ThreeSixtyGiving/grantnav#1081
ThreeSixtyGiving · Jan 14, 2025 · 336875b · 336875b
1 parent a1fee35
commit 336875b
Show file tree

Hide file tree

Showing 2 changed files with 90 additions and 22 deletions.
diff --git a/datastore/additional_data/sources/codelist_code.py b/datastore/additional_data/sources/codelist_code.py
@@ -1,18 +1,18 @@
 import csv
 import requests
+from django.db import transaction
 
 from additional_data.models import CodelistCode
 
 code_lists_urls = [
     "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/grantToIndividualsPurpose.csv",
     "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/grantToIndividualsReason.csv",
     "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/regrantType.csv",
-    # These lists aren't yet ready for use in the datastore
-    # https://github.com/ThreeSixtyGiving/standard/issues/348
-    # https://github.com/ThreeSixtyGiving/standard/issues/349
+    "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/locationScope.csv",
+    "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/geoCodeType.csv",
+    # These codelists aren't yet processed
     # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/countryCode.csv",
     # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/currency.csv",
-    # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/geoCodeType.csv",
 ]
 
 
@@ -22,23 +22,34 @@ class CodeListSource(object):
     """
 
     def import_codelists(self):
-        CodelistCode.objects.all().delete()
-
-        for code_list_url in code_lists_urls:
-            # list name = last item in split -4 to remove extension .csv
-            list_name = code_list_url.split("/")[-1:][0][:-4]
-            with requests.get(code_list_url, stream=True) as r:
-                r.raise_for_status()
-                file_data = csv.DictReader(
-                    r.iter_lines(decode_unicode=True), delimiter=","
-                )
-                for value in file_data:
-                    CodelistCode.objects.create(
-                        code=value["Code"],
-                        title=value["Title"],
-                        description=value["Description"],
-                        list_name=list_name,
+        with transaction.atomic():
+            CodelistCode.objects.all().delete()
+
+            for code_list_url in code_lists_urls:
+                # list name = last item in split -4 to remove extension .csv
+                list_name = code_list_url.split("/")[-1:][0][:-4]
+                print(f"fetching codelist: {list_name}")
+                with requests.get(code_list_url, stream=True) as r:
+                    r.raise_for_status()
+                    file_data = csv.DictReader(
+                        r.iter_lines(decode_unicode=True), delimiter=","
                     )
+                    for value in file_data:
+                        # In https://github.com/ThreeSixtyGiving/standard/blob/main/codelists/geoCodeType.csv
+                        # we have non unique codes with differing descriptions. We have to just take the first
+                        # one we come accross to avoid an integrity error on the unique constraints.
+                        # https://github.com/ThreeSixtyGiving/standard/issues/391
+                        try:
+                            CodelistCode.objects.get(
+                                code=value["Code"], list_name=list_name
+                            )
+                        except CodelistCode.DoesNotExist:
+                            CodelistCode.objects.create(
+                                code=value["Code"],
+                                title=value["Title"],
+                                description=value["Description"],
+                                list_name=list_name,
+                            )
 
     def update_additional_data(self, grant, additional_data):
         # check All the fields in the grant data that use codelists and make additional data field versions of them
@@ -47,6 +58,10 @@ def update_additional_data(self, grant, additional_data):
         secondaryGrantReason = ""
         grantPurpose = []
         regrantType = ""
+        locationScope = ""
+        beneficiary_geoCodeTypes = []
+        recipient_organization_geoCodeType = ""
+        funding_organization_geoCodeType = ""
 
         try:
             code = grant["toIndividualsDetails"]["primaryGrantReason"]
@@ -83,11 +98,52 @@ def update_additional_data(self, grant, additional_data):
         except (KeyError, CodelistCode.DoesNotExist):
             pass
 
+        try:
+            code = grant["locationScope"]
+            locationScope = CodelistCode.objects.get(
+                code=code, list_name="locationScope"
+            ).title
+        except (KeyError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            for location in grant["beneficiaryLocation"]:
+                code = location["geoCodeType"]
+                if code_title := CodelistCode.objects.get(
+                    code=location["geoCodeType"], list_name="geoCodeType"
+                ).title:
+                    beneficiary_geoCodeTypes.append(code_title)
+
+        except (KeyError, IndexError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            code = grant["fundingOrganization"][0]["location"][0]["geoCodeType"]
+            funding_organization_geoCodeType = CodelistCode.objects.get(
+                code=code, list_name="geoCodeType"
+            ).title
+        except (KeyError, IndexError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            code = grant["recipientOrganization"][0]["location"][0]["geoCodeType"]
+            recipient_organization_geoCodeType = CodelistCode.objects.get(
+                code=code, list_name="geoCodeType"
+            ).title
+        except (KeyError, IndexError, CodelistCode.DoesNotExist):
+            pass
+
         additional_data["codeListLookup"] = {
             "toIndividualsDetails": {
                 "primaryGrantReason": primaryGrantReason,
                 "secondaryGrantReason": secondaryGrantReason,
                 "grantPurpose": grantPurpose,
             },
             "regrantType": regrantType,
+            "locationScope": locationScope,
+            "geoCodeType": {
+                "beneficiaryLocations": beneficiary_geoCodeTypes,
+                "recipientOrganization0": recipient_organization_geoCodeType,
+                "fundingOrganization0": funding_organization_geoCodeType,
+            },
         }
diff --git a/datastore/tests/test_additional_data_codelist_code.py b/datastore/tests/test_additional_data_codelist_code.py
@@ -4,6 +4,8 @@
 
 class TestCodeLists(TestCase):
     def test_code_list(self):
+        self.maxDiff = None
+
         source = CodeListSource()
         source.import_codelists()
 
@@ -13,25 +15,35 @@ def test_code_list(self):
                 "grantPurpose": ["GTIP170"],
             },
             "regrantType": "FRG010",
+            "locationScope": "GLS040",
+            "fundingOrganization": [{"location": [{"geoCodeType": "CTY"}]}],
+            "recipientOrganization": [{"location": [{"geoCodeType": "LONB"}]}],
+            "beneficiaryLocation": [{"geoCodeType": "MD"}],
         }
 
         additional_data_in = {}
 
-        additional_data_out = {
+        expected_additional_data_out = {
             "codeListLookup": {
                 "toIndividualsDetails": {
                     "primaryGrantReason": "Mental Health",
                     "secondaryGrantReason": "",
                     "grantPurpose": ["Exceptional costs"],
                 },
                 "regrantType": "Common Regrant",
+                "locationScope": "Subnational region",
+                "geoCodeType": {
+                    "beneficiaryLocations": ["Metropolitan Districts"],
+                    "recipientOrganization0": "London Boroughs",
+                    "fundingOrganization0": "Counties",
+                },
             }
         }
 
         source.update_additional_data(grant, additional_data_in)
 
         self.assertEqual(
             additional_data_in,
-            additional_data_out,
+            expected_additional_data_out,
             "The expected additional data isn't correct",
         )