additional_data: codelist_code: Add some more codelist data

Adds some of the more obscure codelists for easier human reading:
* locationScope
* beneficiary geoCodeType
* recipient_organization geoCodeType
* funding_organization geoCodeType

Related: ThreeSixtyGiving/grantnav#1081

datastore/additional_data/sources/codelist_code.py

@@ -1,18 +1,18 @@
 import csv
 import requests
+from django.db import transaction
 
 from additional_data.models import CodelistCode
 
 code_lists_urls = [
     "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/grantToIndividualsPurpose.csv",
     "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/grantToIndividualsReason.csv",
     "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/regrantType.csv",
-    # These lists aren't yet ready for use in the datastore
-    # https://github.com/ThreeSixtyGiving/standard/issues/348
-    # https://github.com/ThreeSixtyGiving/standard/issues/349
+    "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/locationScope.csv",
+    "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/geoCodeType.csv",
+    # These codelists aren't yet processed
     # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/countryCode.csv",
     # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/currency.csv",
-    # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/geoCodeType.csv",
 ]
 
 
@@ -22,23 +22,33 @@ class CodeListSource(object):
     """
 
     def import_codelists(self):
-        CodelistCode.objects.all().delete()
-
-        for code_list_url in code_lists_urls:
-            # list name = last item in split -4 to remove extension .csv
-            list_name = code_list_url.split("/")[-1:][0][:-4]
-            with requests.get(code_list_url, stream=True) as r:
-                r.raise_for_status()
-                file_data = csv.DictReader(
-                    r.iter_lines(decode_unicode=True), delimiter=","
-                )
-                for value in file_data:
-                    CodelistCode.objects.create(
-                        code=value["Code"],
-                        title=value["Title"],
-                        description=value["Description"],
-                        list_name=list_name,
+        with transaction.atomic():
+            CodelistCode.objects.all().delete()
+
+            for code_list_url in code_lists_urls:
+                # list name = last item in split -4 to remove extension .csv
+                list_name = code_list_url.split("/")[-1:][0][:-4]
+                print(f"fetching codelist: {list_name}")
+                with requests.get(code_list_url, stream=True) as r:
+                    r.raise_for_status()
+                    file_data = csv.DictReader(
+                        r.iter_lines(decode_unicode=True), delimiter=","
                     )
+                    for value in file_data:
+                        # In https://github.com/ThreeSixtyGiving/standard/blob/main/codelists/geoCodeType.csv
+                        # we have non unique codes with differing descriptions. We have to just take the first
+                        # one we come accross to avoid an integrity error on the unique constraints.
+                        try:
+                            CodelistCode.objects.get(
+                                code=value["Code"], list_name=list_name
+                            )
+                        except CodelistCode.DoesNotExist:
+                            CodelistCode.objects.create(
+                                code=value["Code"],
+                                title=value["Title"],
+                                description=value["Description"],
+                                list_name=list_name,
+                            )
 
     def update_additional_data(self, grant, additional_data):
         # check All the fields in the grant data that use codelists and make additional data field versions of them
@@ -47,6 +57,10 @@ def update_additional_data(self, grant, additional_data):
         secondaryGrantReason = ""
         grantPurpose = []
         regrantType = ""
+        locationScope = ""
+        beneficiary_geoCodeType = ""
+        recipient_organization_geoCodeType = ""
+        funding_organization_geoCodeType = ""
 
         try:
             code = grant["toIndividualsDetails"]["primaryGrantReason"]
@@ -83,11 +97,49 @@ def update_additional_data(self, grant, additional_data):
         except (KeyError, CodelistCode.DoesNotExist):
             pass
 
+        try:
+            code = grant["locationScope"]
+            locationScope = CodelistCode.objects.get(
+                code=code, list_name="locationScope"
+            ).title
+        except (KeyError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            code = grant["beneficiaryLocation"][0]["geoCodeType"]
+            locationScope = CodelistCode.objects.get(
+                code=code, list_name="geoCodeType"
+            ).title
+        except (KeyError, IndexError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            code = grant["fundingOrganization"][0]["location"][0]["geoCodeType"]
+            locationScope = CodelistCode.objects.get(
+                code=code, list_name="geoCodeType"
+            ).title
+        except (KeyError, IndexError, CodelistCode.DoesNotExist):
+            pass
+
+        try:
+            code = grant["recipientOrganization"][0]["location"][0]["geoCodeType"]
+            locationScope = CodelistCode.objects.get(
+                code=code, list_name="geoCodeType"
+            ).title
+        except (KeyError, IndexError, CodelistCode.DoesNotExist):
+            pass
+
         additional_data["codeListLookup"] = {
             "toIndividualsDetails": {
                 "primaryGrantReason": primaryGrantReason,
                 "secondaryGrantReason": secondaryGrantReason,
                 "grantPurpose": grantPurpose,
             },
             "regrantType": regrantType,
+            "locationScope": locationScope,
+            "geoCodeType": {
+                "beneficiaryLocation": beneficiary_geoCodeType,
+                "recipientOrganization": recipient_organization_geoCodeType,
+                "fundingOrganization": funding_organization_geoCodeType,
+            },
         }