HDXDSYS-1010 Update csv mapping used for use_live = False in HDX Pyth…

…on Country and allow use_live to be set universally (#52) * Update downloaded live data Allow universal setting of use_live default Update pre-commit config Add test
OCHA-DAP · Sep 6, 2024 · f9697b0 · f9697b0
1 parent 4ae2760
commit f9697b0
Show file tree

Hide file tree

Showing 8 changed files with 329 additions and 298 deletions.
diff --git a/.config/pre-commit-config.yaml b/.config/pre-commit-config.yaml
@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.1
+    rev: v0.6.4
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,7 +17,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.2.13
+    rev: 0.4.5
     hooks:
       # Run the pip compile
       - id: pip-compile

diff --git a/pyproject.toml b/pyproject.toml
@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-utilities>=3.7.2",
+    "hdx-python-utilities>=3.7.3",
     "libhxl>=5.2.1",
     "pyphonetics",
 ]

diff --git a/requirements.txt b/requirements.txt
@@ -2,13 +2,13 @@
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.7.0
     # via pydantic
-attrs==23.2.0
+attrs==24.2.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
     #   referencing
-certifi==2024.7.4
+certifi==2024.8.30
     # via requests
 cfgv==3.4.0
     # via pre-commit
@@ -18,23 +18,23 @@ charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via typer
-coverage==7.6.0
+coverage==7.6.1
     # via pytest-cov
 distlib==0.3.8
     # via virtualenv
 et-xmlfile==1.1.0
     # via openpyxl
 filelock==3.15.4
     # via virtualenv
-frictionless==5.17.0
+frictionless==5.17.1
     # via hdx-python-utilities
-hdx-python-utilities==3.7.2
+hdx-python-utilities==3.7.3
     # via hdx-python-country (pyproject.toml)
 humanize==4.10.0
     # via frictionless
 identify==2.6.0
     # via pre-commit
-idna==3.7
+idna==3.8
     # via requests
 ijson==3.3.0
     # via hdx-python-utilities
@@ -82,17 +82,17 @@ ply==3.11
     # via
     #   jsonpath-ng
     #   libhxl
-pre-commit==3.7.1
+pre-commit==3.8.0
     # via hdx-python-country (pyproject.toml)
-pydantic==2.8.2
+pydantic==2.9.0
     # via frictionless
-pydantic-core==2.20.1
+pydantic-core==2.23.2
     # via pydantic
 pygments==2.18.0
     # via rich
 pyphonetics==0.5.3
     # via hdx-python-country (pyproject.toml)
-pytest==8.3.1
+pytest==8.3.2
     # via
     #   hdx-python-country (pyproject.toml)
     #   pytest-cov
@@ -107,7 +107,7 @@ python-io-wrapper==0.3.1
     # via libhxl
 python-slugify==8.0.4
     # via frictionless
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via
     #   frictionless
     #   pre-commit
@@ -127,9 +127,9 @@ requests-file==2.1.0
     # via hdx-python-utilities
 rfc3986==2.0.0
     # via frictionless
-rich==13.7.1
+rich==13.8.0
     # via typer
-rpds-py==0.19.0
+rpds-py==0.20.0
     # via
     #   jsonschema
     #   referencing
@@ -155,14 +155,16 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typer==0.12.3
+typer==0.12.5
     # via frictionless
 typing-extensions==4.12.2
     # via
     #   frictionless
     #   pydantic
     #   pydantic-core
     #   typer
+tzdata==2024.1
+    # via pydantic
 unidecode==1.3.8
     # via
     #   libhxl
@@ -171,16 +173,18 @@ urllib3==2.2.2
     # via
     #   libhxl
     #   requests
-validators==0.33.0
+validators==0.34.0
     # via frictionless
 virtualenv==20.26.3
     # via pre-commit
-wheel==0.43.0
+wheel==0.44.0
     # via libhxl
 xlrd==2.0.1
     # via hdx-python-utilities
 xlrd3==1.1.0
     # via libhxl
+xlsx2csv==0.8.3
+    # via hdx-python-utilities
 xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0

diff --git a/src/hdx/location/Countries & Territories Taxonomy MVP - C&T Taxonomy with HXL Tags.csv b/src/hdx/location/Countries & Territories Taxonomy MVP - C&T Taxonomy with HXL Tags.csv
diff --git a/src/hdx/location/country.py b/src/hdx/location/country.py
@@ -68,6 +68,8 @@ class Country:
         "KINGDOM",
         "PROTECTORATE",
     ]
+    _use_live_default = True
+    _use_live = _use_live_default
     _countriesdata = None
     _ochaurl_default = "https://docs.google.com/spreadsheets/d/1NjSI2LaS3SqbgYc0HdD8oIb7lofGtiHgoKKATCpwVdY/export?format=csv&gid=1088874596"
     _ochaurl = _ochaurl_default
@@ -204,7 +206,7 @@ def sort_list(colname):
     @classmethod
     def countriesdata(
         cls,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         country_name_overrides: Dict = None,
         country_name_mappings: Dict = None,
     ) -> List[Dict[str, Dict]]:
@@ -245,6 +247,22 @@ def countriesdata(
             cls.set_countriesdata(countries)
         return cls._countriesdata
 
+    @classmethod
+    def set_use_live_default(cls, use_live: Optional[bool] = None) -> None:
+        """
+        Set the default for use_live which defines if latest data is obtained
+        from the web rather than taking data from a static file in the package.
+
+        Args:
+            use_live (bool): Default value to use for use_live. Defaults to internal value (True).
+
+        Returns:
+            None
+        """
+        if use_live is None:
+            use_live = cls._use_live_default
+        cls._use_live = use_live
+
     @classmethod
     def set_ocha_url(cls, url: Optional[str] = None) -> None:
         """
@@ -290,7 +308,7 @@ def set_country_name_mappings(cls, country_name_mappings: Dict) -> None:
     def get_country_info_from_iso3(
         cls,
         iso3: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[Dict[str, str]]:
         """Get country information from ISO3 code
@@ -316,7 +334,7 @@ def get_country_info_from_iso3(
     def get_country_name_from_iso3(
         cls,
         iso3: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
         formal: bool = False,
     ) -> Optional[str]:
@@ -353,7 +371,7 @@ def get_country_name_from_iso3(
     def get_currency_from_iso3(
         cls,
         iso3: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[int]:
         """Get currency code from ISO3 code
@@ -379,7 +397,7 @@ def get_currency_from_iso3(
     def get_iso2_from_iso3(
         cls,
         iso3: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[str]:
         """Get ISO2 from ISO3 code
@@ -405,7 +423,7 @@ def get_iso2_from_iso3(
     def get_iso3_from_iso2(
         cls,
         iso2: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[str]:
         """Get ISO3 from ISO2 code
@@ -431,7 +449,7 @@ def get_iso3_from_iso2(
     def get_country_info_from_iso2(
         cls,
         iso2: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[Dict[str, str]]:
         """Get country name from ISO2 code
@@ -457,7 +475,7 @@ def get_country_info_from_iso2(
     def get_country_name_from_iso2(
         cls,
         iso2: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
         formal: bool = False,
     ) -> Optional[str]:
@@ -485,7 +503,7 @@ def get_country_name_from_iso2(
     def get_currency_from_iso2(
         cls,
         iso2: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[str]:
         """Get currency from ISO2 code
@@ -511,7 +529,7 @@ def get_currency_from_iso2(
     def get_m49_from_iso3(
         cls,
         iso3: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[int]:
         """Get M49 from ISO3 code
@@ -537,7 +555,7 @@ def get_m49_from_iso3(
     def get_iso3_from_m49(
         cls,
         m49: int,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[str]:
         """Get ISO3 from M49 code
@@ -563,7 +581,7 @@ def get_iso3_from_m49(
     def get_country_info_from_m49(
         cls,
         m49: int,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[Dict[str, str]]:
         """Get country name from M49 code
@@ -587,7 +605,7 @@ def get_country_info_from_m49(
     def get_country_name_from_m49(
         cls,
         m49: int,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
         formal: bool = False,
     ) -> Optional[str]:
@@ -615,7 +633,7 @@ def get_country_name_from_m49(
     def get_currency_from_m49(
         cls,
         m49: int,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[str]:
         """Get currency from M49 code
@@ -718,7 +736,7 @@ def simplify_countryname(cls, country: str) -> (str, List[str]):
     def get_iso3_country_code(
         cls,
         country: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> Optional[str]:
         """Get ISO3 code for cls. Only exact matches or None are returned.
@@ -778,7 +796,7 @@ def get_iso3_country_code(
     def get_iso3_country_code_fuzzy(
         cls,
         country: str,
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
         min_chars: int = 5,
     ) -> Tuple[Optional[str], bool]:
@@ -868,7 +886,7 @@ def remove_matching_from_list(wordlist, word_or_part):
     def get_countries_in_region(
         cls,
         region: Union[int, str],
-        use_live: bool = True,
+        use_live: bool = _use_live,
         exception: Optional[ExceptionUpperBound] = None,
     ) -> List[str]:
         """Get countries (ISO3 codes) in region

diff --git a/tests/hdx/location/__init__.py b/tests/hdx/location/__init__.py
@@ -1,7 +1,8 @@
 from hdx.location.country import Country
 
+Country.set_use_live_default(False)
+
 Country.countriesdata(
-    use_live=False,
     country_name_overrides={"PSE": "oPt"},
     country_name_mappings={"Congo DR": "COD"},
 )
diff --git a/tests/hdx/location/test_country.py b/tests/hdx/location/test_country.py
@@ -207,7 +207,7 @@ def test_get_country_info_from_iso2(self):
             "#country+formal+i_en+name+v_unterm": "",
             "#country+name+preferred": "Taiwan (Province of China)",
             "#country+name+short+v_reliefweb": "",
-            "#country+regex": ".*taiwan|.*taipei|.*formosa|^(?!.*\\bdem)(?!.*\\bpe)(?!.*\\bdr)(^rep.*).*\\bchina.*(?!.*\\bdem.*)(?!\\bpe.*)(?!.*\\bdr.*).*|^ROC$|^taiwan r\\.?o\\.?c\\.?$",
+            "#country+regex": "taiwan|taipei|formosa|^(?!.*peo)(?=.*rep).*china",
             "#currency+code": "TWD",
             "#date+start": "1974-01-01",
             "#geo+admin_level": "0",
@@ -373,7 +373,7 @@ def test_get_country_info_from_m49(self):
             "#country+formal+i_en+name+v_unterm": "the Independent State of Samoa",
             "#country+name+preferred": "Samoa",
             "#country+name+short+v_reliefweb": "",
-            "#country+regex": "^(?!.*amer.*)samoa|(\\bindep.*samoa)|^west.*samoa",
+            "#country+regex": "^(?!.*amer).*samoa",
             "#currency+code": "WST",
             "#date+start": "1998-02-05",
             "#geo+admin_level": "0",
@@ -691,6 +691,14 @@ def test_get_countries_in_region(self):
                 "NOTEXIST", exception=LocationError
             )
 
+    def test_use_live_default(self):
+        Country.set_use_live_default(True)
+        assert Country._use_live is True
+        Country.set_use_live_default(False)
+        assert Country._use_live is False
+        Country.set_use_live_default(None)
+        assert Country._use_live is True
+
     def test_ocha_feed_file_working(self):
         countries = hxl.data(
             script_dir_plus_file("Countries_UZB_Deleted.csv", TestCountry),

diff --git a/tests/hdx/location/test_wfp_exchangerates.py b/tests/hdx/location/test_wfp_exchangerates.py
@@ -26,7 +26,7 @@ def date(self):
 
         def test_get_currencies(self, wfp_fx):
             currencies = wfp_fx.get_currencies()
-            assert len(currencies) == 126
+            assert len(currencies) == 127
 
         def test_get_historic_rates(self, wfp_fx, currency, date):
             assert (