Merge branch 'main' into HDXDSYS-843-add-dtm

CHANGELOG.md

@@ -4,12 +4,36 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [0.10.0] - 2024-09-12
+## [0.10.0] - 2024-09-19
 
 ### Added
 
 - IDP scraper
 
+## [0.9.58] - 2024-09-18
+
+### Fixed
+
+- Fix for TCD population dataset change (again)
+
+## [0.9.57] - 2024-09-17
+
+### Added
+
+- Info on IPC country processing
+
+## [0.9.56] - 2024-09-17
+
+### Fixed
+
+- Fix for TCD population dataset change
+
+## [0.9.55] - 2024-09-17
+
+### Fixed
+
+- Changes in HDX Python Scraper package names
+
 ## [0.9.54] - 2024-09-12
 
 ### Fixed

pyproject.toml

@@ -35,10 +35,10 @@ requires-python = ">=3.8"
 
 dependencies = [
     "hapi-schema>=0.8.17",
-    "hdx-python-api>= 6.3.2",
-    "hdx-python-country>= 3.7.7",
+    "hdx-python-api>= 6.3.4",
+    "hdx-python-country>= 3.7.8",
     "hdx-python-database[postgresql]>= 1.3.1",
-    "hdx-python-scraper>= 2.4.1",
+    "hdx-python-scraper>= 2.5.0",
     "hdx-python-utilities>= 3.7.3",
     "libhxl",
     "sqlalchemy"

src/hapi/pipelines/app/__main__.py

@@ -12,8 +12,8 @@
     get_params_from_connection_uri,
 )
 from hdx.facades.keyword_arguments import facade
-from hdx.scraper.utilities import string_params_to_dict
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities import string_params_to_dict
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.dateparse import now_utc
 from hdx.utilities.dictandlist import args_to_dict
 from hdx.utilities.easy_logging import setup_logging

src/hapi/pipelines/app/pipelines.py

@@ -4,8 +4,8 @@
 
 from hdx.api.configuration import Configuration
 from hdx.location.adminlevel import AdminLevel
-from hdx.scraper.runner import Runner
-from hdx.scraper.utilities.sources import Sources
+from hdx.scraper.framework.runner import Runner
+from hdx.scraper.framework.utilities.sources import Sources
 from hdx.utilities.errors_onexit import ErrorsOnExit
 from hdx.utilities.typehint import ListTuple
 from sqlalchemy.orm import Session

src/hapi/pipelines/configs/population.yaml

@@ -945,14 +945,15 @@ population_national:
 
   population_tcd:
     dataset: "cod-ps-tcd"
-    resource: "tcd_admpop_adm0_2021.csv"
-    format: "csv"
+    resource: "tcd_admpop_2023.xlsx"
+    format: "xlsx"
+    sheet: "tcd_admpop_adm0_2023"
     use_hxl: False
     admin_single: "TCD"
     input:
-      - "F_80PLUS"
-      - "M_80PLUS"
-      - "T_80PLUS"
+      - "F_80Plus"
+      - "M_80Plus"
+      - "T_80Plus"
     output:
       - "F_80plus"
       - "M_80plus"
@@ -1913,16 +1914,17 @@ population_adminone:
 
   population_tcd:
     dataset: "cod-ps-tcd"
-    resource: "tcd_admpop_adm1_2021.csv"
-    format: "csv"
+    resource: "tcd_admpop_2023.xlsx"
+    format: "xlsx"
+    sheet: "tcd_admpop_adm1_2023"
     use_hxl: False
     admin:
       - ~
       - "ADM1_PCODE"
     input:
-      - "F_80PLUS"
-      - "M_80PLUS"
-      - "T_80PLUS"
+      - "F_80Plus"
+      - "M_80Plus"
+      - "T_80Plus"
     output:
       - "F_80plus"
       - "M_80plus"
@@ -2798,27 +2800,6 @@ population_admintwo:
       - "#population+m+age_61_plus"
       - "#population+age_61_plus+total"
 
-  population_tcd:
-    dataset: "cod-ps-tcd"
-    resource: "tcd_admpop_adm2_2021.csv"
-    format: "csv"
-    use_hxl: False
-    admin:
-      - ~
-      - "ADM2_PCODE"
-    input:
-      - "F_80PLUS"
-      - "M_80PLUS"
-      - "T_80PLUS"
-    output:
-      - "F_80plus"
-      - "M_80plus"
-      - "T_80plus"
-    output_hxl:
-      - "#population+f+age_80_plus"
-      - "#population+m+age_80_plus"
-      - "#population+age_80_plus+total"
-
   population_ven:
     dataset: "cod-ps-ven"
     resource: "ven_admpop_adm2_2011_v3.csv"

src/hapi/pipelines/database/currency.py

@@ -4,7 +4,7 @@
 
 from hapi_schema.db_currency import DBCurrency
 from hdx.api.configuration import Configuration
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from sqlalchemy.orm import Session
 
 from .base_uploader import BaseUploader

src/hapi/pipelines/database/food_price.py

@@ -5,7 +5,7 @@
 
 from dateutil.relativedelta import relativedelta
 from hapi_schema.db_food_price import DBFoodPrice
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.dateparse import parse_date
 from sqlalchemy.orm import Session
 

src/hapi/pipelines/database/food_security.py

@@ -7,7 +7,7 @@
 from hapi_schema.db_food_security import DBFoodSecurity
 from hdx.api.configuration import Configuration
 from hdx.location.adminlevel import AdminLevel
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.dateparse import parse_date
 from hdx.utilities.typehint import ListTuple
 from sqlalchemy.orm import Session
@@ -45,6 +45,7 @@ def __init__(
         self._adminone = adminone
         self._admintwo = admintwo
         self._configuration = configuration
+        self._country_status = {}
 
     @staticmethod
     def get_admin_level_from_resource_name(
@@ -220,6 +221,9 @@ def process_subnational(
             if admin_level == "adminone":
                 return None
             adminoneinfo = AdminInfo(countryiso3, "NOT GIVEN", "", None, False)
+            self._country_status[countryiso3] = (
+                "Level 1: ignored, Area: Admin 2"
+            )
             return self.get_admintwo_admin2_ref(
                 food_sec_config,
                 warnings,
@@ -261,6 +265,9 @@ def process_subnational(
                 )
                 if not adminoneinfo:
                     return None
+                self._country_status[countryiso3] = (
+                    "Level 1: ignored, Area: Admin 1"
+                )
                 return self.get_adminone_admin2_ref(
                     food_sec_config,
                     warnings,
@@ -278,6 +285,14 @@ def process_subnational(
         )
         if not adminoneinfo:
             return None
+        if countryiso3 in food_sec_config["adm1_only"]:
+            self._country_status[countryiso3] = (
+                "Level 1: Admin 1, Area: ignored"
+            )
+        else:
+            self._country_status[countryiso3] = (
+                "Level 1: Admin 1, Area: Admin 2"
+            )
         if admin_level == "adminone":
             return self.get_adminone_admin2_ref(
                 food_sec_config, warnings, errors, dataset_name, adminoneinfo
@@ -355,3 +370,7 @@ def populate(self) -> None:
             logger.warning(warning)
         for error in sorted(errors):
             logger.error(error)
+        logger.info(f"{dataset_name} - Country Status")
+        for countryiso3 in sorted(self._country_status):
+            status = self._country_status[countryiso3]
+            logger.info(f"{countryiso3}: {status}")

src/hapi/pipelines/database/humanitarian_needs.py

@@ -5,7 +5,7 @@
 
 from hapi_schema.db_humanitarian_needs import DBHumanitarianNeeds
 from hdx.api.configuration import Configuration
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.dictandlist import dict_of_lists_add
 from hdx.utilities.text import get_numeric_if_possible
 from sqlalchemy.orm import Session

src/hapi/pipelines/database/locations.py

@@ -3,7 +3,7 @@
 from hapi_schema.db_location import DBLocation
 from hdx.api.configuration import Configuration
 from hdx.location.country import Country
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.dateparse import parse_date
 from sqlalchemy.orm import Session
 

src/hapi/pipelines/database/metadata.py

@@ -6,8 +6,8 @@
 from hapi_schema.db_resource import DBResource
 from hdx.data.dataset import Dataset
 from hdx.data.resource import Resource
-from hdx.scraper.runner import Runner
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.runner import Runner
+from hdx.scraper.framework.utilities.reader import Read
 from sqlalchemy.orm import Session
 
 from .base_uploader import BaseUploader

src/hapi/pipelines/database/org.py

@@ -6,7 +6,7 @@
 from typing import Dict, NamedTuple
 
 from hapi_schema.db_org import DBOrg
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.dictandlist import write_list_to_csv
 from hdx.utilities.text import normalise
 from sqlalchemy.orm import Session

src/hapi/pipelines/database/org_type.py

@@ -4,7 +4,7 @@
 from typing import Dict
 
 from hapi_schema.db_org_type import DBOrgType
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.text import normalise
 from sqlalchemy.orm import Session
 

src/hapi/pipelines/database/sector.py

@@ -4,7 +4,7 @@
 from typing import Dict
 
 from hapi_schema.db_sector import DBSector
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.text import normalise
 from sqlalchemy.orm import Session
 

src/hapi/pipelines/database/wfp_commodity.py

@@ -5,7 +5,7 @@
 
 from hapi_schema.db_wfp_commodity import DBWFPCommodity
 from hapi_schema.utils.enums import CommodityCategory
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from sqlalchemy.orm import Session
 
 from .base_uploader import BaseUploader

src/hapi/pipelines/database/wfp_market.py

@@ -5,7 +5,7 @@
 
 from hapi_schema.db_wfp_market import DBWFPMarket
 from hdx.location.adminlevel import AdminLevel
-from hdx.scraper.utilities.reader import Read
+from hdx.scraper.framework.utilities.reader import Read
 from hdx.utilities.dictandlist import dict_of_dicts_add
 from sqlalchemy.orm import Session