Merge pull request #162 from OCHA-DAP/nga_population_fix

HDXDSYS-1029 HAPI Pipelines fix for NGA population dataset update

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.9.54] - 2024-09-12
+
+### Fixed
+
+- Updated Nigeria population and tests
+
 ## [0.9.53] - 2024-09-06
 
 ### Fixed

requirements.txt

@@ -12,8 +12,6 @@ cachetools==5.5.0
     # via google-auth
 certifi==2024.8.30
     # via requests
-cffi==1.17.0
-    # via cryptography
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
@@ -26,8 +24,6 @@ click==8.1.7
     # via typer
 coverage==7.6.1
     # via pytest-cov
-cryptography==43.0.1
-    # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
 distlib==0.3.8
@@ -44,7 +40,7 @@ email-validator==2.2.0
     # via hdx-python-api
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.15.4
+filelock==3.16.0
     # via virtualenv
 frictionless==5.17.1
     # via hdx-python-utilities
@@ -54,17 +50,17 @@ google-auth==2.34.0
     #   gspread
 google-auth-oauthlib==1.2.1
     # via gspread
-greenlet==3.0.3
+greenlet==3.1.0
     # via sqlalchemy
 gspread==6.1.2
     # via hdx-python-scraper
 hapi-schema==0.8.16
     # via hapi-pipelines (pyproject.toml)
-hdx-python-api==6.3.2
+hdx-python-api==6.3.4
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-scraper
-hdx-python-country==3.7.7
+hdx-python-country==3.7.8
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -89,7 +85,7 @@ idna==3.8
     #   requests
 ijson==3.3.0
     # via hdx-python-utilities
-inflect==7.3.1
+inflect==7.4.0
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
@@ -124,10 +120,8 @@ markupsafe==2.1.5
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
-more-itertools==10.4.0
+more-itertools==10.5.0
     # via inflect
-ndg-httpsclient==0.5.1
-    # via hdx-python-api
 nodeenv==1.9.1
     # via pre-commit
 num2words==0.5.13
@@ -140,7 +134,7 @@ packaging==24.1
     # via pytest
 petl==1.7.15
     # via frictionless
-platformdirs==4.2.2
+platformdirs==4.3.2
     # via virtualenv
 pluggy==1.5.0
     # via pytest
@@ -156,29 +150,21 @@ psycopg==3.2.1
     # via hdx-python-database
 psycopg-binary==3.2.1
     # via psycopg
-pyasn1==0.6.0
+pyasn1==0.6.1
     # via
-    #   hdx-python-api
-    #   ndg-httpsclient
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.4.0
+pyasn1-modules==0.4.1
     # via google-auth
-pycparser==2.22
-    # via cffi
-pydantic==2.8.2
+pydantic==2.9.1
     # via frictionless
-pydantic-core==2.20.1
+pydantic-core==2.23.3
     # via pydantic
 pygments==2.18.0
     # via rich
-pyopenssl==24.2.1
-    # via
-    #   hdx-python-api
-    #   ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
-pytest==8.3.2
+pytest==8.3.3
     # via
     #   hapi-pipelines (pyproject.toml)
     #   pytest-check
@@ -214,7 +200,7 @@ referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
-regex==2024.7.24
+regex==2024.9.11
     # via hdx-python-scraper
 requests==2.32.3
     # via
@@ -230,7 +216,7 @@ requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
-rich==13.8.0
+rich==13.8.1
     # via typer
 rpds-py==0.20.0
     # via
@@ -242,7 +228,7 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==74.1.1
+setuptools==74.1.2
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -259,7 +245,7 @@ six==1.16.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-sqlalchemy==2.0.33
+sqlalchemy==2.0.34
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hapi-schema
@@ -297,7 +283,7 @@ urllib3==2.2.2
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.26.3
+virtualenv==20.26.4
     # via pre-commit
 wheel==0.44.0
     # via libhxl

src/hapi/pipelines/configs/population.yaml

@@ -801,7 +801,7 @@ population_national:
 
   population_nga:
     dataset: "cod-ps-nga"
-    resource: "nga_admpop_adm0_2020.csv"
+    resource: "nga_admpop_adm0_2022.csv"
     format: "csv"
     use_hxl: False
     admin_single: "NGA"
@@ -1777,7 +1777,7 @@ population_adminone:
 
   population_nga:
     dataset: "cod-ps-nga"
-    resource: "nga_admpop_adm1_2020.csv"
+    resource: "nga_admpop_adm1_2022.csv"
     format: "csv"
     use_hxl: False
     admin:
@@ -2684,28 +2684,6 @@ population_admintwo:
       - "#population+m+age_80_plus"
       - "#population+age_80_plus+total"
 
-  population_nga:
-    dataset: "cod-ps-nga"
-    resource: "nga_admpop_adm2_2020.csv"
-    format: "csv"
-    use_hxl: False
-    admin:
-      - ~
-      - "ADM2_PCODE"
-    admin_exact: True
-    input:
-      - "F_80Plus"
-      - "M_80Plus"
-      - "T_80Plus"
-    output:
-      - "F_80plus"
-      - "M_80plus"
-      - "T_80plus"
-    output_hxl:
-      - "#population+f+age_80_plus"
-      - "#population+m+age_80_plus"
-      - "#population+age_80_plus+total"
-
   population_sdn:
     dataset: "cod-ps-sdn"
     resource: "sdn_admpop_adm2_2024.csv"

tests/fixtures/input/population_nga_nga_admpop_adm0_2022.csv

@@ -0,0 +1,2 @@
+year,ISO3,ADM0_EN,ADM0_PCODE,F_TL,M_TL,T_TL,F_00_04,F_05_09,F_10_14,F_15_19,F_20_24,F_25_29,F_30_34,F_35_39,F_40_44,F_45_49,F_50_54,F_55_59,F_60_64,F_65_69,F_70_74,F_75_79,F_80Plus,M_00_04,M_05_09,M_10_14,M_15_19,M_20_24,M_25_29,M_30_34,M_35_39,M_40_44,M_45_49,M_50_54,M_55_59,M_60_64,M_65_69,M_70_74,M_75_79,M_80Plus,T_00_04,T_05_09,T_10_14,T_15_19,T_20_24,T_25_29,T_30_34,T_35_39,T_40_44,T_45_49,T_50_54,T_55_59,T_60_64,T_65_69,T_70_74,T_75_79,T_80Plus
+2022,NGA,Nigeria,NG,108459809,108363459,216798930,16705671,15024834,13802175,10797472,9192425,7621127,6725245,6864567,6162995,4625048,3518298,2616803,1900536,1251477,791337,469872,363089,16890357,15046183,13775731,11063894,9701073,8257454,7005019,5969371,5068459,4075201,3310355,2756987,2153817,1460421,932404,515436,368248,33596028,30071017,27577906,21861366,18893498,15878581,13730264,12833938,11231454,8700249,6828653,5373790,4054353,2711898,1723741,985308,731337