diff --git a/CHANGELOG.md b/CHANGELOG.md index 25d1c0fb..ebfde8df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [0.10.0] - 2024-09-12 + +### Added + +- IDP scraper + ## [0.9.54] - 2024-09-12 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 9b57c5f9..737c428b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,8 +34,7 @@ classifiers = [ requires-python = ">=3.8" dependencies = [ - "hapi-schema@git+ssh://git@github.com/OCHA-DAP/hapi-sqlalchemy-schema#HDXDSYS-843-add-dtm-bugfix", - "hapi-schema>=0.8.15", + "hapi-schema>=0.8.17", "hdx-python-api>= 6.3.2", "hdx-python-country>= 3.7.7", "hdx-python-database[postgresql]>= 1.3.1", diff --git a/requirements.txt b/requirements.txt index d386ed27..3bf1551b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,7 @@ email-validator==2.2.0 # via hdx-python-api et-xmlfile==1.1.0 # via openpyxl -filelock==3.16.0 +filelock==3.16.1 # via virtualenv frictionless==5.17.1 # via hdx-python-utilities @@ -54,7 +54,7 @@ greenlet==3.1.0 # via sqlalchemy gspread==6.1.2 # via hdx-python-scraper -hapi-schema @ git+ssh://git@github.com/OCHA-DAP/hapi-sqlalchemy-schema@744423d3c309dc2ddc106d51ef28e01f8e03b876#HDXDSYS-843-add-dtm-bugfix +hapi-schema==0.8.17 # via hapi-pipelines (pyproject.toml) hdx-python-api==6.3.4 # via @@ -134,7 +134,7 @@ packaging==24.1 # via pytest petl==1.7.15 # via frictionless -platformdirs==4.3.3 +platformdirs==4.3.6 # via virtualenv pluggy==1.5.0 # via pytest @@ -156,9 +156,9 @@ pyasn1==0.6.1 # rsa pyasn1-modules==0.4.1 # via google-auth -pydantic==2.9.1 +pydantic==2.9.2 # via frictionless -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via rich @@ -283,7 +283,7 @@ urllib3==2.2.3 # requests validators==0.34.0 # via frictionless -virtualenv==20.26.4 +virtualenv==20.26.5 # via pre-commit wheel==0.44.0 # via libhxl diff --git a/src/hapi/pipelines/database/idps.py b/src/hapi/pipelines/database/idps.py index 13a926e2..29ed8cf9 100644 --- a/src/hapi/pipelines/database/idps.py +++ b/src/hapi/pipelines/database/idps.py @@ -28,6 +28,8 @@ def __init__( self._results = results def populate(self) -> None: + # TODO: This might be better suited to just work with the DTM resource + # directly as done with HNO, rather than using a configurable scraper logger.info("Populating IDPs table") errors = set() # self._results is a dictionary where the keys are the HDX dataset ID and the