diff --git a/CHANGELOG.md b/CHANGELOG.md index e09a5bf8..ff038596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [0.9.47] - 2024-08-20 + +### Fixed + +- Read HNO data from global annual dataset with multiple resources + ## [0.9.46] - 2024-08-19 ### Fixed diff --git a/requirements.txt b/requirements.txt index 455d26a7..5de8788b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ attrs==24.2.0 # jsonlines # jsonschema # referencing -cachetools==5.4.0 +cachetools==5.5.0 # via google-auth certifi==2024.7.4 # via requests @@ -48,7 +48,7 @@ filelock==3.15.4 # via virtualenv frictionless==5.17.0 # via hdx-python-utilities -google-auth==2.33.0 +google-auth==2.34.0 # via # google-auth-oauthlib # gspread @@ -242,13 +242,13 @@ ruamel-yaml==0.18.6 # via hdx-python-utilities ruamel-yaml-clib==0.2.8 # via ruamel-yaml -setuptools==72.2.0 +setuptools==73.0.1 # via ckanapi shellingham==1.5.4 # via typer simpleeval==0.9.13 # via frictionless -simplejson==3.19.2 +simplejson==3.19.3 # via ckanapi six==1.16.0 # via @@ -276,7 +276,7 @@ text-unidecode==1.3 # via python-slugify typeguard==4.3.0 # via inflect -typer==0.12.3 +typer==0.12.4 # via frictionless typing-extensions==4.12.2 # via diff --git a/src/hapi/pipelines/database/admins.py b/src/hapi/pipelines/database/admins.py index bf39c64a..aa785474 100644 --- a/src/hapi/pipelines/database/admins.py +++ b/src/hapi/pipelines/database/admins.py @@ -39,7 +39,7 @@ def __init__( self.admin1_data = {} self.admin2_data = {} - def populate(self): + def populate(self) -> None: logger.info("Populating admin1 table") self._update_admin_table( desired_admin_level="1", diff --git a/src/hapi/pipelines/database/conflict_event.py b/src/hapi/pipelines/database/conflict_event.py index e0b8919f..9387628a 100644 --- a/src/hapi/pipelines/database/conflict_event.py +++ b/src/hapi/pipelines/database/conflict_event.py @@ -32,7 +32,7 @@ def __init__( self._results = results self._config = config - def populate(self): + def populate(self) -> None: logger.info("Populating conflict event table") errors = set() for dataset in self._results.values(): diff --git a/src/hapi/pipelines/database/currency.py b/src/hapi/pipelines/database/currency.py index 0cb1b15a..65dfc9fb 100644 --- a/src/hapi/pipelines/database/currency.py +++ b/src/hapi/pipelines/database/currency.py @@ -21,7 +21,7 @@ def __init__( super().__init__(session) self._configuration = configuration - def populate(self): + def populate(self) -> None: logger.info("Populating currencies table") reader = Read.get_reader("wfp_token") bearer_json = reader.download_json( diff --git a/src/hapi/pipelines/database/food_price.py b/src/hapi/pipelines/database/food_price.py index 0e1e6ea1..e5cb9c1d 100644 --- a/src/hapi/pipelines/database/food_price.py +++ b/src/hapi/pipelines/database/food_price.py @@ -38,7 +38,7 @@ def __init__( self._commodity = commodity self._market = market - def populate(self): + def populate(self) -> None: logger.info("Populating WFP price table") reader = Read.get_reader("hdx") headers, country_iterator = reader.read(datasetinfo=self._datasetinfo) diff --git a/src/hapi/pipelines/database/food_security.py b/src/hapi/pipelines/database/food_security.py index 63cb605f..d14ad6e8 100644 --- a/src/hapi/pipelines/database/food_security.py +++ b/src/hapi/pipelines/database/food_security.py @@ -30,7 +30,7 @@ def __init__( self._admins = admins self._results = results - def populate(self): + def populate(self) -> None: logger.info("Populating food security table") for dataset in self._results.values(): for admin_level, admin_results in dataset["results"].items(): diff --git a/src/hapi/pipelines/database/funding.py b/src/hapi/pipelines/database/funding.py index 80ab3ec3..f264a9cb 100644 --- a/src/hapi/pipelines/database/funding.py +++ b/src/hapi/pipelines/database/funding.py @@ -28,7 +28,7 @@ def __init__( self._locations = locations self._results = results - def populate(self): + def populate(self) -> None: logger.info("Populating funding table") errors = set() for dataset in self._results.values(): diff --git a/src/hapi/pipelines/database/humanitarian_needs.py b/src/hapi/pipelines/database/humanitarian_needs.py index e1e7edd6..470cdd55 100644 --- a/src/hapi/pipelines/database/humanitarian_needs.py +++ b/src/hapi/pipelines/database/humanitarian_needs.py @@ -1,5 +1,6 @@ """Functions specific to the humanitarian needs theme.""" +from datetime import datetime from logging import getLogger from hapi_schema.db_humanitarian_needs import DBHumanitarianNeeds @@ -58,86 +59,84 @@ def get_admin2_ref(self, row, dataset_name, errors): ) return ref - def populate(self): + def populate(self) -> None: logger.info("Populating humanitarian needs table") - reader = Read.get_reader("hdx") - datasets = reader.search_datasets( - filename="Global HPC HNO*", - fq="name:global-hpc-hno-*", - configuration=self._configuration, - ) warnings = set() errors = set() - for dataset in datasets: - negative_values = [] - rounded_values = [] - dataset_name = dataset["name"] - self._metadata.add_dataset(dataset) - time_period = dataset.get_time_period() - time_period_start = time_period["startdate_str"] - time_period_end = time_period["enddate_str"] - resource = dataset.get_resource() - resource_id = resource["id"] - url = resource["url"] - headers, rows = reader.get_tabular_rows(url, dict_form=True) - # Admin 1 PCode,Admin 2 PCode,Sector,Gender,Age Group,Disabled,Population Group,Population,In Need,Targeted,Affected,Reached - for row in rows: - admin2_ref = self.get_admin2_ref(row, dataset_name, errors) - if not admin2_ref: - continue - population_group = row["Population Group"] - if population_group == "ALL": - population_group = "all" - sector = row["Sector"] - sector_code = self._sector.get_sector_code(sector) - if not sector_code: - add_missing_value_message( - errors, dataset_name, "sector", sector - ) - continue - gender = row["Gender"] - if gender == "a": - gender = "all" - age_range = row["Age Range"] - min_age = row["Min Age"] - max_age = row["Max Age"] - disabled_marker = row["Disabled"] - if disabled_marker == "a": - disabled_marker = "all" + reader = Read.get_reader("hdx") + dataset = reader.read_dataset("global-hpc-hno", self._configuration) + self._metadata.add_dataset(dataset) + dataset_id = dataset["id"] + dataset_name = dataset["name"] + resource = dataset.get_resource() # assumes first resource is latest! + self._metadata.add_resource(dataset_id, resource) + negative_values = [] + rounded_values = [] + resource_id = resource["id"] + resource_name = resource["name"] + year = int(resource_name[-4:]) + time_period_start = datetime(year, 1, 1) + time_period_end = datetime(year, 12, 31, 23, 59, 59) + url = resource["url"] + headers, rows = reader.get_tabular_rows(url, dict_form=True) + # Admin 1 PCode,Admin 2 PCode,Sector,Gender,Age Group,Disabled,Population Group,Population,In Need,Targeted,Affected,Reached + for row in rows: + admin2_ref = self.get_admin2_ref(row, dataset_name, errors) + if not admin2_ref: + continue + population_group = row["Population Group"] + if population_group == "ALL": + population_group = "all" + sector = row["Sector"] + sector_code = self._sector.get_sector_code(sector) + if not sector_code: + add_missing_value_message( + errors, dataset_name, "sector", sector + ) + continue + gender = row["Gender"] + if gender == "a": + gender = "all" + age_range = row["Age Range"] + min_age = row["Min Age"] + max_age = row["Max Age"] + disabled_marker = row["Disabled"] + if disabled_marker == "a": + disabled_marker = "all" - def create_row(in_col, population_status): - value = row[in_col] - if value is None: - return - value = get_numeric_if_possible(value) - if value < 0: - negative_values.append(str(value)) - return - if isinstance(value, float): - rounded_values.append(str(value)) - value = round(value) - humanitarian_needs_row = DBHumanitarianNeeds( - resource_hdx_id=resource_id, - admin2_ref=admin2_ref, - gender=gender, - age_range=age_range, - min_age=min_age, - max_age=max_age, - sector_code=sector_code, - population_group=population_group, - population_status=population_status, - disabled_marker=disabled_marker, - population=value, - reference_period_start=time_period_start, - reference_period_end=time_period_end, - ) - self._session.add(humanitarian_needs_row) + def create_row(in_col, population_status): + value = row[in_col] + if value is None: + return + value = get_numeric_if_possible(value) + if value < 0: + negative_values.append(str(value)) + return + if isinstance(value, float): + rounded_values.append(str(value)) + value = round(value) + humanitarian_needs_row = DBHumanitarianNeeds( + resource_hdx_id=resource_id, + admin2_ref=admin2_ref, + gender=gender, + age_range=age_range, + min_age=min_age, + max_age=max_age, + sector_code=sector_code, + population_group=population_group, + population_status=population_status, + disabled_marker=disabled_marker, + population=value, + reference_period_start=time_period_start, + reference_period_end=time_period_end, + ) + self._session.add(humanitarian_needs_row) - create_row("Population", "all") - create_row("Affected", "AFF") - create_row("In Need", "INN") - create_row("Targeted", "TGT") - create_row("Reached", "REA") + create_row("Population", "all") + create_row("Affected", "AFF") + create_row("In Need", "INN") + create_row("Targeted", "TGT") + create_row("Reached", "REA") self._session.commit() add_multi_valued_message( diff --git a/src/hapi/pipelines/database/locations.py b/src/hapi/pipelines/database/locations.py index 443e4e55..67fec9ac 100644 --- a/src/hapi/pipelines/database/locations.py +++ b/src/hapi/pipelines/database/locations.py @@ -27,7 +27,7 @@ def __init__( self.data = {} self._datasetinfo = configuration["locations_hrp_gho"] - def populate(self): + def populate(self) -> None: has_hrp, in_gho = self.read_hrp_gho_data() for country in Country.countriesdata()["countries"].values(): code = country["#country+code+v_iso3"] diff --git a/src/hapi/pipelines/database/metadata.py b/src/hapi/pipelines/database/metadata.py index 1fdb19ff..51936bf3 100644 --- a/src/hapi/pipelines/database/metadata.py +++ b/src/hapi/pipelines/database/metadata.py @@ -5,6 +5,7 @@ from hapi_schema.db_dataset import DBDataset from hapi_schema.db_resource import DBResource from hdx.data.dataset import Dataset +from hdx.data.resource import Resource from hdx.scraper.runner import Runner from hdx.scraper.utilities.reader import Read from sqlalchemy.orm import Session @@ -15,13 +16,15 @@ class Metadata(BaseUploader): - def __init__(self, runner: Runner, session: Session, today: datetime): + def __init__( + self, runner: Runner, session: Session, today: datetime + ) -> None: super().__init__(session) self.runner = runner self.today = today self.dataset_data = [] - def populate(self): + def populate(self) -> None: logger.info("Populating metadata") datasets = self.runner.get_hapi_metadata() for dataset_id, dataset in datasets.items(): @@ -59,9 +62,7 @@ def populate(self): self._session.add(resource_row) self._session.commit() - def add_hapi_metadata( - self, hapi_dataset_metadata: Dict, hapi_resource_metadata: Dict - ): + def add_hapi_dataset_metadata(self, hapi_dataset_metadata: Dict) -> str: dataset_id = hapi_dataset_metadata["hdx_id"] dataset_row = DBDataset( hdx_id=dataset_id, @@ -71,17 +72,28 @@ def add_hapi_metadata( hdx_provider_name=hapi_dataset_metadata["hdx_provider_name"], ) self._session.add(dataset_row) + + self.dataset_data.append(dataset_id) + return dataset_id + + def add_hapi_resource_metadata( + self, dataset_id: str, hapi_resource_metadata: Dict + ) -> None: hapi_resource_metadata["dataset_hdx_id"] = dataset_id hapi_resource_metadata["is_hxl"] = True hapi_resource_metadata["hapi_updated_date"] = self.today resource_row = DBResource(**hapi_resource_metadata) self._session.add(resource_row) - self._session.commit() - self.dataset_data.append(dataset_id) + def add_hapi_metadata( + self, hapi_dataset_metadata: Dict, hapi_resource_metadata: Dict + ) -> None: + dataset_id = self.add_hapi_dataset_metadata(hapi_dataset_metadata) + self.add_hapi_resource_metadata(dataset_id, hapi_resource_metadata) + self._session.commit() - def add_dataset(self, dataset: Dataset): + def get_hapi_dataset_metadata(self, dataset: Dataset) -> Dict: time_period = dataset.get_time_period() hapi_time_period = { "time_period": { @@ -89,9 +101,18 @@ def add_dataset(self, dataset: Dataset): "end": time_period["enddate"], } } - hapi_dataset_metadata = Read.get_hapi_dataset_metadata( - dataset, hapi_time_period - ) + return Read.get_hapi_dataset_metadata(dataset, hapi_time_period) + + def add_dataset(self, dataset: Dataset) -> None: + hapi_dataset_metadata = self.get_hapi_dataset_metadata(dataset) + self.add_hapi_dataset_metadata(hapi_dataset_metadata) + + def add_resource(self, dataset_id: str, resource: Resource) -> None: + hapi_resource_metadata = Read.get_hapi_resource_metadata(resource) + self.add_hapi_resource_metadata(dataset_id, hapi_resource_metadata) + + def add_dataset_first_resource(self, dataset: Dataset) -> None: + hapi_dataset_metadata = self.get_hapi_dataset_metadata(dataset) hapi_resource_metadata = Read.get_hapi_resource_metadata( dataset.get_resource() ) diff --git a/src/hapi/pipelines/database/national_risk.py b/src/hapi/pipelines/database/national_risk.py index 670f2ed5..6848cc3f 100644 --- a/src/hapi/pipelines/database/national_risk.py +++ b/src/hapi/pipelines/database/national_risk.py @@ -26,7 +26,7 @@ def __init__( self._locations = locations self._results = results - def populate(self): + def populate(self) -> None: logger.info("Populating national risk table") for dataset in self._results.values(): time_period_start = dataset["time_period"]["start"] diff --git a/src/hapi/pipelines/database/operational_presence.py b/src/hapi/pipelines/database/operational_presence.py index 2b90c648..5fa3bb44 100644 --- a/src/hapi/pipelines/database/operational_presence.py +++ b/src/hapi/pipelines/database/operational_presence.py @@ -75,7 +75,7 @@ def complete_org_info( # * Org matching self._org.add_or_match_org(org_info) - def populate(self): + def populate(self) -> None: logger.info("Populating operational presence table") operational_presence_rows = [] errors = set() diff --git a/src/hapi/pipelines/database/org.py b/src/hapi/pipelines/database/org.py index fab74d6c..385896c4 100644 --- a/src/hapi/pipelines/database/org.py +++ b/src/hapi/pipelines/database/org.py @@ -47,7 +47,7 @@ def __init__( self.data = {} self._org_map = {} - def populate(self): + def populate(self) -> None: logger.info("Populating org mapping") reader = Read.get_reader() headers, iterator = reader.get_tabular_rows( diff --git a/src/hapi/pipelines/database/org_type.py b/src/hapi/pipelines/database/org_type.py index a9a02659..1b2feb25 100644 --- a/src/hapi/pipelines/database/org_type.py +++ b/src/hapi/pipelines/database/org_type.py @@ -26,7 +26,7 @@ def __init__( self.data = org_type_map self.unmatched = [] - def populate(self): + def populate(self) -> None: logger.info("Populating org type table") def parse_org_type_values(code: str, description: str) -> None: diff --git a/src/hapi/pipelines/database/population.py b/src/hapi/pipelines/database/population.py index f4b2d69a..84cc14df 100644 --- a/src/hapi/pipelines/database/population.py +++ b/src/hapi/pipelines/database/population.py @@ -35,7 +35,7 @@ def __init__( self._admins = admins self._results = results - def populate(self): + def populate(self) -> None: logger.info("Populating population table") for dataset in self._results.values(): time_period_start = dataset["time_period"]["start"] diff --git a/src/hapi/pipelines/database/poverty_rate.py b/src/hapi/pipelines/database/poverty_rate.py index cdb3b891..2b1251d0 100644 --- a/src/hapi/pipelines/database/poverty_rate.py +++ b/src/hapi/pipelines/database/poverty_rate.py @@ -33,7 +33,7 @@ def __init__( self._results = results self._config = config - def populate(self): + def populate(self) -> None: logger.info("Populating poverty rate table") # Loop through datasets (countries) for dataset in self._results.values(): diff --git a/src/hapi/pipelines/database/refugees.py b/src/hapi/pipelines/database/refugees.py index 11f6d65b..20d7052c 100644 --- a/src/hapi/pipelines/database/refugees.py +++ b/src/hapi/pipelines/database/refugees.py @@ -32,7 +32,7 @@ def __init__( self._locations = locations self._results = results - def populate(self): + def populate(self) -> None: logger.info("Populating refugees table") for dataset in self._results.values(): for admin_level, admin_results in dataset["results"].items(): diff --git a/src/hapi/pipelines/database/sector.py b/src/hapi/pipelines/database/sector.py index 5429f03d..dc05e5a9 100644 --- a/src/hapi/pipelines/database/sector.py +++ b/src/hapi/pipelines/database/sector.py @@ -26,7 +26,7 @@ def __init__( self.data = sector_map self.unmatched = [] - def populate(self): + def populate(self) -> None: logger.info("Populating sector table") def parse_sector_values(code: str, name: str): diff --git a/src/hapi/pipelines/database/wfp_commodity.py b/src/hapi/pipelines/database/wfp_commodity.py index 5f6f9382..ad5ebbcc 100644 --- a/src/hapi/pipelines/database/wfp_commodity.py +++ b/src/hapi/pipelines/database/wfp_commodity.py @@ -23,7 +23,7 @@ def __init__( self._datasetinfo = datasetinfo self.data = {} - def populate(self): + def populate(self) -> None: logger.info("Populating WFP commodity table") reader = Read.get_reader("hdx") headers, iterator = reader.read(datasetinfo=self._datasetinfo) diff --git a/src/hapi/pipelines/database/wfp_market.py b/src/hapi/pipelines/database/wfp_market.py index 41c95e60..5fe0d708 100644 --- a/src/hapi/pipelines/database/wfp_market.py +++ b/src/hapi/pipelines/database/wfp_market.py @@ -35,7 +35,7 @@ def __init__( self.data = {} self.name_to_code = {} - def populate(self): + def populate(self) -> None: logger.info("Populating WFP market table") reader = Read.get_reader("hdx") headers, iterator = reader.read(datasetinfo=self._datasetinfo) diff --git a/tests/fixtures/input/Global HPC HNO*_0.json b/tests/fixtures/input/Global HPC HNO*_0.json deleted file mode 100644 index 30ac25ce..00000000 --- a/tests/fixtures/input/Global HPC HNO*_0.json +++ /dev/null @@ -1 +0,0 @@ -{"archived": false, "batch": "c7c4915c-26bf-43bb-804b-fdf39b16229d", "caveats": "", "creator_user_id": "196196be-6037-4488-8b71-d786adf4c081", "data_update_frequency": "7", "dataset_date": "[2024-01-01T00:00:00 TO 2024-12-31T23:59:59]", "dataset_preview": "resource_id", "dataset_source": "Humanitarian partners", "due_date": "2024-08-24T21:33:06", "has_geodata": false, "has_quickcharts": true, "has_showcases": false, "id": "8326ed53-8f3a-47f9-a2aa-83ab4ecee476", "is_requestdata_type": false, "isopen": false, "last_modified": "2024-08-17T21:33:06.471845", "license_id": "cc-by-igo", "license_title": "Creative Commons Attribution for Intergovernmental Organisations", "license_url": "http://creativecommons.org/licenses/by/3.0/igo/legalcode", "maintainer": "196196be-6037-4488-8b71-d786adf4c081", "metadata_created": "2024-08-14T00:01:21.675285", "metadata_modified": "2024-08-17T21:33:51.273750", "methodology": "Registry", "name": "global-hpc-hno-2024", "notes": "This dataset contains standardised Humanitarian Needs Overview data taken from the OCHA HPC Tools system which is under active development. For more detailed but less standardized data on humanitarian needs, see the [Humanitarian Needs Overview data series](https://data.humdata.org/dataset/?dataseries_name=Humanitarian+Needs+Overview).", "num_resources": 1, "num_tags": 3, "organization": {"id": "49f12a06-1605-4f98-89f1-eaec37a0fdfe", "name": "ocha-hpc-tools", "title": "OCHA Humanitarian Programme Cycle Tools (HPC Tools)", "type": "organization", "description": "OCHA\u2019s HPC.tools are online platforms that enable the humanitarian community to structure and manage information around the Humanitarian Programme Cycle (HPC).", "image_url": "", "created": "2024-08-13T10:12:58.027482", "is_organization": true, "approval_status": "approved", "state": "active"}, "overdue_date": "2024-08-31T21:33:06", "owner_org": "49f12a06-1605-4f98-89f1-eaec37a0fdfe", "package_creator": "mcarans", "pageviews_last_14_days": 32, "private": false, "qa_completed": false, "solr_additions": "{\"countries\": [\"Afghanistan\", \"Burkina Faso\", \"Cameroon\", \"Central African Republic\", \"Chad\", \"Colombia\", \"Democratic Republic of the Congo\", \"El Salvador\", \"Ethiopia\", \"Guatemala\", \"Haiti\", \"Honduras\", \"Mali\", \"Mozambique\", \"Myanmar\", \"Niger\", \"Nigeria\", \"Somalia\", \"South Sudan\", \"Sudan\", \"Syrian Arab Republic\", \"Ukraine\", \"Venezuela (Bolivarian Republic of)\", \"Yemen\"]}", "state": "active", "subnational": "1", "title": "Global Humanitarian Programme Cycle, Humanitarian Needs 2024", "total_res_downloads": 21, "type": "dataset", "updated_by_script": "HDX Scraper: HPC HNO (2024-08-17T21:33:06.002523)", "url": null, "version": null, "groups": [{"description": "", "display_name": "Afghanistan", "id": "afg", "image_display_url": "", "name": "afg", "title": "Afghanistan"}, {"description": "", "display_name": "Burkina Faso", "id": "bfa", "image_display_url": "", "name": "bfa", "title": "Burkina Faso"}, {"description": "", "display_name": "Cameroon", "id": "cmr", "image_display_url": "", "name": "cmr", "title": "Cameroon"}, {"description": "", "display_name": "Central African Republic", "id": "caf", "image_display_url": "", "name": "caf", "title": "Central African Republic"}, {"description": "", "display_name": "Chad", "id": "tcd", "image_display_url": "", "name": "tcd", "title": "Chad"}, {"description": "", "display_name": "Colombia", "id": "col", "image_display_url": "", "name": "col", "title": "Colombia"}, {"description": "", "display_name": "Democratic Republic of the Congo", "id": "cod", "image_display_url": "", "name": "cod", "title": "Democratic Republic of the Congo"}, {"description": "", "display_name": "El Salvador", "id": "slv", "image_display_url": "", "name": "slv", "title": "El Salvador"}, {"description": "", "display_name": "Ethiopia", "id": "eth", "image_display_url": "", "name": "eth", "title": "Ethiopia"}, {"description": "", "display_name": "Guatemala", "id": "gtm", "image_display_url": "", "name": "gtm", "title": "Guatemala"}, {"description": "", "display_name": "Haiti", "id": "hti", "image_display_url": "", "name": "hti", "title": "Haiti"}, {"description": "", "display_name": "Honduras", "id": "hnd", "image_display_url": "", "name": "hnd", "title": "Honduras"}, {"description": "", "display_name": "Mali", "id": "mli", "image_display_url": "", "name": "mli", "title": "Mali"}, {"description": "", "display_name": "Mozambique", "id": "moz", "image_display_url": "", "name": "moz", "title": "Mozambique"}, {"description": "", "display_name": "Myanmar", "id": "mmr", "image_display_url": "", "name": "mmr", "title": "Myanmar"}, {"description": "", "display_name": "Niger", "id": "ner", "image_display_url": "", "name": "ner", "title": "Niger"}, {"description": "", "display_name": "Nigeria", "id": "nga", "image_display_url": "", "name": "nga", "title": "Nigeria"}, {"description": "", "display_name": "Somalia", "id": "som", "image_display_url": "", "name": "som", "title": "Somalia"}, {"description": "", "display_name": "South Sudan", "id": "ssd", "image_display_url": "", "name": "ssd", "title": "South Sudan"}, {"description": "", "display_name": "Sudan", "id": "sdn", "image_display_url": "", "name": "sdn", "title": "Sudan"}, {"description": "", "display_name": "Syrian Arab Republic", "id": "syr", "image_display_url": "", "name": "syr", "title": "Syrian Arab Republic"}, {"description": "", "display_name": "Ukraine", "id": "ukr", "image_display_url": "", "name": "ukr", "title": "Ukraine"}, {"description": "", "display_name": "Venezuela (Bolivarian Republic of)", "id": "ven", "image_display_url": "", "name": "ven", "title": "Venezuela (Bolivarian Republic of)"}, {"description": "", "display_name": "Yemen", "id": "yem", "image_display_url": "", "name": "yem", "title": "Yemen"}], "tags": [{"display_name": "humanitarian needs overview-hno", "id": "4d810352-78d9-453c-a48f-6a17b8e6761a", "name": "humanitarian needs overview-hno", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}, {"display_name": "hxl", "id": "a0fbb23a-6aad-4ccc-8062-e9ef9f20e5d2", "name": "hxl", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}, {"display_name": "people in need-pin", "id": "3e0df740-714b-476a-ad14-4fc6fc4fc0ba", "name": "people in need-pin", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}], "relationships_as_subject": [], "relationships_as_object": [], "resources": [{"alt_url": "https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/", "cache_last_updated": null, "cache_url": null, "created": "2024-08-14T00:01:21.681002", "dataset_preview_enabled": true, "datastore_active": false, "description": "HNO data with HXL tags", "download_url": "https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv", "format": "CSV", "fs_check_info": "[{\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-14T01:39:59.885999\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-14T01:40:03.246163\", \"sheet_changes\": [{\"name\": \"__DEFAULT__\", \"event_type\": \"spreadsheet-sheet-changed\", \"changed_fields\": [{\"field\": \"nrows\", \"new_value\": 149727, \"new_display_value\": 149727, \"old_value\": 149971, \"old_display_value\": 149971}]}], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}, {\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-14T01:47:52.757188\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-14T01:47:55.616125\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}, {\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-14T21:02:57.804301\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-14T21:03:00.490415\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}, {\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-15T21:42:53.016245\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-15T21:42:56.186587\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}, {\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-17T21:33:06.167478\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-17T21:33:50.995998\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}]", "hash": "", "hdx_rel_url": "/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv", "id": "8e3931a5-452b-4583-9d02-2247a34e397b", "last_modified": "2024-08-17T21:33:06.471845", "metadata_modified": "2024-08-17T21:33:51.307219", "microdata": false, "mimetype": null, "mimetype_inner": null, "name": "Global HPC HNO 2024", "package_id": "8326ed53-8f3a-47f9-a2aa-83ab4ecee476", "position": 0, "resource_type": "file.upload", "size": 7668047, "state": "active", "url": "https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv", "url_type": "upload"}]} diff --git a/tests/fixtures/input/global-hpc-hno.json b/tests/fixtures/input/global-hpc-hno.json new file mode 100644 index 00000000..559c0000 --- /dev/null +++ b/tests/fixtures/input/global-hpc-hno.json @@ -0,0 +1 @@ +{"archived": false, "batch": "dbe785aa-103a-4aef-b2de-094747f25288", "caveats": "", "creator_user_id": "196196be-6037-4488-8b71-d786adf4c081", "data_update_frequency": "365", "dataset_date": "[2024-01-01T00:00:00 TO 2024-12-31T23:59:59]", "dataset_preview": "resource_id", "dataset_source": "Humanitarian partners", "due_date": "2025-08-20T00:02:09", "has_geodata": false, "has_quickcharts": true, "has_showcases": false, "id": "8326ed53-8f3a-47f9-a2aa-83ab4ecee476", "is_requestdata_type": false, "isopen": false, "last_modified": "2024-08-20T00:02:09.715895", "license_id": "cc-by-igo", "license_title": "Creative Commons Attribution for Intergovernmental Organisations", "license_url": "http://creativecommons.org/licenses/by/3.0/igo/legalcode", "maintainer": "196196be-6037-4488-8b71-d786adf4c081", "maintainer_email": null, "metadata_created": "2024-08-14T00:01:21.675285", "metadata_modified": "2024-08-20T00:02:12.842012", "methodology": "Registry", "name": "global-hpc-hno", "notes": "This dataset contains standardised Humanitarian Needs Overview data taken from the OCHA HPC Tools system which is under active development. For more detailed but less standardized data on humanitarian needs, see the [Humanitarian Needs Overview data series](https://data.humdata.org/dataset/?dataseries_name=Humanitarian+Needs+Overview).", "num_resources": 1, "num_tags": 3, "organization": {"id": "49f12a06-1605-4f98-89f1-eaec37a0fdfe", "name": "ocha-hpc-tools", "title": "OCHA Humanitarian Programme Cycle Tools (HPC Tools)", "type": "organization", "description": "OCHA\u2019s HPC.tools are online platforms that enable the humanitarian community to structure and manage information around the Humanitarian Programme Cycle (HPC).", "image_url": "", "created": "2024-08-13T10:12:58.027482", "is_organization": true, "approval_status": "approved", "state": "active"}, "overdue_date": "2025-10-19T00:02:09", "owner_org": "49f12a06-1605-4f98-89f1-eaec37a0fdfe", "package_creator": "mcarans", "pageviews_last_14_days": 42, "private": false, "qa_completed": false, "solr_additions": "{\"countries\": [\"Afghanistan\", \"Burkina Faso\", \"Cameroon\", \"Central African Republic\", \"Chad\", \"Colombia\", \"Democratic Republic of the Congo\", \"El Salvador\", \"Ethiopia\", \"Guatemala\", \"Haiti\", \"Honduras\", \"Mali\", \"Mozambique\", \"Myanmar\", \"Niger\", \"Nigeria\", \"Somalia\", \"South Sudan\", \"Sudan\", \"Syrian Arab Republic\", \"Ukraine\", \"Venezuela (Bolivarian Republic of)\", \"Yemen\"]}", "state": "active", "subnational": "1", "title": "Global Humanitarian Programme Cycle, Humanitarian Needs", "total_res_downloads": 24, "type": "dataset", "updated_by_script": "HDX Scraper: HPC HNO (2024-08-20T00:02:09.172034)", "url": null, "version": null, "groups": [{"description": "", "display_name": "Afghanistan", "id": "afg", "image_display_url": "", "name": "afg", "title": "Afghanistan"}, {"description": "", "display_name": "Burkina Faso", "id": "bfa", "image_display_url": "", "name": "bfa", "title": "Burkina Faso"}, {"description": "", "display_name": "Cameroon", "id": "cmr", "image_display_url": "", "name": "cmr", "title": "Cameroon"}, {"description": "", "display_name": "Central African Republic", "id": "caf", "image_display_url": "", "name": "caf", "title": "Central African Republic"}, {"description": "", "display_name": "Chad", "id": "tcd", "image_display_url": "", "name": "tcd", "title": "Chad"}, {"description": "", "display_name": "Colombia", "id": "col", "image_display_url": "", "name": "col", "title": "Colombia"}, {"description": "", "display_name": "Democratic Republic of the Congo", "id": "cod", "image_display_url": "", "name": "cod", "title": "Democratic Republic of the Congo"}, {"description": "", "display_name": "El Salvador", "id": "slv", "image_display_url": "", "name": "slv", "title": "El Salvador"}, {"description": "", "display_name": "Ethiopia", "id": "eth", "image_display_url": "", "name": "eth", "title": "Ethiopia"}, {"description": "", "display_name": "Guatemala", "id": "gtm", "image_display_url": "", "name": "gtm", "title": "Guatemala"}, {"description": "", "display_name": "Haiti", "id": "hti", "image_display_url": "", "name": "hti", "title": "Haiti"}, {"description": "", "display_name": "Honduras", "id": "hnd", "image_display_url": "", "name": "hnd", "title": "Honduras"}, {"description": "", "display_name": "Mali", "id": "mli", "image_display_url": "", "name": "mli", "title": "Mali"}, {"description": "", "display_name": "Mozambique", "id": "moz", "image_display_url": "", "name": "moz", "title": "Mozambique"}, {"description": "", "display_name": "Myanmar", "id": "mmr", "image_display_url": "", "name": "mmr", "title": "Myanmar"}, {"description": "", "display_name": "Niger", "id": "ner", "image_display_url": "", "name": "ner", "title": "Niger"}, {"description": "", "display_name": "Nigeria", "id": "nga", "image_display_url": "", "name": "nga", "title": "Nigeria"}, {"description": "", "display_name": "Somalia", "id": "som", "image_display_url": "", "name": "som", "title": "Somalia"}, {"description": "", "display_name": "South Sudan", "id": "ssd", "image_display_url": "", "name": "ssd", "title": "South Sudan"}, {"description": "", "display_name": "Sudan", "id": "sdn", "image_display_url": "", "name": "sdn", "title": "Sudan"}, {"description": "", "display_name": "Syrian Arab Republic", "id": "syr", "image_display_url": "", "name": "syr", "title": "Syrian Arab Republic"}, {"description": "", "display_name": "Ukraine", "id": "ukr", "image_display_url": "", "name": "ukr", "title": "Ukraine"}, {"description": "", "display_name": "Venezuela (Bolivarian Republic of)", "id": "ven", "image_display_url": "", "name": "ven", "title": "Venezuela (Bolivarian Republic of)"}, {"description": "", "display_name": "Yemen", "id": "yem", "image_display_url": "", "name": "yem", "title": "Yemen"}], "tags": [{"display_name": "humanitarian needs overview-hno", "id": "4d810352-78d9-453c-a48f-6a17b8e6761a", "name": "humanitarian needs overview-hno", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}, {"display_name": "hxl", "id": "a0fbb23a-6aad-4ccc-8062-e9ef9f20e5d2", "name": "hxl", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}, {"display_name": "people in need-pin", "id": "3e0df740-714b-476a-ad14-4fc6fc4fc0ba", "name": "people in need-pin", "state": "active", "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1"}], "relationships_as_subject": [], "relationships_as_object": [], "is_fresh": true, "update_status": "fresh", "x_resource_grouping": [], "resources": [{"alt_url": "https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/", "cache_last_updated": null, "cache_url": null, "created": "2024-08-14T00:01:21.681002", "dataset_preview_enabled": true, "datastore_active": false, "description": "HNO data with HXL tags", "download_url": "https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv", "format": "CSV", "fs_check_info": "[{\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-14T01:47:52.757188\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-14T01:47:55.616125\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}, {\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-14T21:02:57.804301\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-14T21:03:00.490415\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}, {\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-15T21:42:53.016245\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-15T21:42:56.186587\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}, {\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-17T21:33:06.167478\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-17T21:33:50.995998\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}, {\"state\": \"processing\", \"message\": \"The processing of the file structure check has started\", \"timestamp\": \"2024-08-20T00:02:09.356315\"}, {\"state\": \"success\", \"message\": \"File structure check completed\", \"timestamp\": \"2024-08-20T00:02:12.571127\", \"sheet_changes\": [], \"hxl_proxy_response\": {\"url_or_filename\": \"https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv\", \"format\": \"CSV\", \"sheets\": [{\"name\": \"__DEFAULT__\", \"nrows\": 149727, \"ncols\": 15, \"is_hidden\": false, \"has_merged_cells\": false, \"is_hxlated\": true, \"header_hash\": \"8a779ecf4d9b710b445d1029ec0d77a6\", \"hxl_header_hash\": \"e00a86817121c4e0b700bd58b2d87ec3\", \"headers\": [\"Country ISO3\", \"Admin 1 PCode\", \"Admin 2 PCode\", \"Sector\", \"Gender\", \"Age Range\", \"Min Age\", \"Max Age\", \"Disabled\", \"Population Group\", \"Population\", \"In Need\", \"Targeted\", \"Affected\", \"Reached\"], \"hxl_headers\": [\"#country+code\", \"#adm1+code\", \"#adm2+code\", \"#sector+code\", \"#gender+code\", \"#age+range\", \"#age+min\", \"#age+max\", \"#is_disabled\", \"#population_group+code\", \"#population\", \"#inneed\", \"#targeted\", \"#affected\", \"#reached\"]}]}}]", "hash": "", "hdx_rel_url": "/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv", "id": "8e3931a5-452b-4583-9d02-2247a34e397b", "last_modified": "2024-08-20T00:02:09.715895", "metadata_modified": "2024-08-20T00:02:12.883425", "microdata": false, "mimetype": null, "mimetype_inner": null, "name": "Global HPC HNO 2024", "package_id": "8326ed53-8f3a-47f9-a2aa-83ab4ecee476", "position": 0, "resource_type": "file.upload", "size": 7668047, "state": "active", "url": "https://data.humdata.org/dataset/8326ed53-8f3a-47f9-a2aa-83ab4ecee476/resource/8e3931a5-452b-4583-9d02-2247a34e397b/download/hpc_hno_2024.csv", "url_type": "upload"}]}