From b6445816d88f137d71530ff5655a47ffc2e4b8d4 Mon Sep 17 00:00:00 2001 From: mcarans <rans@email.com> Date: Wed, 21 Aug 2024 17:25:20 +1200 Subject: [PATCH] Fix logging of HNO warnings and errors --- CHANGELOG.md | 6 +++++ .../pipelines/database/humanitarian_needs.py | 26 ++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff038596..f989292c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [0.9.48] - 2024-08-21 + +### Fixed + +- Fix logging of HNO warnings and errors + ## [0.9.47] - 2024-08-20 ### Fixed diff --git a/src/hapi/pipelines/database/humanitarian_needs.py b/src/hapi/pipelines/database/humanitarian_needs.py index 470cdd55..0c1c167b 100644 --- a/src/hapi/pipelines/database/humanitarian_needs.py +++ b/src/hapi/pipelines/database/humanitarian_needs.py @@ -6,6 +6,7 @@ from hapi_schema.db_humanitarian_needs import DBHumanitarianNeeds from hdx.api.configuration import Configuration from hdx.scraper.utilities.reader import Read +from hdx.utilities.dictandlist import dict_of_lists_add from hdx.utilities.text import get_numeric_if_possible from sqlalchemy.orm import Session @@ -70,8 +71,8 @@ def populate(self) -> None: dataset_name = dataset["name"] resource = dataset.get_resource() # assumes first resource is latest! self._metadata.add_resource(dataset_id, resource) - negative_values = [] - rounded_values = [] + negative_values_by_iso3 = {} + rounded_values_by_iso3 = {} resource_id = resource["id"] resource_name = resource["name"] year = int(resource_name[-4:]) @@ -84,6 +85,7 @@ def populate(self) -> None: admin2_ref = self.get_admin2_ref(row, dataset_name, errors) if not admin2_ref: continue + countryiso3 = row["Country ISO3"] population_group = row["Population Group"] if population_group == "ALL": population_group = "all" @@ -110,10 +112,14 @@ def create_row(in_col, population_status): return value = get_numeric_if_possible(value) if value < 0: - negative_values.append(str(value)) + dict_of_lists_add( + negative_values_by_iso3, countryiso3, str(value) + ) return if isinstance(value, float): - rounded_values.append(str(value)) + dict_of_lists_add( + rounded_values_by_iso3, countryiso3, str(value) + ) value = round(value) humanitarian_needs_row = DBHumanitarianNeeds( resource_hdx_id=resource_id, @@ -138,18 +144,20 @@ def create_row(in_col, population_status): create_row("Targeted", "TGT") create_row("Reached", "REA") - self._session.commit() + self._session.commit() + for countryiso3, values in negative_values_by_iso3.items(): add_multi_valued_message( errors, - dataset_name, + f"{dataset_name} - {countryiso3}", "negative values removed", - negative_values, + values, ) + for countryiso3, values in rounded_values_by_iso3.items(): add_multi_valued_message( warnings, - dataset_name, + f"{dataset_name} - {countryiso3}", "float values rounded", - rounded_values, + values, ) for warning in sorted(warnings):