From b6445816d88f137d71530ff5655a47ffc2e4b8d4 Mon Sep 17 00:00:00 2001
From: mcarans <rans@email.com>
Date: Wed, 21 Aug 2024 17:25:20 +1200
Subject: [PATCH] Fix logging of HNO warnings and errors

---
 CHANGELOG.md                                  |  6 +++++
 .../pipelines/database/humanitarian_needs.py  | 26 ++++++++++++-------
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ff038596..f989292c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.9.48] - 2024-08-21
+
+### Fixed
+
+- Fix logging of HNO warnings and errors
+
 ## [0.9.47] - 2024-08-20
 
 ### Fixed
diff --git a/src/hapi/pipelines/database/humanitarian_needs.py b/src/hapi/pipelines/database/humanitarian_needs.py
index 470cdd55..0c1c167b 100644
--- a/src/hapi/pipelines/database/humanitarian_needs.py
+++ b/src/hapi/pipelines/database/humanitarian_needs.py
@@ -6,6 +6,7 @@
 from hapi_schema.db_humanitarian_needs import DBHumanitarianNeeds
 from hdx.api.configuration import Configuration
 from hdx.scraper.utilities.reader import Read
+from hdx.utilities.dictandlist import dict_of_lists_add
 from hdx.utilities.text import get_numeric_if_possible
 from sqlalchemy.orm import Session
 
@@ -70,8 +71,8 @@ def populate(self) -> None:
         dataset_name = dataset["name"]
         resource = dataset.get_resource()  # assumes first resource is latest!
         self._metadata.add_resource(dataset_id, resource)
-        negative_values = []
-        rounded_values = []
+        negative_values_by_iso3 = {}
+        rounded_values_by_iso3 = {}
         resource_id = resource["id"]
         resource_name = resource["name"]
         year = int(resource_name[-4:])
@@ -84,6 +85,7 @@ def populate(self) -> None:
             admin2_ref = self.get_admin2_ref(row, dataset_name, errors)
             if not admin2_ref:
                 continue
+            countryiso3 = row["Country ISO3"]
             population_group = row["Population Group"]
             if population_group == "ALL":
                 population_group = "all"
@@ -110,10 +112,14 @@ def create_row(in_col, population_status):
                     return
                 value = get_numeric_if_possible(value)
                 if value < 0:
-                    negative_values.append(str(value))
+                    dict_of_lists_add(
+                        negative_values_by_iso3, countryiso3, str(value)
+                    )
                     return
                 if isinstance(value, float):
-                    rounded_values.append(str(value))
+                    dict_of_lists_add(
+                        rounded_values_by_iso3, countryiso3, str(value)
+                    )
                     value = round(value)
                 humanitarian_needs_row = DBHumanitarianNeeds(
                     resource_hdx_id=resource_id,
@@ -138,18 +144,20 @@ def create_row(in_col, population_status):
             create_row("Targeted", "TGT")
             create_row("Reached", "REA")
 
-            self._session.commit()
+        self._session.commit()
+        for countryiso3, values in negative_values_by_iso3.items():
             add_multi_valued_message(
                 errors,
-                dataset_name,
+                f"{dataset_name} - {countryiso3}",
                 "negative values removed",
-                negative_values,
+                values,
             )
+        for countryiso3, values in rounded_values_by_iso3.items():
             add_multi_valued_message(
                 warnings,
-                dataset_name,
+                f"{dataset_name} - {countryiso3}",
                 "float values rounded",
-                rounded_values,
+                values,
             )
 
         for warning in sorted(warnings):