Handle no results on all datasets, fix NIH date params

si2-urssi · Mar 4, 2024 · 7d6d81b · 7d6d81b
1 parent 56c78a8
commit 7d6d81b
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 10 deletions.
diff --git a/award_pynder/sources/mellon.py b/award_pynder/sources/mellon.py
@@ -337,6 +337,7 @@ def get_data(
             results.append(chunk)
 
         # Concatenate all results
-        all_results = pd.concat(results, ignore_index=True)
+        if len(results) == 0:
+            return pd.DataFrame(columns=ALL_DATASET_FIELDS)
 
-        return all_results
+        return pd.concat(results, ignore_index=True)
diff --git a/award_pynder/sources/nih.py b/award_pynder/sources/nih.py
@@ -37,11 +37,9 @@
 _DEFAULT_CHUNK_SIZE = 500
 _DEFAULT_PARAMS: dict = {
     "criteria": {
-        "award": {
-            "award_notice_date": {
-                "from_date": None,
-                "to_date": None,
-            }
+        "award_notice_date": {
+            "from_date": None,
+            "to_date": None,
         },
         "exclude_subprojects": True,
         "advanced_text_search": {
@@ -77,10 +75,10 @@ def _format_query(
         """Format the full API string with query parameters."""
         # Fill info with always known values
         params = deepcopy(_DEFAULT_PARAMS)
-        params["criteria"]["award"]["award_notice_date"]["from_date"] = (
+        params["criteria"]["award_notice_date"]["from_date"] = (
             NIH._format_datetime(from_datetime) if from_datetime else None
         )
-        params["criteria"]["award"]["award_notice_date"]["to_date"] = (
+        params["criteria"]["award_notice_date"]["to_date"] = (
             NIH._format_datetime(to_datetime) if to_datetime else None
         )
         params["criteria"]["advanced_text_search"]["search_text"] = query or ""
@@ -277,4 +275,7 @@ def get_data(
             chunks.append(chunk)
 
         # Concatenate the chunks
+        if len(chunks) == 0:
+            return pd.DataFrame(columns=ALL_DATASET_FIELDS)
+
         return pd.concat(chunks, ignore_index=True).reset_index(drop=True)
diff --git a/award_pynder/sources/nsf.py b/award_pynder/sources/nsf.py
@@ -277,6 +277,9 @@ def get_data(
                 pbar.update(1)
 
         # Concatenate the chunks
+        if len(chunks) == 0:
+            return pd.DataFrame(columns=ALL_DATASET_FIELDS)
+
         return (
             pd.concat(chunks, ignore_index=True)
             .drop_duplicates(subset="id")

diff --git a/award_pynder/tests/sources/test_nih.py b/award_pynder/tests/sources/test_nih.py
@@ -12,7 +12,7 @@ def test_nih_basics() -> None:
     df = NIH.get_data(
         query="ethnography",
         from_datetime="2023-01-01",
-        to_datetime="2023-02-01",
+        to_datetime="2023-06-01",
         tqdm_kwargs={"leave": False},
     )