Wait for iNaturalist load completion before compiling statistics (#4104)

WordPress · Apr 15, 2024 · 4051522 · 4051522
1 parent 079bf0e
commit 4051522
Showing 1 changed file with 2 additions and 4 deletions.
diff --git a/catalog/dags/providers/provider_api_scripts/inaturalist.py b/catalog/dags/providers/provider_api_scripts/inaturalist.py
@@ -438,16 +438,14 @@ def create_ingestion_workflow():
                     task_id="consolidate_load_statistics",
                     python_callable=INaturalistDataIngester.consolidate_load_statistics,
                     op_kwargs={
-                        "all_results": XCOM_PULL_TEMPLATE.format(
-                            load_transformed_data.task_id, "return_value"
-                        ),
+                        "all_results": load_transformed_data.output,
                     },
                     doc_md=(
                         "Total load counts across batches from load_transformed_data."
                     ),
                     retries=0,
+                    trigger_rule=TriggerRule.NONE_SKIPPED,
                 )
-
                 (
                     create_loading_table
                     >> get_batches