Remove all testing shortcuts and revert to FULL ETL.

catalyst-cooperative · Nov 17, 2024 · 5904506 · 5904506
1 parent 6834e9e
commit 5904506
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 16 deletions.
diff --git a/.github/workflows/build-deploy-pudl.yml b/.github/workflows/build-deploy-pudl.yml
@@ -137,7 +137,7 @@ jobs:
             --container-env OMP_NUM_THREADS=4 \
             --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \
             --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_GCS_OUTPUT }} \
-            --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_fast.yml" \
+            --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \
             --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \
             --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \
             --container-env ZENODO_TARGET_ENV=${{ (startsWith(github.ref_name, 'v20') && 'production') || 'sandbox' }} \

diff --git a/docker/gcp_pudl_etl.sh b/docker/gcp_pudl_etl.sh
@@ -52,19 +52,19 @@ function run_pudl_etl() {
         --loglevel DEBUG \
         --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
         "$PUDL_SETTINGS_YML" \
+    && pytest \
+        -n auto \
+        --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
+        --etl-settings "$PUDL_SETTINGS_YML" \
+        --live-dbs test/integration test/unit \
+        --no-cov \
+    && pytest \
+        -n auto \
+        --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
+        --etl-settings "$PUDL_SETTINGS_YML" \
+        --live-dbs test/validate \
+        --no-cov \
     && touch "$PUDL_OUTPUT/success"
-    #&& pytest \
-    #    -n auto \
-    #    --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
-    #    --etl-settings "$PUDL_SETTINGS_YML" \
-    #    --live-dbs test/integration test/unit \
-    #    --no-cov \
-    #&& pytest \
-    #    -n auto \
-    #    --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
-    #    --etl-settings "$PUDL_SETTINGS_YML" \
-    #    --live-dbs test/validate \
-    #    --no-cov \
 }
 
 function save_outputs_to_gcs() {
@@ -314,8 +314,7 @@ elif [[ "$BUILD_TYPE" == "workflow_dispatch" ]]; then
 
     copy_outputs_to_distribution_bucket "$BUILD_ID" | tee -a "$LOGFILE"
     DISTRIBUTION_BUCKET_SUCCESS=${PIPESTATUS[0]}
-    # UNCOMMENT AFTER TESTING
-    # remove_dist_path "$BUILD_ID" | tee -a "$LOGFILE"
+    remove_dist_path "$BUILD_ID" | tee -a "$LOGFILE"
 
     # Remove individual parquet outputs and distribute just the zipped parquet
     # archives on Zenodo, due to their number of files limit

diff --git a/src/pudl/transform/vcerare.py b/src/pudl/transform/vcerare.py
@@ -356,7 +356,7 @@ def _load_duckdb_table():
 
 @asset_check(
     asset=out_vcerare__hourly_available_capacity_factor,
-    blocking=False,  # Can only fail for fast ETL!!!
+    blocking=True,
     description="Check that row count matches expected.",
 )
 def check_rows() -> AssetCheckResult: