diff --git a/.github/workflows/build-deploy-pudl.yml b/.github/workflows/build-deploy-pudl.yml index 24f6ff4c2..6ef415e52 100644 --- a/.github/workflows/build-deploy-pudl.yml +++ b/.github/workflows/build-deploy-pudl.yml @@ -137,7 +137,7 @@ jobs: --container-env OMP_NUM_THREADS=4 \ --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \ --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_GCS_OUTPUT }} \ - --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_fast.yml" \ + --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \ --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \ --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \ --container-env ZENODO_TARGET_ENV=${{ (startsWith(github.ref_name, 'v20') && 'production') || 'sandbox' }} \ diff --git a/docker/gcp_pudl_etl.sh b/docker/gcp_pudl_etl.sh index 0cada763d..0df156a9d 100644 --- a/docker/gcp_pudl_etl.sh +++ b/docker/gcp_pudl_etl.sh @@ -52,19 +52,19 @@ function run_pudl_etl() { --loglevel DEBUG \ --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ "$PUDL_SETTINGS_YML" \ + && pytest \ + -n auto \ + --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ + --etl-settings "$PUDL_SETTINGS_YML" \ + --live-dbs test/integration test/unit \ + --no-cov \ + && pytest \ + -n auto \ + --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ + --etl-settings "$PUDL_SETTINGS_YML" \ + --live-dbs test/validate \ + --no-cov \ && touch "$PUDL_OUTPUT/success" - #&& pytest \ - # -n auto \ - # --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ - # --etl-settings "$PUDL_SETTINGS_YML" \ - # --live-dbs test/integration test/unit \ - # --no-cov \ - #&& pytest \ - # -n auto \ - # --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \ - # --etl-settings "$PUDL_SETTINGS_YML" \ - # --live-dbs test/validate \ - # --no-cov \ } function save_outputs_to_gcs() { @@ -314,8 +314,7 @@ elif [[ "$BUILD_TYPE" == "workflow_dispatch" ]]; then copy_outputs_to_distribution_bucket "$BUILD_ID" | tee -a "$LOGFILE" DISTRIBUTION_BUCKET_SUCCESS=${PIPESTATUS[0]} - # UNCOMMENT AFTER TESTING - # remove_dist_path "$BUILD_ID" | tee -a "$LOGFILE" + remove_dist_path "$BUILD_ID" | tee -a "$LOGFILE" # Remove individual parquet outputs and distribute just the zipped parquet # archives on Zenodo, due to their number of files limit diff --git a/src/pudl/transform/vcerare.py b/src/pudl/transform/vcerare.py index 899c4f03e..6ee9e6ce8 100644 --- a/src/pudl/transform/vcerare.py +++ b/src/pudl/transform/vcerare.py @@ -356,7 +356,7 @@ def _load_duckdb_table(): @asset_check( asset=out_vcerare__hourly_available_capacity_factor, - blocking=False, # Can only fail for fast ETL!!! + blocking=True, description="Check that row count matches expected.", ) def check_rows() -> AssetCheckResult: