Consolidate optional dependencies; test nuke; parallelize tests

* Move previous dev, test, and datasette optional dependencies into the required dependencies to simplify application installation. * Test make nuke; parallelize --live-dbs tests * Move prettier into conda-only dependencies * Update conda-lock.yml and rendered conda environment files. * Remove action test file hashlog
catalyst-cooperative · Nov 6, 2023 · aaa8082 · aaa8082
1 parent c7a9c9f
commit aaa8082
Show file tree

Hide file tree

Showing 12 changed files with 1,944 additions and 1,103 deletions.
diff --git a/.github/workflows/tox-pytest.yml b/.github/workflows/tox-pytest.yml
@@ -34,7 +34,6 @@ jobs:
           environment-file: environments/conda-lock.yml
           environment-name: pudl-dev
           cache-environment: true
-          create-args: --category main dev docs test datasette
 
       - name: Log environment details
         run: |
@@ -74,7 +73,6 @@ jobs:
           environment-file: environments/conda-lock.yml
           environment-name: pudl-dev
           cache-environment: true
-          create-args: --category main dev docs test datasette
 
       - name: Log environment details
         run: |
@@ -125,7 +123,6 @@ jobs:
           environment-file: environments/conda-lock.yml
           environment-name: pudl-dev
           cache-environment: true
-          create-args: --category main dev docs test datasette
 
       - name: Log environment details
         run: |

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -10,7 +10,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: mambaforge-4.10
+    python: mambaforge-22.9
 
 # Define the python environment using conda / mamba
 conda:

diff --git a/Makefile b/Makefile
@@ -5,7 +5,6 @@ coverage_report := coverage report --sort=cover
 pytest_args := --durations 20 ${pytest_covargs} ${gcs_cache_path}
 etl_fast_yml := src/pudl/package_data/settings/etl_fast.yml
 etl_full_yml := src/pudl/package_data/settings/etl_full.yml
-pip_install_pudl := pip install --no-deps --editable ./
 
 # We use mamba locally, but micromamba in CI, so choose the right binary:
 ifdef GITHUB_ACTION
@@ -47,21 +46,13 @@ conda-lock.yml: pyproject.toml
 	cd environments && conda-lock render \
 		--kind env \
 		--dev-dependencies \
-		--extras docs \
-		--extras datasette \
 		conda-lock.yml
 	prettier --write environments/*.yml
 
 # Create the pudl-dev conda environment based on the universal lockfile
 .PHONY: pudl-dev
 pudl-dev: conda-lock.yml
-	conda-lock install \
-		--name pudl-dev \
-		--${mamba} \
-		--dev \
-		--extras docs \
-		--extras datasette \
-		environments/conda-lock.yml
+	conda-lock install --name pudl-dev --${mamba} --dev environments/conda-lock.yml
 
 .PHONY: install-pudl
 install-pudl: pudl-dev
@@ -92,19 +83,18 @@ docs-build: docs-clean
 ########################################################################################
 
 # Extract all FERC DBF and XBRL data to SQLite.
-ferc1.sqlite ferc1_xbrl.sqlite:
+.PHONY: ferc
+ferc:
 	coverage run ${covargs} -- \
 		src/pudl/ferc_to_sqlite/cli.py \
 		--clobber \
 		${gcs_cache_path} \
 		${etl_full_yml}
 
 # Run the full PUDL ETL
-pudl.sqlite:
-	coverage run ${covargs} -- \
-		src/pudl/cli/etl.py \
-		${gcs_cache_path} \
-		${etl_full_yml}
+.PHONY: pudl
+pudl:
+	coverage run ${covargs} -- src/pudl/cli/etl.py ${gcs_cache_path} ${etl_full_yml}
 
 ########################################################################################
 # pytest
@@ -140,11 +130,10 @@ pytest-validate:
 # Backgrounding the data validation and integration tests and using wait allows them to
 # run in parallel.
 .PHONY: nuke
-nuke: coverage-erase docs-build pytest-unit ferc1.sqlite ferc1_xbrl.sqlite pudl.sqlite
+nuke: coverage-erase docs-build pytest-unit ferc pudl
 	pudl_check_fks
-	pytest ${pytest_args} --live-dbs --etl-settings ${etl_full_yml} test/integration & \
-	pytest ${pytest_args} --live-dbs test/validate & \
-	wait
+	pytest ${pytest_args} -n auto --live-dbs --etl-settings ${etl_full_yml} test/integration
+	pytest ${pytest_args} -n auto --live-dbs test/validate
 	${coverage_report}
 
 # Check that designated Jupyter notebooks can be run against the current DB

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -40,7 +40,7 @@ COPY docker/dagster.yaml ${DAGSTER_HOME}/dagster.yaml
 
 # Create a conda environment based on the specification in the repo
 COPY environments/conda-lock.yml environments/conda-lock.yml
-RUN micromamba create --prefix ${CONDA_PREFIX} --yes --category main dev docs test datasette --file environments/conda-lock.yml && \
+RUN micromamba create --prefix ${CONDA_PREFIX} --yes --file environments/conda-lock.yml && \
     micromamba clean -afy
 # Copy the cloned pudl repository into the user's home directory
 COPY --chown=${MAMBA_USER}:${MAMBA_USER} . ${CONTAINER_HOME}

diff --git a/docker/gcp_pudl_etl.sh b/docker/gcp_pudl_etl.sh
@@ -24,21 +24,23 @@ function run_pudl_etl() {
     alembic upgrade head && \
     pudl_setup && \
     ferc_to_sqlite \
-        --loglevel=DEBUG \
-        --gcs-cache-path=gs://internal-zenodo-cache.catalyst.coop \
-        --workers=8 \
+        --loglevel DEBUG \
+        --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
+        --workers 8 \
         $PUDL_SETTINGS_YML && \
     pudl_etl \
         --loglevel DEBUG \
         --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
         $PUDL_SETTINGS_YML && \
     pytest \
-        --gcs-cache-path=gs://internal-zenodo-cache.catalyst.coop \
-        --etl-settings=$PUDL_SETTINGS_YML \
+        -n auto \
+        --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
+        --etl-settings $PUDL_SETTINGS_YML \
         --live-dbs test/integration test/unit && \
     pytest \
-        --gcs-cache-path=gs://internal-zenodo-cache.catalyst.coop \
-        --etl-settings=$PUDL_SETTINGS_YML \
+        -n auto \
+        --gcs-cache-path gs://internal-zenodo-cache.catalyst.coop \
+        --etl-settings $PUDL_SETTINGS_YML \
         --live-dbs test/validate
 }
 

diff --git a/environments/conda-linux-64.lock.yml b/environments/conda-linux-64.lock.yml
@@ -1,6 +1,6 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: 874b20e543d8ad29e094c126a85321cefaf15e80f7f713bc52572ed16a1facf3
+# input_hash: f851b393014a2b6c2baa7f0ffaf38ba4c78b8b3dc631f2427d6e2c241f3a305b
 
 channels:
   - conda-forge
@@ -25,13 +25,15 @@ dependencies:
   - fonts-conda-ecosystem=1=0
   - libgcc-ng=13.2.0=h807b86a_2
   - aws-c-common=0.9.3=hd590300_0
-  - bzip2=1.0.8=h7f98852_4
+  - bzip2=1.0.8=hd590300_5
   - c-ares=1.21.0=hd590300_0
+  - fribidi=1.0.10=h36c2ea0_0
   - geos=3.12.0=h59595ed_0
   - gettext=0.21.1=h27087fc_0
   - gflags=2.2.2=he1b5a44_1004
   - giflib=5.2.1=h0b41bf4_3
   - gmp=6.2.1=h58526e2_0
+  - graphite2=1.3.13=h58526e2_1001
   - icu=73.2=h59595ed_0
   - json-c=0.17=h7ab15ed_0
   - keyutils=1.6.1=h166bdaf_0
@@ -51,6 +53,7 @@ dependencies:
   - libnuma=2.0.16=h0b41bf4_1
   - libsodium=1.0.18=h36c2ea0_1
   - libspatialindex=1.9.3=h9c3ff4c_4
+  - libtool=2.4.7=h27087fc_0
   - libutf8proc=2.8.0=h166bdaf_0
   - libuuid=2.38.1=h0b41bf4_0
   - libuv=1.46.0=hd590300_0
@@ -115,7 +118,7 @@ dependencies:
   - freetype=2.12.1=h267a509_2
   - krb5=1.21.2=h659d440_0
   - libarchive=3.7.2=h039dbb9_0
-  - libglib=2.78.0=hebfc3b9_0
+  - libglib=2.78.1=hebfc3b9_0
   - libgrpc=1.57.0=ha4d0f93_2
   - libopenblas=0.3.24=pthreads_h413a1c8_0
   - libthrift=0.19.0=hb90f79a_1
@@ -134,6 +137,7 @@ dependencies:
   - anyascii=0.3.2=pyhd8ed1ab_0
   - appdirs=1.4.4=pyh9f0ad1d_0
   - astroid=3.0.1=py311h38be061_0
+  - atk-1.0=2.38.0=hd4edc92_1
   - attrs=23.1.0=pyh71513ae_1
   - aws-c-event-stream=0.3.2=h6fea174_2
   - aws-c-http=0.7.13=hb59894b_2
@@ -169,15 +173,18 @@ dependencies:
   - entrypoints=0.4=pyhd8ed1ab_0
   - et_xmlfile=1.1.0=pyhd8ed1ab_0
   - exceptiongroup=1.1.3=pyhd8ed1ab_0
+  - execnet=2.0.2=pyhd8ed1ab_0
   - executing=2.0.1=pyhd8ed1ab_0
   - filelock=3.13.1=pyhd8ed1ab_0
   - fontconfig=2.14.2=h14ed4e7_0
   - freexl=2.0.0=h743c826_0
   - frozenlist=1.4.0=py311h459d7ec_1
   - fsspec=2023.10.0=pyhca7485f_0
+  - gdk-pixbuf=2.42.10=h829c605_4
   - google-cloud-sdk=453.0.0=py311h38be061_0
   - greenlet=3.0.1=py311hb755f60_0
   - grpcio=1.57.0=py311ha6695c7_2
+  - gts=0.7.6=h977cf35_4
   - hpack=4.0.0=pyh9f0ad1d_0
   - httptools=0.6.1=py311h459d7ec_0
   - humanfriendly=10.0=pyhd8ed1ab_6
@@ -199,6 +206,7 @@ dependencies:
   - libblas=3.9.0=19_linux64_openblas
   - libcurl=8.4.0=hca28451_0
   - libpq=16.0=hfc447b1_1
+  - libwebp=1.3.2=h658648e_1
   - llvmlite=0.40.1=py311ha6695c7_0
   - locket=1.0.0=pyhd8ed1ab_0
   - lxml=4.9.3=py311h1a07684_1
@@ -344,6 +352,7 @@ dependencies:
   - jupyterlab_pygments=0.2.2=pyhd8ed1ab_0
   - latexcodec=2.0.1=pyh9f0ad1d_0
   - libcblas=3.9.0=19_linux64_openblas
+  - libgd=2.3.3=h119a65a_9
   - libgoogle-cloud=2.12.0=h8d7e28b_2
   - liblapack=3.9.0=19_linux64_openblas
   - linear-tsv=1.1.0=py_1
@@ -402,6 +411,7 @@ dependencies:
   - gql=3.4.1=pyhd8ed1ab_0
   - graphql-relay=3.2.0=pyhd8ed1ab_0
   - grpcio-health-checking=1.57.0=pyhd8ed1ab_0
+  - harfbuzz=8.2.1=h3d44ed6_0
   - httpcore=1.0.1=pyhd8ed1ab_0
   - importlib_metadata=6.8.0=hd8ed1ab_0
   - jsonschema-specifications=2023.7.1=pyhd8ed1ab_0
@@ -418,10 +428,11 @@ dependencies:
   - psycopg2-binary=2.9.7=pyhd8ed1ab_1
   - pybtex=0.24.0=pyhd8ed1ab_2
   - pydantic=1.10.13=py311h459d7ec_1
-  - pyproj=3.6.1=py311h1facc83_3
+  - pyproj=3.6.1=py311h1facc83_4
   - pytest-console-scripts=1.4.1=pyhd8ed1ab_0
   - pytest-cov=4.1.0=pyhd8ed1ab_0
   - pytest-mock=3.12.0=pyhd8ed1ab_0
+  - pytest-xdist=3.3.1=pyhd8ed1ab_0
   - python-build=1.0.3=pyhd8ed1ab_0
   - requests=2.31.0=pyhd8ed1ab_0
   - rich=13.6.0=pyhd8ed1ab_0
@@ -459,6 +470,7 @@ dependencies:
   - numexpr=2.8.7=py311h039bad6_104
   - oauthlib=3.2.2=pyhd8ed1ab_0
   - pandas=2.1.2=py311h320fe9a_0
+  - pango=1.50.14=ha41ecd1_2
   - prompt-toolkit=3.0.39=pyha770c72_0
   - pybtex-docutils=1.0.3=py311h38be061_1
   - pyopenssl=23.3.0=pyhd8ed1ab_0
@@ -486,9 +498,11 @@ dependencies:
   - geopandas-base=0.14.0=pyha770c72_1
   - google-auth=2.23.4=pyhca7485f_0
   - gql-with-requests=3.4.1=pyhd8ed1ab_0
+  - gtk2=2.24.33=h90689f9_2
   - jsonschema-with-format-nongpl=4.19.2=pyhd8ed1ab_0
   - jupyter_client=8.5.0=pyhd8ed1ab_0
   - keyring=24.2.0=py311h38be061_1
+  - librsvg=2.56.3=h98fae49_0
   - matplotlib-base=3.8.1=py311h54ef318_0
   - nbformat=5.9.2=pyhd8ed1ab_0
   - pandera-core=0.17.2=pyhd8ed1ab_0
@@ -504,6 +518,7 @@ dependencies:
   - fiona=1.9.5=py311hbac4ec9_0
   - google-api-core=2.12.0=pyhd8ed1ab_0
   - google-auth-oauthlib=1.1.0=pyhd8ed1ab_0
+  - graphviz=8.1.0=h28d9a01_0
   - ipython=8.17.2=pyh41d4057_0
   - jupyter_events=0.8.0=pyhd8ed1ab_0
   - libarrow=13.0.0=h0f80be4_7_cpu
@@ -518,6 +533,7 @@ dependencies:
   - ipywidgets=8.1.1=pyhd8ed1ab_0
   - nbconvert-core=7.10.0=pyhd8ed1ab_0
   - pyarrow=13.0.0=py311h39c9aba_7_cpu
+  - pygraphviz=1.11=py311h72a77b7_1
   - tableschema=1.19.3=pyh9f0ad1d_0
   - datapackage=1.15.2=pyh44b312d_0
   - google-cloud-storage=2.13.0=pyhca7485f_0