Merge pull request #2861 from cal-itp/publish-pypi

publish python packages to pypi in ci

.github/workflows/build-calitp-data-analysis.yml

@@ -18,8 +18,10 @@ concurrency:
 
 jobs:
   check_and_build:
-    name: check python
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: packages/calitp-data-analysis
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -30,11 +32,20 @@ jobs:
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           export_default_credentials: true
       - name: Run checks
-        working-directory: packages/calitp-data-analysis
         run: |
           curl -sSL https://install.python-poetry.org | python -
           poetry install
           poetry run mypy .
           poetry run pytest
           poetry build
-# TODO: add publishing
+      - if: ${{ github.ref != 'refs/heads/main' }}
+        run: |
+          poetry config repositories.test-pypi https://test.pypi.org/legacy/
+          poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI
+          poetry publish -r test-pypi --skip-existing
+        env:
+          POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_ANALYSIS_TOKEN }}
+      - if: ${{ github.ref == 'refs/heads/main' }}
+        run: poetry publish --skip-existing
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_ANALYSIS_TOKEN }}

.github/workflows/build-calitp-data-infra.yml

@@ -18,15 +18,20 @@ concurrency:
 
 jobs:
   check_and_build:
-    name: check python
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: packages/calitp-data-infra
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
+      - uses: google-github-actions/setup-gcloud@v0
+        with:
+          export_default_credentials: true
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
       - name: Run checks
-        working-directory: packages/calitp-data-infra
         run: |
           curl -sSL https://install.python-poetry.org | python -
           poetry install
@@ -38,4 +43,14 @@ jobs:
           CALITP_BUCKET__GTFS_DOWNLOAD_CONFIG: gs://this-does-not-exist
           CALITP_BUCKET__GTFS_SCHEDULE_RAW: gs://this-does-not-exist
           CALITP_BUCKET__GTFS_RT_RAW: gs://this-does-not-exist
-# TODO: add publishing
+      - if: ${{ github.ref != 'refs/heads/main' }}
+        run: |
+          poetry config repositories.test-pypi https://test.pypi.org/legacy/
+          poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI
+          poetry publish -r test-pypi --skip-existing
+        env:
+          POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_INFRA_TOKEN }}
+      - if: ${{ github.ref == 'refs/heads/main' }}
+        run: poetry publish --skip-existing
+        env:
+          POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }}

.github/workflows/build-calitp-map-utils.yml

@@ -18,18 +18,32 @@ concurrency:
 
 jobs:
   check_and_build:
-    name: check python
     runs-on: ubuntu-latest
-    working-directory: apps/maps
+    defaults:
+      run:
+        working-directory: apps/maps
     steps:
-      - run: sudo apt-get install -y libgraphviz-dev graphviz-dev
       - uses: actions/checkout@v3
+      - run: sudo apt-get install -y libgraphviz-dev graphviz-dev
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
-      - run: curl -sSL https://install.python-poetry.org | python -
-      - run: poetry install
-      - run: poetry run mypy calitp_map_utils
-      - run: poetry run pytest --spec
-      - run: poetry run build
-      # TODO: should we actually publish to pypi?
+      - run: |
+          curl -sSL https://install.python-poetry.org | python -
+          poetry install
+          poetry run mypy calitp_map_utils
+          poetry run pytest --spec
+          poetry build
+        env:
+          CALITP_MAP_APP_URL: https://embeddable-maps.calitp.org/
+      - if: ${{ github.ref != 'refs/heads/main' }}
+        run: |
+          poetry config repositories.test-pypi https://test.pypi.org/legacy/
+          poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI
+          poetry publish -r test-pypi --skip-existing
+        env:
+          POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_MAP_UTILS_TOKEN }}
+      - if: ${{ github.ref == 'refs/heads/main' }}
+        run: poetry publish --skip-existing
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_MAP_UTILS_TOKEN }}

.github/workflows/build-dbt.yml

@@ -44,8 +44,8 @@ jobs:
           export_default_credentials: true
           service_account_key: ${{ secrets.GCP_SA_KEY }}
       - name: Compile dbt project
+        working-directory: warehouse
         run: |
-          cd warehouse
           poetry install
           poetry run dbt deps
           poetry run dbt compile --target prod --full-refresh

apps/maps/calitp_map_utils/__init__.py

@@ -3,12 +3,12 @@
 import json
 import os
 from enum import Enum
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
 
-import gcsfs
+import gcsfs  # type: ignore[import]
 import requests
 import typer
-from furl import furl
+from furl import furl  # type: ignore[import]
 from geojson_pydantic import Feature, FeatureCollection, MultiPolygon, Point, Polygon
 from geojson_pydantic.geometries import Geometry
 from geojson_pydantic.types import Position
@@ -32,16 +32,23 @@ class Tooltip(BaseModel):
     style: Optional[Dict[str, Any]]
 
 
+# https://github.com/pydantic/pydantic/discussions/5528#discussioncomment-5663312
+if TYPE_CHECKING:
+    Color = Optional[list[int]]
+else:
+    # we add alpha in the JS if only 3 colors are passed
+    Color = Optional[conlist(int, min_items=3, max_items=4)]
+
+
 class Speedmap(BaseModel):
     stop_id: Optional[str]
     stop_name: Optional[str]
     route_id: Optional[str]
     tooltip: Optional[Tooltip]
     avg_mph: Optional[float]
     p20_mph: Optional[float] = Field(alias="_20p_mph")
-    # we add alpha in the JS if only 3 colors are passed
-    color: Optional[conlist(int, min_items=3, max_items=4)]
-    highlight_color: Optional[conlist(int, min_items=3, max_items=4)]
+    color: Color
+    highlight_color: Color
 
     @root_validator
     def some_identifier_exists(cls, values):
@@ -89,7 +96,10 @@ def validate_geojson(
 
     collection = FeatureCollection(**d)
 
-    layer_type_class = LAYER_FEATURE_TYPES.get(layer_type, Feature[Geometry, Dict])
+    if layer_type:
+        layer_type_class = LAYER_FEATURE_TYPES.get(layer_type, Feature[Geometry, Dict])
+    else:
+        layer_type_class = Feature[Geometry, Dict]
 
     if verbose:
         typer.secho(
@@ -117,13 +127,18 @@ class BasemapConfig(BaseModel):
     options: Dict[str, Any]
 
 
+if TYPE_CHECKING:
+    LayerList = list[Layer]
+else:
+    # this will not render in erdantic; needs to be List[Layer] but then pydantic2ts will not set min_items
+    LayerList = conlist(Layer, min_items=1)
+
+
 # Any positions in this are flipped from typical geojson
 # leaflet wants lat/lon
 class State(BaseModel):
     name: Optional[str]
-    layers: conlist(
-        Layer, min_items=1
-    )  # this will not render in erdantic; needs to be List[Layer] but then pydantic2ts will not set min_items
+    layers: LayerList
     lat_lon: Optional[Position]
     zoom: Optional[int]
     bbox: Optional[Tuple[Position, Position]]
@@ -151,7 +166,7 @@ def validate_layers(
             if data:
                 validate_geojson(layer.url, layer.typ, verbose=verbose)
 
-    def iframe_url(self, host: str = None) -> str:
+    def iframe_url(self, host: Optional[str] = None) -> str:
         host = host or MAP_APP_URL
         if not host:
             raise RuntimeError(

apps/maps/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "calitp_map_utils"
-version = "2023.8.1"
+version = "2023.8.2"
 description = ""
 authors = ["Andrew Vaccaro <[email protected]>"]
 readme = "README.md"

apps/maps/tests/test_calitp_map_utils.py

@@ -46,7 +46,7 @@
             {
                 "name": "LA Metro Bus Speed Maps AM Peak",
                 "url": "https://storage.googleapis.com/calitp-map-tiles/metro_am.geojson.gz",
-                "type": "speedmaps",
+                "type": "speedmap",
             },
         ],
         "bbox": [[34.1, -118.5], [33.9, -118]],
@@ -86,9 +86,12 @@ def test_iframe_url_works_with_env_var():
     with pytest.MonkeyPatch.context() as mp:
         mp.setattr("calitp_map_utils.MAP_APP_URL", "https://some.domain")
 
-        assert "some.domain" in State(**TEST_STATES[0]).iframe_url
+        assert "some.domain" in State(**TEST_STATES[0]).iframe_url()
 
 
 def test_iframe_url_missing_env_throws_exception(monkeypatch):
-    with pytest.raises(RuntimeError):
-        _ = State(**TEST_STATES[0]).iframe_url
+    with pytest.MonkeyPatch.context() as mp:
+        mp.setattr("calitp_map_utils.MAP_APP_URL", None)
+
+        with pytest.raises(RuntimeError):
+            _ = State(**TEST_STATES[0]).iframe_url()