Merge branch 'main' into ah_docs

.github/CODEOWNERS

@@ -4,21 +4,21 @@
 *                                       @hunterowens @evansiroky
 
 # infra components
-/.github/workflows                      @evansiroky @SorenSpicknall
-/ci                                     @evansiroky @SorenSpicknall
-/iac                                    @evansiroky @SorenSpicknall
-/kubernetes                             @evansiroky @SorenSpicknall
+/.github/workflows                      @evansiroky @vevetron
+/ci                                     @evansiroky @vevetron
+/iac                                    @evansiroky @vevetron
+/kubernetes                             @evansiroky @vevetron
 
 # jupyter service, e.g. updating the image
-/kubernetes/apps/charts/jupyterhub      @evansiroky @SorenSpicknall
-/images                                 @evansiroky @SorenSpicknall
+/kubernetes/apps/charts/jupyterhub      @evansiroky @vevetron
+/images                                 @evansiroky @vevetron
 
 # project documentation
 /docs                                   @tiffanychu90
 
 # actual pipeline code
-/airflow                                @evansiroky @SorenSpicknall
-/jobs                                   @evansiroky @SorenSpicknall
-/runbooks                               @evansiroky @SorenSpicknall
-/services                               @evansiroky @SorenSpicknall
-/warehouse                              @evansiroky @SorenSpicknall @tiffanychu90
+/airflow                                @evansiroky @vevetron
+/jobs                                   @evansiroky @vevetron
+/runbooks                               @evansiroky @vevetron
+/services                               @evansiroky @vevetron
+/warehouse                              @evansiroky @vevetron @tiffanychu90

.github/ISSUE_TEMPLATE/offboarding.md

@@ -0,0 +1,40 @@
+---
+name: Offboarding Team Member
+about: Checklist for offboarding process.
+title: Team Member - [Name]
+labels: admin
+---
+
+Name:
+
+Email Address:
+GitHub Username:
+Slack Username:
+
+**Offboard Privileges:**
+
+- [ ] Email
+
+- [ ] Slack
+
+- [ ] Added to tools:
+
+- [ ] Github
+
+  - [ ] Organization: Cal-ITP
+  - [ ] Team: warehouse-users and warehouse-contributors (affects JupyterHub)
+  - [ ] Update or remove code references to designated hand-off staff or [email protected]
+  - [ ] Remove administrative privileges / pull request approvals
+
+- [ ] Google IAM (Cloud and BQ)
+
+- [ ] Metabase
+
+- [ ] Airtable
+
+- [ ] Meeting invites:
+
+  - [ ] Standing meetings
+  - [ ] Lunch n' Learn
+  - [ ] All-hands
+  - [ ] Data & Digital Services email list

.github/workflows/build-calitp-data-analysis.yml

@@ -28,7 +28,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - uses: 'google-github-actions/auth@v2'
         with:
           credentials_json: '${{ secrets.GCP_SA_KEY }}'

.github/workflows/build-calitp-map-utils.yml

@@ -27,7 +27,7 @@ jobs:
       - run: sudo apt-get install -y libgraphviz-dev graphviz-dev
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - run: |
           curl -sSL https://install.python-poetry.org | python -
           poetry install

.github/workflows/deploy-kubernetes.yml

@@ -57,7 +57,7 @@ jobs:
       # Invoke deployment tasks
       - run: poetry install
         working-directory: ci
-      - run: poetry run invoke secrets -f "./channels/prod.yaml"
+      - run: poetry run invoke secrets -f "./kubernetes-workloads.yaml"
         working-directory: ci
-      - run: poetry run invoke release -f "./channels/prod.yaml"
+      - run: poetry run invoke release -f "./kubernetes-workloads.yaml"
         working-directory: ci

.github/workflows/lint.yml

@@ -12,6 +12,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pre-commit-
       - uses: pre-commit/[email protected]
       - uses: crate-ci/[email protected] # Set back to `master` after #967 on the typos repo is fixed
         with:

.github/workflows/preview-kubernetes.yml

@@ -57,7 +57,7 @@ jobs:
       # Diff and write back to PR
       - run: poetry install
         working-directory: ci
-      - run: poetry run invoke diff -f "./channels/prod.yaml" --outfile=diff.md
+      - run: poetry run invoke diff -f "./kubernetes-workloads.yaml" --outfile=diff.md
         working-directory: ci
       - uses: peter-evans/find-comment@v2
         id: fc

.pre-commit-config.yaml

@@ -35,10 +35,10 @@ repos:
         args: ["-ll", "--skip=B108,B608,B310,B303,B324,B113"]
         files: .py$
   - repo: https://github.com/sqlfluff/sqlfluff
-    rev: 2.1.1
+    rev: 3.0.6
     hooks:
       - id: sqlfluff-lint
-        additional_dependencies: ['dbt-bigquery', 'sqlfluff-templater-dbt']
+        additional_dependencies: ['dbt-bigquery==1.8.1', 'sqlfluff-templater-dbt']
         # skip: L003 indentation stuff -- TODO
         # skip: L010 uppercase keywords -- TODO
         # skip: L011 implicit/explicit aliasing of tables? -- TODO
@@ -55,8 +55,8 @@ repos:
         # skip: joins should not include subqueries -- TODO
         # skip: use left join instead of right join -- TODO
         # skip: use single quotes instead of double -- TODO
-        args: [--dialect, "bigquery", --ignore, "parsing,templating",--exclude-rules, "L003,L010,L011,L022,L036,L038,L039,L059,L016,L029,L027,L032,L034,L014,L042,L055,L064"]
-        files: "warehouse/models" # TODO: should also lint tests etc. but we want to skip packages at least
+        # args: [-vvv, --dialect, "bigquery", --ignore, "parsing,templating",--exclude-rules, "L003,L010,L011,L022,L036,L038,L039,L059,L016,L029,L027,L032,L034,L014,L042,L055,L064"]
+        # files: "warehouse/models" # TODO: should also lint tests etc. but we want to skip packages at least
   - repo: https://github.com/kynan/nbstripout
     rev: 0.6.1
     hooks:

.sqlfluff

@@ -0,0 +1,10 @@
+[sqlfluff]
+dialect = bigquery
+exclude_rules = AL09,L003,L010,L011,L022,L036,L038,L039,L059,L016,L029,L027,L032,L034,L014,L042,L055,L064,AM04,ST09
+ignore = templating, parsing
+
+# Optional: If you have a specific dbt profile you want to use:
+profiles_dir = ./warehouse/
+
+[sqlfluff:templater:dbt]
+project_dir = ./warehouse/models/

.sqlfluffignore

@@ -0,0 +1 @@
+warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql

airflow/dags/create_external_tables/ntd_data_products/annual_database_service.yml

@@ -0,0 +1,16 @@
+operator: operators.ExternalTable
+bucket: gs://calitp-ntd-data-products
+prefix_bucket: true
+post_hook: |
+  SELECT *
+  FROM `{{ get_project_id() }}`.external_ntd_data_products.annual_database_service
+  LIMIT 1;
+source_objects:
+  - "annual-database-service/*.jsonl.gz"
+destination_project_dataset_table: "external_ntd_data_products.annual_database_service"
+source_format: NEWLINE_DELIMITED_JSON
+use_bq_client: true
+hive_options:
+  mode: CUSTOM
+  require_partition_filter: false
+  source_uri_prefix: "annual-database-service/{dt:DATE}/{ts:TIMESTAMP}/{year:INTEGER}/"

apps/maps/README.md

@@ -15,8 +15,7 @@ data validation as well as Typescript types for type-hinting in the Svelte app.
 a few CLI commands to facilitate validating pre-existing data with those types and/or generate a quick URL
 with state for testing.
 
-TODO: a GitHub Action workflow exists to build the package, but it does not currently publish to pypi; right now
-that is done manually with `poetry publish`.
+A GitHub Action workflow exists to build the package and publish it to PyPi. Be sure to bump the version date in `pyproject.toml` in any pull request that updates the library.
 
 ## Developing the maps app