Merge branch 'main' into soren-littlepay_uniqueness_adjustments

cal-itp · Oct 12, 2023 · 701dfd3 · 701dfd3
2 parents 4156439 + 232cdc0
commit 701dfd3
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 8 deletions.
diff --git a/...create_external_tables/airtable/external_airtable_california_transit_funding_programs.yml b/...create_external_tables/airtable/external_airtable_california_transit_funding_programs.yml
@@ -32,16 +32,13 @@ schema_fields:
     mode: NULLABLE
   - name: services
     type: STRING
-    mode: NULLABLE
+    mode: REPEATED
   - name: organization
     type: STRING
-    mode: NULLABLE
+    mode: REPEATED
   - name: category
     type: STRING
     mode: NULLABLE
   - name: drmt_data
     type: STRING
-    mode: NULLABLE
-  - name: dt
-    type: STRING
-    mode: NULLABLE
+    mode: REPEATED
diff --git a/warehouse/models/mart/payments/_payments.yml b/warehouse/models/mart/payments/_payments.yml
@@ -69,6 +69,13 @@ models:
         description: The route_long_name of the first tap transaction
       - name: route_short_name
         description: The route_short_name of the first tap transaction
+      - name: agency_id
+        description: |
+          Agency for the specified route.
+
+          This field is required when the dataset provides data for routes from more than one agency in agency.txt, otherwise it is optional.
+      - name: agency_name
+        description: Full name of the transit agency.
       - name: direction
         description: The direction of the first tap transaction
       - name: vehicle_id

diff --git a/warehouse/models/mart/payments/fct_payments_rides_v2.sql b/warehouse/models/mart/payments/fct_payments_rides_v2.sql
@@ -76,6 +76,10 @@ dim_routes AS (
     SELECT * FROM {{ ref('dim_routes') }}
 ),
 
+dim_agency AS (
+    SELECT * FROM {{ ref('dim_agency') }}
+),
+
 dim_gtfs_datasets AS (
     SELECT * FROM {{ ref('dim_gtfs_datasets') }}
 ),
@@ -124,20 +128,25 @@ stg_littlepay__product_data AS (
     FROM {{ ref('stg_littlepay__product_data') }}
 ),
 
-participants_to_routes AS (
+participants_to_routes_and_agency AS (
     SELECT
         pf.participant_id,
         f.date,
         r.route_id,
         r.route_short_name,
         r.route_long_name,
+        a.agency_id,
+        a.agency_name,
     FROM payments_gtfs_datasets AS pf
     LEFT JOIN dim_gtfs_datasets d
         ON pf.gtfs_dataset_source_record_id = d.source_record_id
     LEFT JOIN fct_daily_schedule_feeds AS f
         ON d.key = f.gtfs_dataset_key
     LEFT JOIN dim_routes AS r
         ON f.feed_key = r.feed_key
+    LEFT JOIN dim_agency AS a
+        ON r.agency_id = a.agency_id
+            AND r.feed_key = r.feed_key
 ),
 
 debited_micropayments AS (
@@ -292,6 +301,8 @@ join_table AS (
 --         Common transaction info
         r.route_long_name,
         r.route_short_name,
+        r.agency_id,
+        r.agency_name,
         t1.direction,
         t1.vehicle_id,
         t1.littlepay_transaction_id,
@@ -348,12 +359,13 @@ join_table AS (
     LEFT JOIN stg_littlepay__product_data AS p
         ON m.participant_id = p.participant_id
             AND a.product_id = p.product_id
-    LEFT JOIN participants_to_routes AS r
+    LEFT JOIN participants_to_routes_and_agency AS r
         ON r.participant_id = m.participant_id
             -- here, can just use t1 because transaction date will be populated
             -- (don't have to handle unkowns the way we do with route_id)
             AND EXTRACT(DATE FROM TIMESTAMP(t1.transaction_date_time_utc)) = r.date
             AND r.route_id = COALESCE(t1.route_id, t2.route_id)
+
 ),
 
 fct_payments_rides_v2 AS (