Add new Littlepay columns + correct mistyped columns (#2996)

* Add missing Littlepay columns to external tables

* Correct column names and add new columns to models

airflow/dags/create_external_tables/payments/littlepay/customer_funding_source.yml

@@ -30,5 +30,7 @@ schema_fields:
     type: STRING
   - name: principal_customer_id
     type: STRING
+  - name: participant_id
+    type: STRING
   - name: _line_number
     type: STRING

airflow/dags/create_external_tables/payments/littlepay/device_transactions.yml

@@ -26,7 +26,7 @@ schema_fields:
      type: STRING
    - name: transaction_outcome
      type: STRING
-   - name: transction_deny_reason
+   - name: transaction_deny_reason
      type: STRING
    - name: transaction_date_time_utc
      type: STRING
@@ -54,5 +54,7 @@ schema_fields:
      type: STRING
    - name: onward_zone_ids
      type: STRING
+   - name: processed_timestamp_utc
+     type: STRING
    - name: _line_number
      type: STRING

airflow/dags/create_external_tables/payments/littlepay/micropayment_adjustments.yml

@@ -30,7 +30,9 @@ schema_fields:
     type: STRING  # Peak, Off Peak
   - name: applied
     type: STRING
-  - name: zone_ids_us
+  - name: zone_ids_used
+    type: STRING
+  - name: incentive_product_id
     type: STRING
   - name: _line_number
     type: STRING

airflow/dags/create_external_tables/payments/littlepay/product_data.yml

@@ -14,6 +14,10 @@ schema_fields:
     type: STRING
   - name: product_id
     type: STRING
+  - name: previous_version_id
+    type: STRING
+  - name: original_version_id
+    type: STRING
   - name: product_code
     type: STRING
   - name: product_description
@@ -24,6 +28,8 @@ schema_fields:
     type: STRING  # (Customer, Card)
   - name: product_status
     type: STRING  # (Active, Inactive)
+  - name: superseded
+    type: STRING
   - name: created_date
     type: STRING
   - name: capping_type
@@ -48,6 +54,8 @@ schema_fields:
     type: STRING
   - name: weekly_cap_start_day
     type: STRING  # (Monday, Sunday)
+  - name: weekly_cap_end_day
+    type: STRING
   - name: number_of_days_in_cap_window
     type: STRING
   - name: capping_duration
@@ -56,9 +64,31 @@ schema_fields:
     type: STRING
   - name: capping_time_zone
     type: STRING
-  - name: capping_overlap
+  - name: capping_overlap_time
     type: STRING
   - name: capping_application_level
     type: STRING  # (Customer, Card)
+  - name: route_capping_enabled
+    type: STRING
+  - name: routes
+    type: STRING
+  - name: zoned_capping_enabled
+    type: BOOLEAN
+  - name: zoned_capping_mode
+    type: STRING
+  - name: zoned_capping_pricing_type
+    type: STRING
+  - name: on_peak_zones
+    type: STRING
+  - name: off_peak_zones
+    type: STRING
+  - name: incentive_enabled
+    type: BOOLEAN
+  - name: incentive_type
+    type: STRING
+  - name: discount_qualifier
+    type: STRING
+  - name: configuration
+    type: STRING
   - name: _line_number
     type: STRING

warehouse/models/mart/payments/fct_payments_rides_v2.sql

@@ -199,7 +199,7 @@ initial_transactions AS (
         dt.device_id_issuer,
         dt.type,
         dt.transaction_outcome,
-        dt.transction_deny_reason,
+        dt.transaction_deny_reason,
         dt.transaction_date_time_utc,
         dt.location_scheme,
         dt.location_name,
@@ -238,7 +238,7 @@ second_transactions AS (
         dt.device_id_issuer,
         dt.type,
         dt.transaction_outcome,
-        dt.transction_deny_reason,
+        dt.transaction_deny_reason,
         dt.transaction_date_time_utc,
         dt.location_scheme,
         dt.location_name,

warehouse/models/staging/payments/littlepay/_littlepay.yml

@@ -321,14 +321,23 @@ models:
 
           Possible values are `bus` and `train`.
       - name: onward_zone_ids
-        description: ""
+        description: |
+          Zones passed for incomplete variable trips.
+
+          Can be a comma-separated string.
+      - name: processed_timestamp_utc
+        description: |
+          Timestamp of when the device data was processed by Littlepay.
+
+          The difference between this field and transaction_date_time_utc can be sizeable when data
+          is sent by the customer via the Back Office API
       - name: participant_id
         description: The unique identifier of the participant that the device belongs to.
       - name: route_id
         description: The route identifier provided by the device.
       - name: transaction_date_time_utc
         description: The date and time that the customer tapped on the device.
-      - name: transction_deny_reason
+      - name: transaction_deny_reason
         description: |
           The reason a transaction was denied.
 
@@ -379,7 +388,7 @@ models:
             - amount
             - time_period_type
             - applied
-            - zone_ids_us
+            - zone_ids_used
     columns:
       - name: adjustment_id
         description: '{{ doc("micropayment_adjustments_adjustment_id") }}'
@@ -593,7 +602,7 @@ models:
         description: The number of free transfers after the initial tap.
       - name: capping_time_zone
         description: The time zone where the capping product was used.
-      - name: capping_overlap
+      - name: capping_overlap_time
         description: The time a customer can overlap within a 24-hour period.
       - name: capping_application_level
         description: This is the customer level tapping as customers can have multiple cards (physical / digital).

warehouse/models/staging/payments/littlepay/stg_littlepay__customer_funding_source.sql

@@ -14,6 +14,7 @@ clean_columns AS (
         {{ trim_make_empty_string_null('issuer_country') }} AS issuer_country,
         {{ trim_make_empty_string_null('form_factor') }} AS form_factor,
         {{ trim_make_empty_string_null('principal_customer_id') }} AS principal_customer_id,
+        {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
         CAST(_line_number AS INTEGER) AS _line_number,
         `instance`,
         extract_filename,
@@ -24,7 +25,7 @@ clean_columns AS (
         -- hashing at this step will preserve distinction between nulls and empty strings in case that is meaningful upstream
         {{ dbt_utils.generate_surrogate_key(['funding_source_id', 'funding_source_vault_id',
             'customer_id', 'bin', 'masked_pan', 'card_scheme', 'issuer', 'issuer_country',
-            'form_factor']) }} AS _content_hash,
+            'form_factor', 'principal_customer_id', 'participant_id']) }} AS _content_hash,
     FROM source
 ),
 
@@ -60,6 +61,7 @@ stg_littlepay__customer_funding_source AS (
         issuer_country,
         form_factor,
         principal_customer_id,
+        participant_id,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__device_transactions.sql

@@ -12,7 +12,7 @@ clean_columns AS (
         {{ trim_make_empty_string_null('device_id_issuer') }} AS device_id_issuer,
         {{ trim_make_empty_string_null('type') }} AS type,
         {{ trim_make_empty_string_null('transaction_outcome') }} AS transaction_outcome,
-        {{ trim_make_empty_string_null('transction_deny_reason') }} AS transction_deny_reason,
+        {{ trim_make_empty_string_null('transaction_deny_reason') }} AS transaction_deny_reason,
         {{ trim_make_empty_string_null('transaction_date_time_utc') }} AS transaction_date_time_utc,
         DATETIME(
             TIMESTAMP(transaction_date_time_utc), "America/Los_Angeles"
@@ -32,6 +32,7 @@ clean_columns AS (
         {{ trim_make_empty_string_null('vehicle_id') }} AS vehicle_id,
         {{ trim_make_empty_string_null('granted_zone_ids') }} AS granted_zone_ids,
         {{ trim_make_empty_string_null('onward_zone_ids') }} AS onward_zone_ids,
+        {{ trim_make_empty_string_null('processed_timestamp_utc') }} AS processed_timestamp_utc,
         CAST(_line_number AS INTEGER) AS _line_number,
         `instance`,
         extract_filename,
@@ -42,10 +43,10 @@ clean_columns AS (
         -- hashing at this step will preserve distinction between nulls and empty strings in case that is meaningful upstream
         {{ dbt_utils.generate_surrogate_key(['participant_id', 'customer_id',
             'device_transaction_id', 'littlepay_transaction_id', 'device_id', 'device_id_issuer',
-            'type', 'transaction_outcome', 'transction_deny_reason', 'transaction_date_time_utc',
+            'type', 'transaction_outcome', 'transaction_deny_reason', 'transaction_date_time_utc',
             'location_id', 'location_scheme', 'location_name', 'zone_id', 'route_id', 'mode',
             'direction', 'latitude', 'longitude', 'vehicle_id', 'granted_zone_ids',
-            'onward_zone_ids']) }} AS _content_hash,
+            'onward_zone_ids', 'processed_timestamp_utc']) }} AS _content_hash,
     FROM source
 ),
 
@@ -68,7 +69,7 @@ stg_littlepay__device_transactions AS (
             device_id_issuer,
             type,
             transaction_outcome,
-            transction_deny_reason,
+            transaction_deny_reason,
             transaction_date_time_utc,
             transaction_date_time_pacific,
             location_id,
@@ -84,6 +85,7 @@ stg_littlepay__device_transactions AS (
             vehicle_id,
             granted_zone_ids,
             onward_zone_ids,
+            processed_timestamp_utc,
             _line_number,
             `instance`,
             extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__micropayment_adjustments.sql

@@ -14,7 +14,8 @@ clean_columns AS (
         {{ trim_make_empty_string_null('amount') }} AS amount,
         {{ trim_make_empty_string_null('time_period_type') }} AS time_period_type,
         {{ safe_cast('applied', type_boolean()) }} AS applied,
-        {{ trim_make_empty_string_null('zone_ids_us') }} AS zone_ids_us,
+        {{ trim_make_empty_string_null('zone_ids_used') }} AS zone_ids_used,
+        {{ trim_make_empty_string_null('incentive_product_id') }} AS incentive_product_id,
         CAST(_line_number AS INTEGER) AS _line_number,
         `instance`,
         extract_filename,
@@ -25,7 +26,7 @@ clean_columns AS (
         -- hashing at this step will preserve distinction between nulls and empty strings in case that is meaningful upstream
         {{ dbt_utils.generate_surrogate_key(['micropayment_id', 'adjustment_id', 'participant_id',
             'customer_id', 'product_id', 'type', 'description', 'amount', 'time_period_type',
-            'applied', 'zone_ids_us']) }} AS _content_hash,
+            'applied', 'zone_ids_used', 'incentive_product_id']) }} AS _content_hash,
     FROM source
 ),
 
@@ -51,7 +52,8 @@ stg_littlepay__micropayment_adjustments AS (
         amount,
         time_period_type,
         applied,
-        zone_ids_us,
+        zone_ids_used,
+        incentive_product_id,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__product_data.sql

@@ -6,11 +6,14 @@ clean_columns AS (
     SELECT
         {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
         {{ trim_make_empty_string_null('product_id') }} AS product_id,
+        {{ trim_make_empty_string_null('previous_version_id') }} AS previous_version_id,
+        {{ trim_make_empty_string_null('original_version_id') }} AS original_version_id,
         {{ trim_make_empty_string_null('product_code') }} AS product_code,
         {{ trim_make_empty_string_null('product_description') }} AS product_description,
         {{ trim_make_empty_string_null('product_type') }} AS product_type,
         {{ trim_make_empty_string_null('activation_type') }} AS activation_type,
         {{ trim_make_empty_string_null('product_status') }} AS product_status,
+        {{ trim_make_empty_string_null('superseded') }} AS superseded,
         {{ trim_make_empty_string_null('created_date') }} AS created_date,
         {{ trim_make_empty_string_null('capping_type') }} AS capping_type,
         {{ safe_cast('multi_operator', type_boolean()) }} AS multi_operator,
@@ -23,12 +26,24 @@ clean_columns AS (
         {{ safe_cast('scheduled_end_date_time', 'DATE') }} AS scheduled_end_date_time,
         {{ safe_cast('all_day', type_boolean()) }} AS all_day,
         {{ trim_make_empty_string_null('weekly_cap_start_day') }} AS weekly_cap_start_day,
+        {{ trim_make_empty_string_null('weekly_cap_end_day') }} AS weekly_cap_end_day,
         {{ safe_cast('number_of_days_in_cap_window', type_float()) }} AS number_of_days_in_cap_window,
         {{ safe_cast('capping_duration', type_float()) }} AS capping_duration,
         {{ safe_cast('number_of_transfer', type_float()) }} AS number_of_transfer,
         {{ trim_make_empty_string_null('capping_time_zone') }} AS capping_time_zone,
-        {{ safe_cast('capping_overlap', 'TIME') }} AS capping_overlap,
+        {{ safe_cast('capping_overlap_time', 'TIME') }} AS capping_overlap_time,
         {{ trim_make_empty_string_null('capping_application_level') }} AS capping_application_level,
+        {{ safe_cast('route_capping_enabled', type_boolean()) }} AS route_capping_enabled,
+        {{ trim_make_empty_string_null('routes') }} AS routes,
+        {{ safe_cast('zoned_capping_enabled', type_boolean()) }} AS zoned_capping_enabled,
+        {{ trim_make_empty_string_null('zoned_capping_mode') }} AS zoned_capping_mode,
+        {{ trim_make_empty_string_null('zoned_capping_pricing_type') }} AS zoned_capping_pricing_type,
+        {{ trim_make_empty_string_null('on_peak_zones') }} AS on_peak_zones,
+        {{ trim_make_empty_string_null('off_peak_zones') }} AS off_peak_zones,
+        {{ safe_cast('incentive_enabled', type_boolean()) }} AS incentive_enabled,
+        {{ trim_make_empty_string_null('incentive_type') }} AS incentive_type,
+        {{ trim_make_empty_string_null('discount_qualifier') }} AS discount_qualifier,
+        {{ trim_make_empty_string_null('configuration') }} AS configuration,
         CAST(_line_number AS INTEGER) AS _line_number,
         `instance`,
         extract_filename,
@@ -37,14 +52,16 @@ clean_columns AS (
         ts,
         -- hash all content not generated by us to enable deduping full dup rows
         -- hashing at this step will preserve distinction between nulls and empty strings in case that is meaningful upstream
-        {{ dbt_utils.generate_surrogate_key(['participant_id', 'product_id', 'product_code',
-            'product_description', 'product_type', 'activation_type', 'product_status',
+        {{ dbt_utils.generate_surrogate_key(['participant_id', 'product_id', 'previous_version_id', 'original_version_id', 'product_code',
+            'product_description', 'product_type', 'activation_type', 'product_status', 'superseded',
             'created_date', 'capping_type', 'multi_operator', 'capping_start_time',
             'capping_end_time', 'rules_transaction_types', 'rules_default_limit',
             'rules_max_fare_value', 'scheduled_start_date_time', 'scheduled_end_date_time',
-            'all_day', 'weekly_cap_start_day', 'number_of_days_in_cap_window',
-            'capping_duration', 'number_of_transfer', 'capping_time_zone', 'capping_overlap',
-            'capping_application_level']) }} AS _content_hash,
+            'all_day', 'weekly_cap_start_day', 'weekly_cap_end_day', 'number_of_days_in_cap_window',
+            'capping_duration', 'number_of_transfer', 'capping_time_zone', 'capping_overlap_time',
+            'capping_application_level', 'route_capping_enabled', 'routes', 'zoned_capping_enabled', 'zoned_capping_mode',
+            'zoned_capping_pricing_type', 'on_peak_zones', 'off_peak_zones', 'incentive_enabled', 'incentive_type',
+            'discount_qualifier', 'configuration']) }} AS _content_hash,
     FROM source
 ),
 
@@ -62,11 +79,14 @@ stg_littlepay__product_data AS (
     SELECT
         participant_id,
         product_id,
+        previous_version_id,
+        original_version_id,
         product_code,
         product_description,
         product_type,
         activation_type,
         product_status,
+        superseded,
         created_date,
         capping_type,
         multi_operator,
@@ -79,12 +99,24 @@ stg_littlepay__product_data AS (
         scheduled_end_date_time,
         all_day,
         weekly_cap_start_day,
+        weekly_cap_end_day,
         number_of_days_in_cap_window,
         capping_duration,
         number_of_transfer,
         capping_time_zone,
-        capping_overlap,
+        capping_overlap_time,
         capping_application_level,
+        route_capping_enabled,
+        routes,
+        zoned_capping_enabled,
+        zoned_capping_mode,
+        zoned_capping_pricing_type,
+        on_peak_zones,
+        off_peak_zones,
+        incentive_enabled,
+        incentive_type,
+        discount_qualifier,
+        configuration,
         _line_number,
         `instance`,
         extract_filename,