From 8005934648a196b274d99668457ab4a7f1bc215f Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Mon, 16 Oct 2023 10:10:23 -0500 Subject: [PATCH] Add new Littlepay columns + correct mistyped columns (#2996) * Add missing Littlepay columns to external tables * Correct column names and add new columns to models --- .../littlepay/customer_funding_source.yml | 2 + .../littlepay/device_transactions.yml | 4 +- .../littlepay/micropayment_adjustments.yml | 4 +- .../payments/littlepay/product_data.yml | 32 ++++++++++++- .../mart/payments/fct_payments_rides_v2.sql | 4 +- .../staging/payments/littlepay/_littlepay.yml | 17 +++++-- ...stg_littlepay__customer_funding_source.sql | 4 +- .../stg_littlepay__device_transactions.sql | 10 ++-- ...tg_littlepay__micropayment_adjustments.sql | 8 ++-- .../littlepay/stg_littlepay__product_data.sql | 46 ++++++++++++++++--- 10 files changed, 107 insertions(+), 24 deletions(-) diff --git a/airflow/dags/create_external_tables/payments/littlepay/customer_funding_source.yml b/airflow/dags/create_external_tables/payments/littlepay/customer_funding_source.yml index 20830d2f0a..a0e43c9d1a 100644 --- a/airflow/dags/create_external_tables/payments/littlepay/customer_funding_source.yml +++ b/airflow/dags/create_external_tables/payments/littlepay/customer_funding_source.yml @@ -30,5 +30,7 @@ schema_fields: type: STRING - name: principal_customer_id type: STRING + - name: participant_id + type: STRING - name: _line_number type: STRING diff --git a/airflow/dags/create_external_tables/payments/littlepay/device_transactions.yml b/airflow/dags/create_external_tables/payments/littlepay/device_transactions.yml index c9f5eec409..35a8121c2c 100644 --- a/airflow/dags/create_external_tables/payments/littlepay/device_transactions.yml +++ b/airflow/dags/create_external_tables/payments/littlepay/device_transactions.yml @@ -26,7 +26,7 @@ schema_fields: type: STRING - name: transaction_outcome type: STRING - - name: transction_deny_reason + - name: transaction_deny_reason type: STRING - name: transaction_date_time_utc type: STRING @@ -54,5 +54,7 @@ schema_fields: type: STRING - name: onward_zone_ids type: STRING + - name: processed_timestamp_utc + type: STRING - name: _line_number type: STRING diff --git a/airflow/dags/create_external_tables/payments/littlepay/micropayment_adjustments.yml b/airflow/dags/create_external_tables/payments/littlepay/micropayment_adjustments.yml index ab70b31dff..1f71b9e735 100644 --- a/airflow/dags/create_external_tables/payments/littlepay/micropayment_adjustments.yml +++ b/airflow/dags/create_external_tables/payments/littlepay/micropayment_adjustments.yml @@ -30,7 +30,9 @@ schema_fields: type: STRING # Peak, Off Peak - name: applied type: STRING - - name: zone_ids_us + - name: zone_ids_used + type: STRING + - name: incentive_product_id type: STRING - name: _line_number type: STRING diff --git a/airflow/dags/create_external_tables/payments/littlepay/product_data.yml b/airflow/dags/create_external_tables/payments/littlepay/product_data.yml index 3ed80e14ff..697da314cb 100644 --- a/airflow/dags/create_external_tables/payments/littlepay/product_data.yml +++ b/airflow/dags/create_external_tables/payments/littlepay/product_data.yml @@ -14,6 +14,10 @@ schema_fields: type: STRING - name: product_id type: STRING + - name: previous_version_id + type: STRING + - name: original_version_id + type: STRING - name: product_code type: STRING - name: product_description @@ -24,6 +28,8 @@ schema_fields: type: STRING # (Customer, Card) - name: product_status type: STRING # (Active, Inactive) + - name: superseded + type: STRING - name: created_date type: STRING - name: capping_type @@ -48,6 +54,8 @@ schema_fields: type: STRING - name: weekly_cap_start_day type: STRING # (Monday, Sunday) + - name: weekly_cap_end_day + type: STRING - name: number_of_days_in_cap_window type: STRING - name: capping_duration @@ -56,9 +64,31 @@ schema_fields: type: STRING - name: capping_time_zone type: STRING - - name: capping_overlap + - name: capping_overlap_time type: STRING - name: capping_application_level type: STRING # (Customer, Card) + - name: route_capping_enabled + type: STRING + - name: routes + type: STRING + - name: zoned_capping_enabled + type: BOOLEAN + - name: zoned_capping_mode + type: STRING + - name: zoned_capping_pricing_type + type: STRING + - name: on_peak_zones + type: STRING + - name: off_peak_zones + type: STRING + - name: incentive_enabled + type: BOOLEAN + - name: incentive_type + type: STRING + - name: discount_qualifier + type: STRING + - name: configuration + type: STRING - name: _line_number type: STRING diff --git a/warehouse/models/mart/payments/fct_payments_rides_v2.sql b/warehouse/models/mart/payments/fct_payments_rides_v2.sql index b157b43869..73661574d7 100644 --- a/warehouse/models/mart/payments/fct_payments_rides_v2.sql +++ b/warehouse/models/mart/payments/fct_payments_rides_v2.sql @@ -199,7 +199,7 @@ initial_transactions AS ( dt.device_id_issuer, dt.type, dt.transaction_outcome, - dt.transction_deny_reason, + dt.transaction_deny_reason, dt.transaction_date_time_utc, dt.location_scheme, dt.location_name, @@ -238,7 +238,7 @@ second_transactions AS ( dt.device_id_issuer, dt.type, dt.transaction_outcome, - dt.transction_deny_reason, + dt.transaction_deny_reason, dt.transaction_date_time_utc, dt.location_scheme, dt.location_name, diff --git a/warehouse/models/staging/payments/littlepay/_littlepay.yml b/warehouse/models/staging/payments/littlepay/_littlepay.yml index ef7ac69eee..3b7a66c2d2 100644 --- a/warehouse/models/staging/payments/littlepay/_littlepay.yml +++ b/warehouse/models/staging/payments/littlepay/_littlepay.yml @@ -321,14 +321,23 @@ models: Possible values are `bus` and `train`. - name: onward_zone_ids - description: "" + description: | + Zones passed for incomplete variable trips. + + Can be a comma-separated string. + - name: processed_timestamp_utc + description: | + Timestamp of when the device data was processed by Littlepay. + + The difference between this field and transaction_date_time_utc can be sizeable when data + is sent by the customer via the Back Office API - name: participant_id description: The unique identifier of the participant that the device belongs to. - name: route_id description: The route identifier provided by the device. - name: transaction_date_time_utc description: The date and time that the customer tapped on the device. - - name: transction_deny_reason + - name: transaction_deny_reason description: | The reason a transaction was denied. @@ -379,7 +388,7 @@ models: - amount - time_period_type - applied - - zone_ids_us + - zone_ids_used columns: - name: adjustment_id description: '{{ doc("micropayment_adjustments_adjustment_id") }}' @@ -593,7 +602,7 @@ models: description: The number of free transfers after the initial tap. - name: capping_time_zone description: The time zone where the capping product was used. - - name: capping_overlap + - name: capping_overlap_time description: The time a customer can overlap within a 24-hour period. - name: capping_application_level description: This is the customer level tapping as customers can have multiple cards (physical / digital). diff --git a/warehouse/models/staging/payments/littlepay/stg_littlepay__customer_funding_source.sql b/warehouse/models/staging/payments/littlepay/stg_littlepay__customer_funding_source.sql index ee7342ccce..c70bcc8b5a 100644 --- a/warehouse/models/staging/payments/littlepay/stg_littlepay__customer_funding_source.sql +++ b/warehouse/models/staging/payments/littlepay/stg_littlepay__customer_funding_source.sql @@ -14,6 +14,7 @@ clean_columns AS ( {{ trim_make_empty_string_null('issuer_country') }} AS issuer_country, {{ trim_make_empty_string_null('form_factor') }} AS form_factor, {{ trim_make_empty_string_null('principal_customer_id') }} AS principal_customer_id, + {{ trim_make_empty_string_null('participant_id') }} AS participant_id, CAST(_line_number AS INTEGER) AS _line_number, `instance`, extract_filename, @@ -24,7 +25,7 @@ clean_columns AS ( -- hashing at this step will preserve distinction between nulls and empty strings in case that is meaningful upstream {{ dbt_utils.generate_surrogate_key(['funding_source_id', 'funding_source_vault_id', 'customer_id', 'bin', 'masked_pan', 'card_scheme', 'issuer', 'issuer_country', - 'form_factor']) }} AS _content_hash, + 'form_factor', 'principal_customer_id', 'participant_id']) }} AS _content_hash, FROM source ), @@ -60,6 +61,7 @@ stg_littlepay__customer_funding_source AS ( issuer_country, form_factor, principal_customer_id, + participant_id, _line_number, `instance`, extract_filename, diff --git a/warehouse/models/staging/payments/littlepay/stg_littlepay__device_transactions.sql b/warehouse/models/staging/payments/littlepay/stg_littlepay__device_transactions.sql index 8f500b6980..3989166441 100644 --- a/warehouse/models/staging/payments/littlepay/stg_littlepay__device_transactions.sql +++ b/warehouse/models/staging/payments/littlepay/stg_littlepay__device_transactions.sql @@ -12,7 +12,7 @@ clean_columns AS ( {{ trim_make_empty_string_null('device_id_issuer') }} AS device_id_issuer, {{ trim_make_empty_string_null('type') }} AS type, {{ trim_make_empty_string_null('transaction_outcome') }} AS transaction_outcome, - {{ trim_make_empty_string_null('transction_deny_reason') }} AS transction_deny_reason, + {{ trim_make_empty_string_null('transaction_deny_reason') }} AS transaction_deny_reason, {{ trim_make_empty_string_null('transaction_date_time_utc') }} AS transaction_date_time_utc, DATETIME( TIMESTAMP(transaction_date_time_utc), "America/Los_Angeles" @@ -32,6 +32,7 @@ clean_columns AS ( {{ trim_make_empty_string_null('vehicle_id') }} AS vehicle_id, {{ trim_make_empty_string_null('granted_zone_ids') }} AS granted_zone_ids, {{ trim_make_empty_string_null('onward_zone_ids') }} AS onward_zone_ids, + {{ trim_make_empty_string_null('processed_timestamp_utc') }} AS processed_timestamp_utc, CAST(_line_number AS INTEGER) AS _line_number, `instance`, extract_filename, @@ -42,10 +43,10 @@ clean_columns AS ( -- hashing at this step will preserve distinction between nulls and empty strings in case that is meaningful upstream {{ dbt_utils.generate_surrogate_key(['participant_id', 'customer_id', 'device_transaction_id', 'littlepay_transaction_id', 'device_id', 'device_id_issuer', - 'type', 'transaction_outcome', 'transction_deny_reason', 'transaction_date_time_utc', + 'type', 'transaction_outcome', 'transaction_deny_reason', 'transaction_date_time_utc', 'location_id', 'location_scheme', 'location_name', 'zone_id', 'route_id', 'mode', 'direction', 'latitude', 'longitude', 'vehicle_id', 'granted_zone_ids', - 'onward_zone_ids']) }} AS _content_hash, + 'onward_zone_ids', 'processed_timestamp_utc']) }} AS _content_hash, FROM source ), @@ -68,7 +69,7 @@ stg_littlepay__device_transactions AS ( device_id_issuer, type, transaction_outcome, - transction_deny_reason, + transaction_deny_reason, transaction_date_time_utc, transaction_date_time_pacific, location_id, @@ -84,6 +85,7 @@ stg_littlepay__device_transactions AS ( vehicle_id, granted_zone_ids, onward_zone_ids, + processed_timestamp_utc, _line_number, `instance`, extract_filename, diff --git a/warehouse/models/staging/payments/littlepay/stg_littlepay__micropayment_adjustments.sql b/warehouse/models/staging/payments/littlepay/stg_littlepay__micropayment_adjustments.sql index 7b0b959ec1..4da3d69bf8 100644 --- a/warehouse/models/staging/payments/littlepay/stg_littlepay__micropayment_adjustments.sql +++ b/warehouse/models/staging/payments/littlepay/stg_littlepay__micropayment_adjustments.sql @@ -14,7 +14,8 @@ clean_columns AS ( {{ trim_make_empty_string_null('amount') }} AS amount, {{ trim_make_empty_string_null('time_period_type') }} AS time_period_type, {{ safe_cast('applied', type_boolean()) }} AS applied, - {{ trim_make_empty_string_null('zone_ids_us') }} AS zone_ids_us, + {{ trim_make_empty_string_null('zone_ids_used') }} AS zone_ids_used, + {{ trim_make_empty_string_null('incentive_product_id') }} AS incentive_product_id, CAST(_line_number AS INTEGER) AS _line_number, `instance`, extract_filename, @@ -25,7 +26,7 @@ clean_columns AS ( -- hashing at this step will preserve distinction between nulls and empty strings in case that is meaningful upstream {{ dbt_utils.generate_surrogate_key(['micropayment_id', 'adjustment_id', 'participant_id', 'customer_id', 'product_id', 'type', 'description', 'amount', 'time_period_type', - 'applied', 'zone_ids_us']) }} AS _content_hash, + 'applied', 'zone_ids_used', 'incentive_product_id']) }} AS _content_hash, FROM source ), @@ -51,7 +52,8 @@ stg_littlepay__micropayment_adjustments AS ( amount, time_period_type, applied, - zone_ids_us, + zone_ids_used, + incentive_product_id, _line_number, `instance`, extract_filename, diff --git a/warehouse/models/staging/payments/littlepay/stg_littlepay__product_data.sql b/warehouse/models/staging/payments/littlepay/stg_littlepay__product_data.sql index d97f40d07e..1fdeb59e78 100644 --- a/warehouse/models/staging/payments/littlepay/stg_littlepay__product_data.sql +++ b/warehouse/models/staging/payments/littlepay/stg_littlepay__product_data.sql @@ -6,11 +6,14 @@ clean_columns AS ( SELECT {{ trim_make_empty_string_null('participant_id') }} AS participant_id, {{ trim_make_empty_string_null('product_id') }} AS product_id, + {{ trim_make_empty_string_null('previous_version_id') }} AS previous_version_id, + {{ trim_make_empty_string_null('original_version_id') }} AS original_version_id, {{ trim_make_empty_string_null('product_code') }} AS product_code, {{ trim_make_empty_string_null('product_description') }} AS product_description, {{ trim_make_empty_string_null('product_type') }} AS product_type, {{ trim_make_empty_string_null('activation_type') }} AS activation_type, {{ trim_make_empty_string_null('product_status') }} AS product_status, + {{ trim_make_empty_string_null('superseded') }} AS superseded, {{ trim_make_empty_string_null('created_date') }} AS created_date, {{ trim_make_empty_string_null('capping_type') }} AS capping_type, {{ safe_cast('multi_operator', type_boolean()) }} AS multi_operator, @@ -23,12 +26,24 @@ clean_columns AS ( {{ safe_cast('scheduled_end_date_time', 'DATE') }} AS scheduled_end_date_time, {{ safe_cast('all_day', type_boolean()) }} AS all_day, {{ trim_make_empty_string_null('weekly_cap_start_day') }} AS weekly_cap_start_day, + {{ trim_make_empty_string_null('weekly_cap_end_day') }} AS weekly_cap_end_day, {{ safe_cast('number_of_days_in_cap_window', type_float()) }} AS number_of_days_in_cap_window, {{ safe_cast('capping_duration', type_float()) }} AS capping_duration, {{ safe_cast('number_of_transfer', type_float()) }} AS number_of_transfer, {{ trim_make_empty_string_null('capping_time_zone') }} AS capping_time_zone, - {{ safe_cast('capping_overlap', 'TIME') }} AS capping_overlap, + {{ safe_cast('capping_overlap_time', 'TIME') }} AS capping_overlap_time, {{ trim_make_empty_string_null('capping_application_level') }} AS capping_application_level, + {{ safe_cast('route_capping_enabled', type_boolean()) }} AS route_capping_enabled, + {{ trim_make_empty_string_null('routes') }} AS routes, + {{ safe_cast('zoned_capping_enabled', type_boolean()) }} AS zoned_capping_enabled, + {{ trim_make_empty_string_null('zoned_capping_mode') }} AS zoned_capping_mode, + {{ trim_make_empty_string_null('zoned_capping_pricing_type') }} AS zoned_capping_pricing_type, + {{ trim_make_empty_string_null('on_peak_zones') }} AS on_peak_zones, + {{ trim_make_empty_string_null('off_peak_zones') }} AS off_peak_zones, + {{ safe_cast('incentive_enabled', type_boolean()) }} AS incentive_enabled, + {{ trim_make_empty_string_null('incentive_type') }} AS incentive_type, + {{ trim_make_empty_string_null('discount_qualifier') }} AS discount_qualifier, + {{ trim_make_empty_string_null('configuration') }} AS configuration, CAST(_line_number AS INTEGER) AS _line_number, `instance`, extract_filename, @@ -37,14 +52,16 @@ clean_columns AS ( ts, -- hash all content not generated by us to enable deduping full dup rows -- hashing at this step will preserve distinction between nulls and empty strings in case that is meaningful upstream - {{ dbt_utils.generate_surrogate_key(['participant_id', 'product_id', 'product_code', - 'product_description', 'product_type', 'activation_type', 'product_status', + {{ dbt_utils.generate_surrogate_key(['participant_id', 'product_id', 'previous_version_id', 'original_version_id', 'product_code', + 'product_description', 'product_type', 'activation_type', 'product_status', 'superseded', 'created_date', 'capping_type', 'multi_operator', 'capping_start_time', 'capping_end_time', 'rules_transaction_types', 'rules_default_limit', 'rules_max_fare_value', 'scheduled_start_date_time', 'scheduled_end_date_time', - 'all_day', 'weekly_cap_start_day', 'number_of_days_in_cap_window', - 'capping_duration', 'number_of_transfer', 'capping_time_zone', 'capping_overlap', - 'capping_application_level']) }} AS _content_hash, + 'all_day', 'weekly_cap_start_day', 'weekly_cap_end_day', 'number_of_days_in_cap_window', + 'capping_duration', 'number_of_transfer', 'capping_time_zone', 'capping_overlap_time', + 'capping_application_level', 'route_capping_enabled', 'routes', 'zoned_capping_enabled', 'zoned_capping_mode', + 'zoned_capping_pricing_type', 'on_peak_zones', 'off_peak_zones', 'incentive_enabled', 'incentive_type', + 'discount_qualifier', 'configuration']) }} AS _content_hash, FROM source ), @@ -62,11 +79,14 @@ stg_littlepay__product_data AS ( SELECT participant_id, product_id, + previous_version_id, + original_version_id, product_code, product_description, product_type, activation_type, product_status, + superseded, created_date, capping_type, multi_operator, @@ -79,12 +99,24 @@ stg_littlepay__product_data AS ( scheduled_end_date_time, all_day, weekly_cap_start_day, + weekly_cap_end_day, number_of_days_in_cap_window, capping_duration, number_of_transfer, capping_time_zone, - capping_overlap, + capping_overlap_time, capping_application_level, + route_capping_enabled, + routes, + zoned_capping_enabled, + zoned_capping_mode, + zoned_capping_pricing_type, + on_peak_zones, + off_peak_zones, + incentive_enabled, + incentive_type, + discount_qualifier, + configuration, _line_number, `instance`, extract_filename,