added trim_make_empty_string_null macro to littlepay staging tables (#…

…2953)

warehouse/models/staging/payments/littlepay/stg_littlepay__authorisations.sql

@@ -4,17 +4,17 @@ WITH source AS (
 
 stg_littlepay__authorisations AS (
     SELECT
-        participant_id,
-        aggregation_id,
-        acquirer_id,
-        request_type,
+        {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
+        {{ trim_make_empty_string_null('aggregation_id') }} AS aggregation_id,
+        {{ trim_make_empty_string_null('acquirer_id') }} AS acquirer_id,
+        {{ trim_make_empty_string_null('request_type') }} AS request_type,
         {{ safe_cast('transaction_amount', type_numeric()) }} AS transaction_amount,
         {{ safe_cast('currency_code', type_int()) }} AS currency_code,
-        retrieval_reference_number,
-        littlepay_reference_number,
-        external_reference_number,
-        response_code,
-        status,
+        {{ trim_make_empty_string_null('retrieval_reference_number') }} AS retrieval_reference_number,
+        {{ trim_make_empty_string_null('littlepay_reference_number') }} AS littlepay_reference_number,
+        {{ trim_make_empty_string_null('external_reference_number') }} AS external_reference_number,
+        {{ trim_make_empty_string_null('response_code') }} AS response_code,
+        {{ trim_make_empty_string_null('status') }} AS status,
         {{ safe_cast('authorisation_date_time_utc', type_timestamp()) }} AS authorisation_date_time_utc,
         _line_number,
         `instance`,

warehouse/models/staging/payments/littlepay/stg_littlepay__customer_funding_source.sql

@@ -4,16 +4,16 @@ WITH source AS (
 
 stg_littlepay__customer_funding_source AS (
     SELECT
-        funding_source_id,
-        funding_source_vault_id,
-        customer_id,
-        bin,
-        masked_pan,
-        card_scheme,
-        issuer,
-        issuer_country,
-        form_factor,
-        principal_customer_id,
+        {{ trim_make_empty_string_null('funding_source_id') }} AS funding_source_id,
+        {{ trim_make_empty_string_null('funding_source_vault_id') }} AS funding_source_vault_id,
+        {{ trim_make_empty_string_null('customer_id') }} AS customer_id,
+        {{ trim_make_empty_string_null('bin') }} AS bin,
+        {{ trim_make_empty_string_null('masked_pan') }} AS masked_pan,
+        {{ trim_make_empty_string_null('card_scheme') }} AS card_scheme,
+        {{ trim_make_empty_string_null('issuer') }} AS issuer,
+        {{ trim_make_empty_string_null('issuer_country') }} AS issuer_country,
+        {{ trim_make_empty_string_null('form_factor') }} AS form_factor,
+        {{ trim_make_empty_string_null('principal_customer_id') }} AS principal_customer_id,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__device_transaction_purchases.sql

@@ -4,11 +4,11 @@ WITH source AS (
 
 stg_littlepay__device_transaction_purchases AS (
     SELECT
-        littlepay_transaction_id,
-        purchase_id,
-        correlated_purchase_id,
-        product_id,
-        description,
+        {{ trim_make_empty_string_null('littlepay_transaction_id') }} AS littlepay_transaction_id,
+        {{ trim_make_empty_string_null('purchase_id') }} AS purchase_id,
+        {{ trim_make_empty_string_null('correlated_purchase_id') }} AS correlated_purchase_id,
+        {{ trim_make_empty_string_null('product_id') }} AS product_id,
+        {{ trim_make_empty_string_null('description') }} AS description,
         {{ safe_cast('indicative_amount', type_numeric()) }} AS indicative_amount,
         {{ safe_cast('transaction_time', type_timestamp()) }} AS transaction_time,
         _line_number,

warehouse/models/staging/payments/littlepay/stg_littlepay__device_transactions.sql

@@ -4,34 +4,34 @@ WITH source AS (
 
 stg_littlepay__device_transactions AS (
     SELECT
-        participant_id,
-        customer_id,
-        device_transaction_id,
-        littlepay_transaction_id,
-        device_id,
-        device_id_issuer,
-        type,
-        transaction_outcome,
-        transction_deny_reason,
-        transaction_date_time_utc,
+        {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
+        {{ trim_make_empty_string_null('customer_id') }} AS customer_id,
+        {{ trim_make_empty_string_null('device_transaction_id') }} AS device_transaction_id,
+        {{ trim_make_empty_string_null('littlepay_transaction_id') }} AS littlepay_transaction_id,
+        {{ trim_make_empty_string_null('device_id') }} AS device_id,
+        {{ trim_make_empty_string_null('device_id_issuer') }} AS device_id_issuer,
+        {{ trim_make_empty_string_null('type') }} AS type,
+        {{ trim_make_empty_string_null('transaction_outcome') }} AS transaction_outcome,
+        {{ trim_make_empty_string_null('transction_deny_reason') }} AS transction_deny_reason,
+        {{ trim_make_empty_string_null('transaction_date_time_utc') }} AS transaction_date_time_utc,
         DATETIME(
             TIMESTAMP(transaction_date_time_utc), "America/Los_Angeles"
         ) AS transaction_date_time_pacific,
          -- trim to align with gtfs cleaning steps
          -- since these fields are used to join with gtfs data
-        TRIM(location_id) AS location_id,
-        TRIM(route_id) AS route_id,
-        location_scheme,
-        location_name,
-        zone_id,
-        mode,
-        direction,
+        {{ trim_make_empty_string_null('location_id') }} AS location_id,
+        {{ trim_make_empty_string_null('route_id') }} AS route_id,
+        {{ trim_make_empty_string_null('location_scheme') }} AS location_scheme,
+        {{ trim_make_empty_string_null('location_name') }} AS location_name,
+        {{ trim_make_empty_string_null('zone_id') }} AS zone_id,
+        {{ trim_make_empty_string_null('mode') }} AS mode,
+        {{ trim_make_empty_string_null('direction') }} AS direction,
         CAST(latitude AS NUMERIC) AS latitude,
         CAST(longitude AS NUMERIC) AS longitude,
         ST_GEOGPOINT(CAST(longitude AS NUMERIC), CAST(latitude AS NUMERIC)) AS geography,
-        vehicle_id,
-        granted_zone_ids,
-        onward_zone_ids,
+        {{ trim_make_empty_string_null('vehicle_id') }} AS vehicle_id,
+        {{ trim_make_empty_string_null('granted_zone_ids') }} AS granted_zone_ids,
+        {{ trim_make_empty_string_null('onward_zone_ids') }} AS onward_zone_ids,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__micropayment_adjustments.sql

@@ -4,17 +4,17 @@ WITH source AS (
 
 stg_littlepay__micropayment_adjustments AS (
     SELECT
-        micropayment_id,
-        adjustment_id,
-        participant_id,
-        customer_id,
-        product_id,
-        type,
-        description,
-        amount,
-        time_period_type,
+        {{ trim_make_empty_string_null('micropayment_id') }} AS micropayment_id,
+        {{ trim_make_empty_string_null('adjustment_id') }} AS adjustment_id,
+        {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
+        {{ trim_make_empty_string_null('customer_id') }} AS customer_id,
+        {{ trim_make_empty_string_null('product_id') }} AS product_id,
+        {{ trim_make_empty_string_null('type') }} AS type,
+        {{ trim_make_empty_string_null('description') }} AS description,
+        {{ trim_make_empty_string_null('amount') }} AS amount,
+        {{ trim_make_empty_string_null('time_period_type') }} AS time_period_type,
         {{ safe_cast('applied', type_boolean()) }} AS applied,
-        zone_ids_us,
+        {{ trim_make_empty_string_null('zone_ids_us') }} AS zone_ids_us,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__micropayment_device_transactions.sql

@@ -4,8 +4,8 @@ WITH source AS (
 
 stg_littlepay__micropayment_device_transactions AS (
     SELECT
-        littlepay_transaction_id,
-        micropayment_id,
+        {{ trim_make_empty_string_null('littlepay_transaction_id') }} AS littlepay_transaction_id,
+        {{ trim_make_empty_string_null('micropayment_id') }} AS micropayment_id,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__micropayments.sql

@@ -4,18 +4,18 @@ WITH source AS (
 
 stg_littlepay__micropayments AS (
     SELECT
-        micropayment_id,
-        aggregation_id,
-        participant_id,
-        customer_id,
-        funding_source_vault_id,
+        {{ trim_make_empty_string_null('micropayment_id') }} AS micropayment_id,
+        {{ trim_make_empty_string_null('aggregation_id') }} AS aggregation_id,
+        {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
+        {{ trim_make_empty_string_null('customer_id') }} AS customer_id,
+        {{ trim_make_empty_string_null('funding_source_vault_id') }} AS funding_source_vault_id,
         TIMESTAMP(transaction_time) AS transaction_time,
-        payment_liability,
+        {{ trim_make_empty_string_null('payment_liability') }} AS payment_liability,
         SAFE_CAST(charge_amount AS NUMERIC) AS charge_amount,
         SAFE_CAST(nominal_amount AS NUMERIC) AS nominal_amount,
-        currency_code,
-        type,
-        charge_type,
+        {{ trim_make_empty_string_null('currency_code') }} AS currency_code,
+        {{ trim_make_empty_string_null('type') }} AS type,
+        {{ trim_make_empty_string_null('charge_type') }} AS charge_type,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__product_data.sql

@@ -4,31 +4,31 @@ WITH source AS (
 
 stg_littlepay__product_data AS (
     SELECT
-        participant_id,
-        product_id,
-        product_code,
-        product_description,
-        product_type,
-        activation_type,
-        product_status,
-        created_date,
-        capping_type,
+        {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
+        {{ trim_make_empty_string_null('product_id') }} AS product_id,
+        {{ trim_make_empty_string_null('product_code') }} AS product_code,
+        {{ trim_make_empty_string_null('product_description') }} AS product_description,
+        {{ trim_make_empty_string_null('product_type') }} AS product_type,
+        {{ trim_make_empty_string_null('activation_type') }} AS activation_type,
+        {{ trim_make_empty_string_null('product_status') }} AS product_status,
+        {{ trim_make_empty_string_null('created_date') }} AS created_date,
+        {{ trim_make_empty_string_null('capping_type') }} AS capping_type,
         {{ safe_cast('multi_operator', type_boolean()) }} AS multi_operator,
         {{ safe_cast('capping_start_time', 'TIME') }} AS capping_start_time,
         {{ safe_cast('capping_end_time', 'TIME') }} AS capping_end_time,
-        rules_transaction_types,
-        rules_default_limit,
-        rules_max_fare_value,
+        {{ trim_make_empty_string_null('rules_transaction_types') }} AS rules_transaction_types,
+        {{ trim_make_empty_string_null('rules_default_limit') }} AS rules_default_limit,
+        {{ trim_make_empty_string_null('rules_max_fare_value') }} AS rules_max_fare_value,
         {{ safe_cast('scheduled_start_date_time', 'DATE') }} AS scheduled_start_date_time,
         {{ safe_cast('scheduled_end_date_time', 'DATE') }} AS scheduled_end_date_time,
         {{ safe_cast('all_day', type_boolean()) }} AS all_day,
-        weekly_cap_start_day,
+        {{ trim_make_empty_string_null('weekly_cap_start_day') }} AS weekly_cap_start_day,
         {{ safe_cast('number_of_days_in_cap_window', type_float()) }} AS number_of_days_in_cap_window,
         {{ safe_cast('capping_duration', type_float()) }} AS capping_duration,
         {{ safe_cast('number_of_transfer', type_float()) }} AS number_of_transfer,
-        capping_time_zone,
+        {{ trim_make_empty_string_null('capping_time_zone') }} AS capping_time_zone,
         {{ safe_cast('capping_overlap', 'TIME') }} AS capping_overlap,
-        capping_application_level,
+        {{ trim_make_empty_string_null('capping_application_level') }} AS capping_application_level,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__refunds.sql

@@ -4,30 +4,30 @@ WITH source AS (
 
 stg_littlepay__refunds AS (
     SELECT
-        refund_id,
-        participant_id,
-        customer_id,
-        micropayment_id,
-        aggregation_id,
-        settlement_id,
-        retrieval_reference_number,
+        {{ trim_make_empty_string_null('refund_id') }} AS refund_id,
+        {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
+        {{ trim_make_empty_string_null('customer_id') }} AS customer_id,
+        {{ trim_make_empty_string_null('micropayment_id') }} AS micropayment_id,
+        {{ trim_make_empty_string_null('aggregation_id') }} AS aggregation_id,
+        {{ trim_make_empty_string_null('settlement_id') }} AS settlement_id,
+        {{ trim_make_empty_string_null('retrieval_reference_number') }} AS retrieval_reference_number,
         {{ safe_cast('transaction_date', 'DATE') }} AS transaction_date,
         {{ safe_cast('transaction_amount', type_numeric()) }} AS transaction_amount,
         {{ safe_cast('proposed_amount', type_numeric()) }} AS proposed_amount,
         {{ safe_cast('refund_amount', type_numeric()) }} AS refund_amount,
         {{ safe_cast('currency_code', type_int()) }} AS currency_code,
-        status,
-        initiator,
-        reason,
-        approval_status,
-        issuer,
-        issuer_comment,
+        {{ trim_make_empty_string_null('status') }} AS status,
+        {{ trim_make_empty_string_null('initiator') }} AS initiator,
+        {{ trim_make_empty_string_null('reason') }} AS reason,
+        {{ trim_make_empty_string_null('approval_status') }} AS approval_status,
+        {{ trim_make_empty_string_null('issuer') }} AS issuer,
+        {{ trim_make_empty_string_null('issuer_comment') }} AS issuer_comment,
         {{ safe_cast('created_time', type_timestamp()) }} AS created_time,
         {{ safe_cast('approved_time', type_timestamp()) }} AS approved_time,
-        settlement_status,
+        {{ trim_make_empty_string_null('settlement_status') }} AS settlement_status,
         {{ safe_cast('settlement_status_time', 'DATE') }} AS settlement_status_time,
-        settlement_reason_code,
-        settlement_response_text,
+        {{ trim_make_empty_string_null('settlement_reason_code') }} AS settlement_reason_code,
+        {{ trim_make_empty_string_null('settlement_response_text') }} AS settlement_response_text,
         _line_number,
         `instance`,
         extract_filename,

warehouse/models/staging/payments/littlepay/stg_littlepay__settlements.sql

@@ -4,17 +4,17 @@ WITH source AS (
 
 stg_littlepay__settlements AS (
     SELECT
-        settlement_id,
-        participant_id,
-        aggregation_id,
-        customer_id,
-        funding_source_id,
+        {{ trim_make_empty_string_null('settlement_id') }} AS settlement_id,
+        {{ trim_make_empty_string_null('participant_id') }} AS participant_id,
+        {{ trim_make_empty_string_null('aggregation_id') }} AS aggregation_id,
+        {{ trim_make_empty_string_null('customer_id') }} AS customer_id,
+        {{ trim_make_empty_string_null('funding_source_id') }} AS funding_source_id,
         {{ safe_cast('transaction_amount', type_numeric()) }} AS transaction_amount,
-        retrieval_reference_number,
-        littlepay_reference_number,
-        external_reference_number,
+        {{ trim_make_empty_string_null('retrieval_reference_number') }} AS retrieval_reference_number,
+        {{ trim_make_empty_string_null('littlepay_reference_number') }} AS littlepay_reference_number,
+        {{ trim_make_empty_string_null('external_reference_number') }} AS external_reference_number,
         {{ safe_cast('settlement_requested_date_time_utc', type_timestamp()) }} AS settlement_requested_date_time_utc,
-        acquirer,
+        {{ trim_make_empty_string_null('acquirer') }} AS acquirer,
         _line_number,
         `instance`,
         extract_filename,