From 4504d27f03f62db63f837c473621b0d0538850ae Mon Sep 17 00:00:00 2001 From: Emiel Date: Thu, 7 Dec 2023 20:29:59 +0000 Subject: [PATCH] Test fixes? --- .../snowplow_ecommerce_events_stg.sql | 138 +++++++++++++++++- macros/bigquery/context_fields.sql | 6 +- .../get_cart_context_fields.sql | 2 +- .../get_page_view_context_fields.sql | 43 +++++- .../get_screen_view_context_fields.sql | 10 +- ...nowplow_ecommerce_base_events_this_run.sql | 54 ++++--- .../snowplow_ecommerce_sessions_this_run.sql | 2 +- 7 files changed, 207 insertions(+), 48 deletions(-) diff --git a/integration_tests/models/source/snowflake/snowplow_ecommerce_events_stg.sql b/integration_tests/models/source/snowflake/snowplow_ecommerce_events_stg.sql index 845375f..0128800 100644 --- a/integration_tests/models/source/snowflake/snowplow_ecommerce_events_stg.sql +++ b/integration_tests/models/source/snowflake/snowplow_ecommerce_events_stg.sql @@ -15,10 +15,144 @@ select from {{ ref('snowplow_ecommerce_events') }} +), flatten as ( + select + *, + contexts_com_snowplowanalytics_snowplow_web_page_1[0].id as page_view_id, + contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].id as ecommerce_user_id, + contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].email as ecommerce_user_email, + contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].is_guest as ecommerce_user_is_guest, + contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].type as ecommerce_page_type, + contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].language as ecommerce_page_language, + contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].locale as ecommerce_page_locale, + unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1:type as ecommerce_action_type, + unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1:name as ecommerce_action_name, + contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].cart_id as ecommerce_cart_id, + contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].currency as ecommerce_cart_currency, + contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].total_value as ecommerce_cart_total_value, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].step as checkout_step_number, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].account_type as checkout_account_type, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].billing_full_address as checkout_billing_full_address, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].billing_postcode as checkout_billing_postcode, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].coupon_code as checkout_coupon_code, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].delivery_method as checkout_delivery_method, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].delivery_provider as checkout_delivery_provider, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].marketing_opt_in as checkout_marketing_opt_in, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].payment_method as checkout_payment_method, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].proof_of_payment as checkout_proof_of_payment, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].shipping_full_address as checkout_shipping_full_address, + contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].shipping_postcode as checkout_shipping_postcode, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].transaction_id as ecommerce_transaction_id, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].currency as ecommerce_currency, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].payment_method as ecommerce_payment_method, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].revenue as ecommerce_revenue, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].total_quantity as ecommerce_total_quantity, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].credit_order as ecommerce_credit_order, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].discount_amount as ecommerce_discount_amount, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].discount_code as ecommerce_discount_code, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].shipping as ecommerce_shipping, + contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].tax as ecommerce_tax, + contexts_com_snowplowanalytics_snowplow_client_session_1[0].eventIndex as event_index, + contexts_com_snowplowanalytics_snowplow_client_session_1[0].firstEventId as first_event_id, + contexts_com_snowplowanalytics_snowplow_client_session_1[0].firstEventTimestamp as first_event_timestamp, + contexts_com_snowplowanalytics_snowplow_client_session_1[0].previousSessionId as previous_session_id, + contexts_com_snowplowanalytics_snowplow_client_session_1[0].sessionId as session_id, + contexts_com_snowplowanalytics_snowplow_client_session_1[0].sessionIndex as session_index, + contexts_com_snowplowanalytics_snowplow_client_session_1[0].storageMechanism as storage_mechanism, + contexts_com_snowplowanalytics_snowplow_client_session_1[0].userId as mobile_user_id, + contexts_com_snowplowanalytics_mobile_screen_1[0].id as screen_view_id, + contexts_com_snowplowanalytics_mobile_screen_1[0].name as screen_view_name + + from prep ) select - * + APP_ID, + PLATFORM, + ETL_TSTAMP, + COLLECTOR_TSTAMP, + DVCE_CREATED_TSTAMP, + EVENT_ID, + NAME_TRACKER, + V_TRACKER, + V_COLLECTOR, + V_ETL, + USER_ID, + USER_IPADDRESS, + USER_FINGERPRINT, + DOMAIN_USERID, + DOMAIN_SESSIONIDX, + NETWORK_USERID, + GEO_COUNTRY, + GEO_REGION, + GEO_CITY, + GEO_ZIPCODE, + GEO_LATITUDE, + GEO_LONGITUDE, + GEO_REGION_NAME, + IP_ISP, + IP_ORGANIZATION, + IP_DOMAIN, + IP_NETSPEED, + PAGE_URL, + PAGE_TITLE, + PAGE_REFERRER, + PAGE_URLSCHEME, + PAGE_URLHOST, + PAGE_URLPORT, + PAGE_URLPATH, + PAGE_URLQUERY, + PAGE_URLFRAGMENT, + REFR_URLSCHEME, + REFR_URLHOST, + REFR_URLPORT, + REFR_URLPATH, + REFR_URLQUERY, + REFR_URLFRAGMENT, + REFR_MEDIUM, + REFR_SOURCE, + REFR_TERM, + MKT_MEDIUM, + MKT_SOURCE, + MKT_TERM, + MKT_CONTENT, + MKT_CAMPAIGN, + SE_CATEGORY, + SE_ACTION, + SE_LABEL, + SE_PROPERTY, + SE_VALUE, + USERAGENT, + OS_NAME, + OS_FAMILY, + OS_MANUFACTURER, + OS_TIMEZONE, + BASE_CURRENCY, + GEO_TIMEZONE, + MKT_CLICKID, + MKT_NETWORK, + ETL_TAGS, + DVCE_SENT_TSTAMP, + REFR_DOMAIN_USERID, + REFR_DVCE_TSTAMP, + DOMAIN_SESSIONID, + DERIVED_TSTAMP, + EVENT_VENDOR, + EVENT_NAME, + EVENT_FORMAT, + EVENT_VERSION, + EVENT_FINGERPRINT, + object_construct('type', ecommerce_action_type, 'name', ecommerce_action_name) as unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1, + -- we do this complicated nonsense because Snowflake's `parse_json` function returns a null object if any of the fields inside the JSON are null... + TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id', page_view_id))) as contexts_com_snowplowanalytics_snowplow_web_page_1, + TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id',ecommerce_user_id,'email',ecommerce_user_email,'isGuest',ecommerce_user_is_guest))) as contexts_com_snowplowanalytics_snowplow_ecommerce_user_1, + TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('type',ecommerce_page_type,'language',ecommerce_page_language,'locale',ecommerce_page_locale))) as contexts_com_snowplowanalytics_snowplow_ecommerce_page_1, + TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('cartId',ecommerce_cart_id,'currency',ecommerce_cart_currency,'totalValue',ecommerce_cart_total_value))) as contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1, + contexts_com_snowplowanalytics_snowplow_ecommerce_product_1, + TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('step',checkout_step_number,'accountType',checkout_account_type,'billingFullAddress',checkout_billing_full_address,'billingPostcode',checkout_billing_postcode,'couponCode',checkout_coupon_code,'deliveryMethod',checkout_delivery_method,'deliveryProvider',checkout_delivery_provider,'marketingOptIn',checkout_marketing_opt_in,'paymentMethod',checkout_payment_method,'proofOfPayment',checkout_proof_of_payment,'shippingFullAddress',checkout_shipping_full_address,'shippingPostcode',checkout_shipping_postcode))) as contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1, + TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('transactionId',ecommerce_transaction_id,'currency',ecommerce_currency,'paymentMethod',ecommerce_payment_method,'revenue',ecommerce_revenue,'totalQuantity',ecommerce_total_quantity,'creditOrder',ecommerce_credit_order,'discountAmount',ecommerce_discount_amount,'discountCode',ecommerce_discount_code,'shipping',ecommerce_shipping,'tax',ecommerce_tax))) as contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1, + TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('eventIndex',event_index,'firstEventId',first_event_id,'firstEventTimestamp',first_event_timestamp,'previousSessionId',previous_session_id,'sessionId',session_id,'sessionIndex',session_index,'storageMechanism',storage_mechanism,'userId',mobile_user_id))) as contexts_com_snowplowanalytics_snowplow_client_session_1, + TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id',screen_view_id,'name',screen_view_name))) as contexts_com_snowplowanalytics_mobile_screen_1 -from prep +from flatten diff --git a/macros/bigquery/context_fields.sql b/macros/bigquery/context_fields.sql index d4d6ba6..4321495 100644 --- a/macros/bigquery/context_fields.sql +++ b/macros/bigquery/context_fields.sql @@ -77,7 +77,7 @@ {% macro page_view_fields() %} {% set page_view_fields = [ - {'field': ('page_view__id', 'page_view_id'), 'dtype': 'string'}, + {'field': ('id', 'page_view_id'), 'dtype': 'string'}, ] %} {{ return(page_view_fields) }} @@ -104,8 +104,8 @@ {% macro mobile_screen_view_context_fields() %} {% set mobile_screen_view_context_fields = [ - {'field': 'screen_view_id', 'dtype': 'string'}, - {'field': 'screen_view_name', 'dtype': 'string'}, + {'field': 'id', 'dtype': 'string'}, + {'field': 'name', 'dtype': 'string'}, ] %} {{ return(mobile_screen_view_context_fields) }} diff --git a/macros/field_extractions/get_cart_context_fields.sql b/macros/field_extractions/get_cart_context_fields.sql index dd831a0..7ca7785 100644 --- a/macros/field_extractions/get_cart_context_fields.sql +++ b/macros/field_extractions/get_cart_context_fields.sql @@ -21,7 +21,7 @@ , cast(NULL as {{ type_string() }}) as cart_currency , cast(NULL as decimal(9,2)) as cart_total_value {% else %} - ,{{ snowplow_utils.get_optional_fields( + , {{ snowplow_utils.get_optional_fields( enabled=true, fields=cart_fields(), col_prefix='contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1_', diff --git a/macros/field_extractions/get_page_view_context_fields.sql b/macros/field_extractions/get_page_view_context_fields.sql index b419c8c..64eb423 100644 --- a/macros/field_extractions/get_page_view_context_fields.sql +++ b/macros/field_extractions/get_page_view_context_fields.sql @@ -14,18 +14,45 @@ {% endmacro %} {% macro bigquery__get_page_view_context_fields() %} - ,{{ snowplow_utils.get_optional_fields( - enabled=true, - fields=page_view_fields(), - col_prefix='contexts_com_snowplowanalytics_snowplow_web_page_1_', - relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'), - relation_alias=none) }} + {% if var('snowplow__enable_mobile_events', false) %} + , coalesce( + {{ snowplow_utils.get_optional_fields( + enabled=true, + fields=[{'field': 'id', 'dtype': 'string'}], + col_prefix='contexts_com_snowplowanalytics_mobile_screen_1_', + relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'), + relation_alias=none, + include_field_alias=false) }}, + {{ snowplow_utils.get_optional_fields( + enabled=true, + fields=page_view_fields(), + col_prefix='contexts_com_snowplowanalytics_snowplow_web_page_1_', + relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'), + relation_alias=none, + include_field_alias=false) }} + ) as page_view_id + {% else %} + ,{{ snowplow_utils.get_optional_fields( + enabled=true, + fields=page_view_fields(), + col_prefix='contexts_com_snowplowanalytics_snowplow_web_page_1_', + relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'), + relation_alias=none) }} + {% endif %} {% endmacro %} {% macro spark__get_page_view_context_fields() %} - , contexts_com_snowplowanalytics_snowplow_web_page_1[0].id::string as page_view_id + {% if var('snowplow__enable_mobile_events', false) %} + , coalesce(contexts_com_snowplowanalytics_mobile_screen_1[0].id::string, contexts_com_snowplowanalytics_snowplow_web_page_1[0].id::string) as page_view_id + {% else %} + , contexts_com_snowplowanalytics_snowplow_web_page_1[0].id::string as page_view_id + {% endif %} {% endmacro %} {% macro snowflake__get_page_view_context_fields() %} - , contexts_com_snowplowanalytics_snowplow_web_page_1[0]:id::varchar as page_view_id + {% if var('snowplow__enable_mobile_events', false) %} + , coalesce(contexts_com_snowplowanalytics_mobile_screen_1[0]:id::varchar, contexts_com_snowplowanalytics_snowplow_web_page_1[0]:id::varchar) as page_view_id + {% else %} + , contexts_com_snowplowanalytics_snowplow_web_page_1[0]:id::varchar as page_view_id + {% endif %} {% endmacro %} diff --git a/macros/field_extractions/get_screen_view_context_fields.sql b/macros/field_extractions/get_screen_view_context_fields.sql index 32c4218..890eaf0 100644 --- a/macros/field_extractions/get_screen_view_context_fields.sql +++ b/macros/field_extractions/get_screen_view_context_fields.sql @@ -18,7 +18,7 @@ ,{{ snowplow_utils.get_optional_fields( enabled=true, fields=mobile_screen_view_context_fields(), - col_prefix='contexts_com_snowplowanalytics_snowplow_mobile_screen_1_', + col_prefix='contexts_com_snowplowanalytics_mobile_screen_1_', relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'), relation_alias=none) }} {% else %} @@ -29,8 +29,8 @@ {% macro spark__get_screen_view_context_fields() %} {% if var('snowplow__enable_mobile_events', false) %} - , contexts_com_snowplowanalytics_snowplow_mobile_screen_1[0].screen_view_id::string as screen_view_id - , contexts_com_snowplowanalytics_snowplow_mobile_screen_1[0].screen_view_name::string as screen_view_name + , contexts_com_snowplowanalytics_mobile_screen_1[0].id::string as screen_view_id + , contexts_com_snowplowanalytics_mobile_screen_1[0].name::string as screen_view_name {% else %} , cast(NULL as {{ type_string() }}) as screen_view_id , cast(NULL as {{ type_string() }}) as screen_view_name @@ -39,8 +39,8 @@ {% macro snowflake__get_screen_view_context_fields() %} {% if var('snowplow__enable_mobile_events', false) %} - , contexts_com_snowplowanalytics_snowplow_mobile_screen_1[0]:screenViewId::varchar as screen_view_id - , contexts_com_snowplowanalytics_snowplow_mobile_screen_1[0]:screenViewName::varchar as screen_view_name + , contexts_com_snowplowanalytics_mobile_screen_1[0]:id::varchar as screen_view_id + , contexts_com_snowplowanalytics_mobile_screen_1[0]:name::varchar as screen_view_name {% else %} , cast(NULL as {{ type_string() }}) as screen_view_id , cast(NULL as {{ type_string() }}) as screen_view_name diff --git a/models/base/scratch/snowplow_ecommerce_base_events_this_run.sql b/models/base/scratch/snowplow_ecommerce_base_events_this_run.sql index ff46aac..dddfbeb 100644 --- a/models/base/scratch/snowplow_ecommerce_base_events_this_run.sql +++ b/models/base/scratch/snowplow_ecommerce_base_events_this_run.sql @@ -62,36 +62,30 @@ with base_query as ( {{ base_events_query }} ), prep as ( - - select - {% for col in base_query_cols | map(attribute='name') | list -%} - {% if col == 'session_identifier' -%} - {% if var('snowplow__enable_mobile_events', false) %} - coalesce( - session__session_id, - session_identifier - ) as domain_sessionid - {% else %} + {% if target.type in ['postgres', 'redshift']%} + select + {% for col in base_query_cols | map(attribute='name') | list -%} + {% if col.lower() == 'session_identifier' -%} session_identifier as domain_sessionid - {% endif %} - {%- elif col == 'domain_sessionid' -%} - domain_sessionid as original_domain_sessionid - {%- elif col == 'user_identifier' -%} - {% if var('snowplow__enable_mobile_events', false) %} - coalesce( - session__user_id, - user_identifier - ) as domain_userid - {% else %} + {%- elif col.lower() == 'domain_sessionid' -%} + domain_sessionid as original_domain_sessionid + {%- elif col.lower() == 'user_identifier' -%} user_identifier as domain_userid - {% endif %} - {%- elif col == 'domain_userid' -%} - domain_userid as original_domain_userid - {%- else -%} - {{col}} - {%- endif -%} - {%- if not loop.last -%},{%- endif %} - {% endfor %} + {%- elif col.lower() == 'domain_userid' -%} + domain_userid as original_domain_userid + {%- else -%} + {{col.lower()}} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {% endfor %} + {% else %} + select * {% if target.type in ['databricks', 'bigquery'] %}except{% else %}exclude{% endif %}(session_identifier, domain_sessionid, user_identifier, domain_userid) + + , session_identifier as domain_sessionid + , domain_sessionid as original_domain_sessionid + , user_identifier as domain_userid + , domain_userid as original_domain_userid + {% endif %} from base_query @@ -107,6 +101,10 @@ select * {{ get_session_context_fields() }} {{ get_screen_view_context_fields () }} + {% if target.type not in ['redshift', 'postgres'] -%} + {{ get_action_context_fields() }} + {%- endif %} + , dense_rank() over (partition by domain_sessionid order by derived_tstamp) AS event_in_session_index from prep diff --git a/models/sessions/scratch/snowplow_ecommerce_sessions_this_run.sql b/models/sessions/scratch/snowplow_ecommerce_sessions_this_run.sql index c775e8c..74d11d3 100644 --- a/models/sessions/scratch/snowplow_ecommerce_sessions_this_run.sql +++ b/models/sessions/scratch/snowplow_ecommerce_sessions_this_run.sql @@ -127,7 +127,7 @@ with cart_session_stats AS ( CAST(NULL as {{ type_int() }}) AS total_transacted_products {%- else -%} - select + select domain_sessionid, MIN(derived_tstamp) AS first_transaction_completed,