Skip to content

Commit

Permalink
Test fixes?
Browse files Browse the repository at this point in the history
  • Loading branch information
@emielver
emielver committed Dec 7, 2023
1 parent 96fcea9 commit 4504d27
Show file tree
Hide file tree
Showing 7 changed files with 207 additions and 48 deletions.
138 changes: 136 additions & 2 deletions integration_tests/models/source/snowflake/snowplow_ecommerce_events_stg.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,144 @@ select


from {{ ref('snowplow_ecommerce_events') }}
), flatten as (
select
*,
contexts_com_snowplowanalytics_snowplow_web_page_1[0].id as page_view_id,
contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].id as ecommerce_user_id,
contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].email as ecommerce_user_email,
contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].is_guest as ecommerce_user_is_guest,
contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].type as ecommerce_page_type,
contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].language as ecommerce_page_language,
contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].locale as ecommerce_page_locale,
unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1:type as ecommerce_action_type,
unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1:name as ecommerce_action_name,
contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].cart_id as ecommerce_cart_id,
contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].currency as ecommerce_cart_currency,
contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].total_value as ecommerce_cart_total_value,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].step as checkout_step_number,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].account_type as checkout_account_type,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].billing_full_address as checkout_billing_full_address,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].billing_postcode as checkout_billing_postcode,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].coupon_code as checkout_coupon_code,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].delivery_method as checkout_delivery_method,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].delivery_provider as checkout_delivery_provider,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].marketing_opt_in as checkout_marketing_opt_in,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].payment_method as checkout_payment_method,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].proof_of_payment as checkout_proof_of_payment,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].shipping_full_address as checkout_shipping_full_address,
contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].shipping_postcode as checkout_shipping_postcode,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].transaction_id as ecommerce_transaction_id,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].currency as ecommerce_currency,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].payment_method as ecommerce_payment_method,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].revenue as ecommerce_revenue,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].total_quantity as ecommerce_total_quantity,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].credit_order as ecommerce_credit_order,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].discount_amount as ecommerce_discount_amount,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].discount_code as ecommerce_discount_code,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].shipping as ecommerce_shipping,
contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].tax as ecommerce_tax,
contexts_com_snowplowanalytics_snowplow_client_session_1[0].eventIndex as event_index,
contexts_com_snowplowanalytics_snowplow_client_session_1[0].firstEventId as first_event_id,
contexts_com_snowplowanalytics_snowplow_client_session_1[0].firstEventTimestamp as first_event_timestamp,
contexts_com_snowplowanalytics_snowplow_client_session_1[0].previousSessionId as previous_session_id,
contexts_com_snowplowanalytics_snowplow_client_session_1[0].sessionId as session_id,
contexts_com_snowplowanalytics_snowplow_client_session_1[0].sessionIndex as session_index,
contexts_com_snowplowanalytics_snowplow_client_session_1[0].storageMechanism as storage_mechanism,
contexts_com_snowplowanalytics_snowplow_client_session_1[0].userId as mobile_user_id,
contexts_com_snowplowanalytics_mobile_screen_1[0].id as screen_view_id,
contexts_com_snowplowanalytics_mobile_screen_1[0].name as screen_view_name

from prep
)


select
*
APP_ID,
PLATFORM,
ETL_TSTAMP,
COLLECTOR_TSTAMP,
DVCE_CREATED_TSTAMP,
EVENT_ID,
NAME_TRACKER,
V_TRACKER,
V_COLLECTOR,
V_ETL,
USER_ID,
USER_IPADDRESS,
USER_FINGERPRINT,
DOMAIN_USERID,
DOMAIN_SESSIONIDX,
NETWORK_USERID,
GEO_COUNTRY,
GEO_REGION,
GEO_CITY,
GEO_ZIPCODE,
GEO_LATITUDE,
GEO_LONGITUDE,
GEO_REGION_NAME,
IP_ISP,
IP_ORGANIZATION,
IP_DOMAIN,
IP_NETSPEED,
PAGE_URL,
PAGE_TITLE,
PAGE_REFERRER,
PAGE_URLSCHEME,
PAGE_URLHOST,
PAGE_URLPORT,
PAGE_URLPATH,
PAGE_URLQUERY,
PAGE_URLFRAGMENT,
REFR_URLSCHEME,
REFR_URLHOST,
REFR_URLPORT,
REFR_URLPATH,
REFR_URLQUERY,
REFR_URLFRAGMENT,
REFR_MEDIUM,
REFR_SOURCE,
REFR_TERM,
MKT_MEDIUM,
MKT_SOURCE,
MKT_TERM,
MKT_CONTENT,
MKT_CAMPAIGN,
SE_CATEGORY,
SE_ACTION,
SE_LABEL,
SE_PROPERTY,
SE_VALUE,
USERAGENT,
OS_NAME,
OS_FAMILY,
OS_MANUFACTURER,
OS_TIMEZONE,
BASE_CURRENCY,
GEO_TIMEZONE,
MKT_CLICKID,
MKT_NETWORK,
ETL_TAGS,
DVCE_SENT_TSTAMP,
REFR_DOMAIN_USERID,
REFR_DVCE_TSTAMP,
DOMAIN_SESSIONID,
DERIVED_TSTAMP,
EVENT_VENDOR,
EVENT_NAME,
EVENT_FORMAT,
EVENT_VERSION,
EVENT_FINGERPRINT,
object_construct('type', ecommerce_action_type, 'name', ecommerce_action_name) as unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1,
-- we do this complicated nonsense because Snowflake's `parse_json` function returns a null object if any of the fields inside the JSON are null...
TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id', page_view_id))) as contexts_com_snowplowanalytics_snowplow_web_page_1,
TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id',ecommerce_user_id,'email',ecommerce_user_email,'isGuest',ecommerce_user_is_guest))) as contexts_com_snowplowanalytics_snowplow_ecommerce_user_1,
TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('type',ecommerce_page_type,'language',ecommerce_page_language,'locale',ecommerce_page_locale))) as contexts_com_snowplowanalytics_snowplow_ecommerce_page_1,
TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('cartId',ecommerce_cart_id,'currency',ecommerce_cart_currency,'totalValue',ecommerce_cart_total_value))) as contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1,
contexts_com_snowplowanalytics_snowplow_ecommerce_product_1,
TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('step',checkout_step_number,'accountType',checkout_account_type,'billingFullAddress',checkout_billing_full_address,'billingPostcode',checkout_billing_postcode,'couponCode',checkout_coupon_code,'deliveryMethod',checkout_delivery_method,'deliveryProvider',checkout_delivery_provider,'marketingOptIn',checkout_marketing_opt_in,'paymentMethod',checkout_payment_method,'proofOfPayment',checkout_proof_of_payment,'shippingFullAddress',checkout_shipping_full_address,'shippingPostcode',checkout_shipping_postcode))) as contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1,
TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('transactionId',ecommerce_transaction_id,'currency',ecommerce_currency,'paymentMethod',ecommerce_payment_method,'revenue',ecommerce_revenue,'totalQuantity',ecommerce_total_quantity,'creditOrder',ecommerce_credit_order,'discountAmount',ecommerce_discount_amount,'discountCode',ecommerce_discount_code,'shipping',ecommerce_shipping,'tax',ecommerce_tax))) as contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1,
TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('eventIndex',event_index,'firstEventId',first_event_id,'firstEventTimestamp',first_event_timestamp,'previousSessionId',previous_session_id,'sessionId',session_id,'sessionIndex',session_index,'storageMechanism',storage_mechanism,'userId',mobile_user_id))) as contexts_com_snowplowanalytics_snowplow_client_session_1,
TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id',screen_view_id,'name',screen_view_name))) as contexts_com_snowplowanalytics_mobile_screen_1

from prep
from flatten
6 changes: 3 additions & 3 deletions macros/bigquery/context_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
{% macro page_view_fields() %}

{% set page_view_fields = [
{'field': ('page_view__id', 'page_view_id'), 'dtype': 'string'},
{'field': ('id', 'page_view_id'), 'dtype': 'string'},
] %}

{{ return(page_view_fields) }}
Expand All @@ -104,8 +104,8 @@
{% macro mobile_screen_view_context_fields() %}

{% set mobile_screen_view_context_fields = [
{'field': 'screen_view_id', 'dtype': 'string'},
{'field': 'screen_view_name', 'dtype': 'string'},
{'field': 'id', 'dtype': 'string'},
{'field': 'name', 'dtype': 'string'},
] %}

{{ return(mobile_screen_view_context_fields) }}
Expand Down
2 changes: 1 addition & 1 deletion macros/field_extractions/get_cart_context_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
, cast(NULL as {{ type_string() }}) as cart_currency
, cast(NULL as decimal(9,2)) as cart_total_value
{% else %}
,{{ snowplow_utils.get_optional_fields(
, {{ snowplow_utils.get_optional_fields(
enabled=true,
fields=cart_fields(),
col_prefix='contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1_',
Expand Down
43 changes: 35 additions & 8 deletions macros/field_extractions/get_page_view_context_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,45 @@
{% endmacro %}

{% macro bigquery__get_page_view_context_fields() %}
,{{ snowplow_utils.get_optional_fields(
enabled=true,
fields=page_view_fields(),
col_prefix='contexts_com_snowplowanalytics_snowplow_web_page_1_',
relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'),
relation_alias=none) }}
{% if var('snowplow__enable_mobile_events', false) %}
, coalesce(
{{ snowplow_utils.get_optional_fields(
enabled=true,
fields=[{'field': 'id', 'dtype': 'string'}],
col_prefix='contexts_com_snowplowanalytics_mobile_screen_1_',
relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'),
relation_alias=none,
include_field_alias=false) }},
{{ snowplow_utils.get_optional_fields(
enabled=true,
fields=page_view_fields(),
col_prefix='contexts_com_snowplowanalytics_snowplow_web_page_1_',
relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'),
relation_alias=none,
include_field_alias=false) }}
) as page_view_id
{% else %}
,{{ snowplow_utils.get_optional_fields(
enabled=true,
fields=page_view_fields(),
col_prefix='contexts_com_snowplowanalytics_snowplow_web_page_1_',
relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'),
relation_alias=none) }}
{% endif %}
{% endmacro %}

{% macro spark__get_page_view_context_fields() %}
, contexts_com_snowplowanalytics_snowplow_web_page_1[0].id::string as page_view_id
{% if var('snowplow__enable_mobile_events', false) %}
, coalesce(contexts_com_snowplowanalytics_mobile_screen_1[0].id::string, contexts_com_snowplowanalytics_snowplow_web_page_1[0].id::string) as page_view_id
{% else %}
, contexts_com_snowplowanalytics_snowplow_web_page_1[0].id::string as page_view_id
{% endif %}
{% endmacro %}

{% macro snowflake__get_page_view_context_fields() %}
, contexts_com_snowplowanalytics_snowplow_web_page_1[0]:id::varchar as page_view_id
{% if var('snowplow__enable_mobile_events', false) %}
, coalesce(contexts_com_snowplowanalytics_mobile_screen_1[0]:id::varchar, contexts_com_snowplowanalytics_snowplow_web_page_1[0]:id::varchar) as page_view_id
{% else %}
, contexts_com_snowplowanalytics_snowplow_web_page_1[0]:id::varchar as page_view_id
{% endif %}
{% endmacro %}
10 changes: 5 additions & 5 deletions macros/field_extractions/get_screen_view_context_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
,{{ snowplow_utils.get_optional_fields(
enabled=true,
fields=mobile_screen_view_context_fields(),
col_prefix='contexts_com_snowplowanalytics_snowplow_mobile_screen_1_',
col_prefix='contexts_com_snowplowanalytics_mobile_screen_1_',
relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'),
relation_alias=none) }}
{% else %}
Expand All @@ -29,8 +29,8 @@

{% macro spark__get_screen_view_context_fields() %}
{% if var('snowplow__enable_mobile_events', false) %}
, contexts_com_snowplowanalytics_snowplow_mobile_screen_1[0].screen_view_id::string as screen_view_id
, contexts_com_snowplowanalytics_snowplow_mobile_screen_1[0].screen_view_name::string as screen_view_name
, contexts_com_snowplowanalytics_mobile_screen_1[0].id::string as screen_view_id
, contexts_com_snowplowanalytics_mobile_screen_1[0].name::string as screen_view_name
{% else %}
, cast(NULL as {{ type_string() }}) as screen_view_id
, cast(NULL as {{ type_string() }}) as screen_view_name
Expand All @@ -39,8 +39,8 @@

{% macro snowflake__get_screen_view_context_fields() %}
{% if var('snowplow__enable_mobile_events', false) %}
, contexts_com_snowplowanalytics_snowplow_mobile_screen_1[0]:screenViewId::varchar as screen_view_id
, contexts_com_snowplowanalytics_snowplow_mobile_screen_1[0]:screenViewName::varchar as screen_view_name
, contexts_com_snowplowanalytics_mobile_screen_1[0]:id::varchar as screen_view_id
, contexts_com_snowplowanalytics_mobile_screen_1[0]:name::varchar as screen_view_name
{% else %}
, cast(NULL as {{ type_string() }}) as screen_view_id
, cast(NULL as {{ type_string() }}) as screen_view_name
Expand Down
54 changes: 26 additions & 28 deletions models/base/scratch/snowplow_ecommerce_base_events_this_run.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,36 +62,30 @@
with base_query as (
{{ base_events_query }}
), prep as (

select
{% for col in base_query_cols | map(attribute='name') | list -%}
{% if col == 'session_identifier' -%}
{% if var('snowplow__enable_mobile_events', false) %}
coalesce(
session__session_id,
session_identifier
) as domain_sessionid
{% else %}
{% if target.type in ['postgres', 'redshift']%}
select
{% for col in base_query_cols | map(attribute='name') | list -%}
{% if col.lower() == 'session_identifier' -%}
session_identifier as domain_sessionid
{% endif %}
{%- elif col == 'domain_sessionid' -%}
domain_sessionid as original_domain_sessionid
{%- elif col == 'user_identifier' -%}
{% if var('snowplow__enable_mobile_events', false) %}
coalesce(
session__user_id,
user_identifier
) as domain_userid
{% else %}
{%- elif col.lower() == 'domain_sessionid' -%}
domain_sessionid as original_domain_sessionid
{%- elif col.lower() == 'user_identifier' -%}
user_identifier as domain_userid
{% endif %}
{%- elif col == 'domain_userid' -%}
domain_userid as original_domain_userid
{%- else -%}
{{col}}
{%- endif -%}
{%- if not loop.last -%},{%- endif %}
{% endfor %}
{%- elif col.lower() == 'domain_userid' -%}
domain_userid as original_domain_userid
{%- else -%}
{{col.lower()}}
{%- endif -%}
{%- if not loop.last -%},{%- endif %}
{% endfor %}
{% else %}
select * {% if target.type in ['databricks', 'bigquery'] %}except{% else %}exclude{% endif %}(session_identifier, domain_sessionid, user_identifier, domain_userid)

, session_identifier as domain_sessionid
, domain_sessionid as original_domain_sessionid
, user_identifier as domain_userid
, domain_userid as original_domain_userid
{% endif %}

from base_query

Expand All @@ -107,6 +101,10 @@ select *
{{ get_session_context_fields() }}
{{ get_screen_view_context_fields () }}

{% if target.type not in ['redshift', 'postgres'] -%}
{{ get_action_context_fields() }}
{%- endif %}

, dense_rank() over (partition by domain_sessionid order by derived_tstamp) AS event_in_session_index

from prep
2 changes: 1 addition & 1 deletion models/sessions/scratch/snowplow_ecommerce_sessions_this_run.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ with cart_session_stats AS (
CAST(NULL as {{ type_int() }}) AS total_transacted_products

{%- else -%}
select
select
domain_sessionid,

MIN(derived_tstamp) AS first_transaction_completed,
Expand Down

0 comments on commit 4504d27

Please sign in to comment.