Add base macro functionality

dbt_project.yml

@@ -50,6 +50,9 @@ vars:
     snowplow__context_mobile_session: 'com_snowplowanalytics_snowplow_client_session_1'
     snowplow__context_screen: 'com_snowplowanalytics_mobile_screen_1'
 
+    snowplow__session_identifiers: []
+    snowplow__user_identifiers: []
+
     # Variables - Standard Config
     snowplow__start_date: '2020-01-01'
     snowplow__backfill_limit_days: 30
@@ -80,7 +83,8 @@ on-run-start:
 
 # Update manifest table with last event consumed per sucessfully executed node/model
 on-run-end:
-  - "{{ snowplow_utils.snowplow_incremental_post_hook('snowplow_ecommerce') }}"
+  - "{{ snowplow_utils.snowplow_incremental_post_hook(package_name='snowplow_ecommerce', incremental_manifest_table_name='snowplow_ecommerce_incremental_manifest', base_events_this_run_table_name='snowplow_ecommerce_base_events_this_run', session_timestamp=var('snowplow__session_timestamp')) }}"
+
 
 
 # Tag 'snowplow_ecommerce_incremental' allows snowplow_incremental_post_hook to identify Snowplow models
@@ -91,25 +95,9 @@ models:
     base:
       manifest:
         +schema: "snowplow_manifest"
-        bigquery:
-          +enabled: "{{ target.type == 'bigquery' | as_bool() }}"
-        databricks:
-          +enabled: "{{ target.type in ['databricks', 'spark'] | as_bool() }}"
-        default:
-          +enabled: "{{ target.type in ['redshift', 'postgres'] | as_bool() }}"
-        snowflake:
-          +enabled: "{{ target.type == 'snowflake' | as_bool() }}"
       scratch:
         +schema: "scratch"
         +tags: "scratch"
-        bigquery:
-          +enabled: "{{ target.type == 'bigquery' | as_bool() }}"
-        databricks:
-          +enabled: "{{ target.type in ['databricks', 'spark'] | as_bool() }}"
-        default:
-          +enabled: "{{ target.type in ['redshift', 'postgres'] | as_bool() }}"
-        snowflake:
-          +enabled: "{{ target.type == 'snowflake' | as_bool() }}"
     carts:
       +schema: "derived"
       +tags: "snowplow_ecommerce_incremental"

docs/markdown/snowplow_ecommerce_common_cols.md

@@ -77,6 +77,10 @@ A user fingerprint generated by looking at the individual browser features e.g.
 User ID set by Snowplow using 1st party cookie e.g. ‘bc2e92ec6c204a14’
 {% enddocs %}
 
+{% docs col_user_identifier %}
+The user identifier as defined in your project variables. Default to `domain_userid`.
+{% enddocs %}
+
 
 {% docs col_domain_sessionidx %}
 A visit / session index e.g. 3
@@ -599,7 +603,11 @@ The time of attaching the domain_userid to the inbound link e.g. ‘2013-11-26 0
 
 
 {% docs col_domain_sessionid %}
-A visit / session UUID e.g. ‘c6ef3124-b53a-4b13-a233-0088f79dcbcb’
+Session identifier specified in your project variables. By default this is the true `domain_sessionid` i.e. a visit / session UUID e.g. ‘c6ef3124-b53a-4b13-a233-0088f79dcbcb’
+{% enddocs %}
+
+{% docs col_session_identifier %}
+The session identifier as defined in your project variables. Default to `domain_sessionid`.
 {% enddocs %}
 
 

integration_tests/dbt_project.yml

@@ -41,10 +41,12 @@ models:
 
 vars:
   snowplow_ecommerce:
+    snowplow__atomic_schema: "{{target.schema ~ '_snplw_ecommerce_int_tests'}}"
     snowplow__events: "{{ ref('snowplow_ecommerce_events_stg') }}"
+    snowplow__events_table: "snowplow_ecommerce_events_stg"
+    snowplow__session_context: 'com_snowplowanalytics_snowplow_client_session_1'
     snowplow__start_date: "2022-10-01"
     snowplow__backfill_limit_days: 30
-    snowplow__atomic_schema: "{{ target.schema }}_snplw_ecommerce_int_tests"
     snowplow__sde_ecommerce_action: 'com_snowplowanalytics_snowplow_ecommerce_action_1'
     snowplow__enable_mobile_events: true
 

integration_tests/models/source/snowflake/snowplow_ecommerce_events_stg.sql

@@ -15,10 +15,144 @@ select
 
 
 from {{ ref('snowplow_ecommerce_events') }}
+), flatten as (
+  select
+    *,
+    contexts_com_snowplowanalytics_snowplow_web_page_1[0].id as page_view_id,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].id as ecommerce_user_id,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].email as ecommerce_user_email,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_user_1[0].is_guest as ecommerce_user_is_guest,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].type as ecommerce_page_type,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].language as ecommerce_page_language,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_page_1[0].locale as ecommerce_page_locale,
+    unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1:type as ecommerce_action_type,
+    unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1:name as ecommerce_action_name,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].cart_id as ecommerce_cart_id,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].currency as ecommerce_cart_currency,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].total_value as ecommerce_cart_total_value,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].step as checkout_step_number,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].account_type as checkout_account_type,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].billing_full_address as checkout_billing_full_address,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].billing_postcode as checkout_billing_postcode,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].coupon_code as checkout_coupon_code,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].delivery_method as checkout_delivery_method,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].delivery_provider as checkout_delivery_provider,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].marketing_opt_in as checkout_marketing_opt_in,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].payment_method as checkout_payment_method,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].proof_of_payment as checkout_proof_of_payment,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].shipping_full_address as checkout_shipping_full_address,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1[0].shipping_postcode as checkout_shipping_postcode,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].transaction_id as ecommerce_transaction_id,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].currency as ecommerce_currency,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].payment_method as ecommerce_payment_method,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].revenue as ecommerce_revenue,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].total_quantity as ecommerce_total_quantity,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].credit_order as ecommerce_credit_order,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].discount_amount as ecommerce_discount_amount,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].discount_code as ecommerce_discount_code,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].shipping as ecommerce_shipping,
+    contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1[0].tax as ecommerce_tax,
+    contexts_com_snowplowanalytics_snowplow_client_session_1[0].eventIndex as event_index,
+    contexts_com_snowplowanalytics_snowplow_client_session_1[0].firstEventId as first_event_id,
+    contexts_com_snowplowanalytics_snowplow_client_session_1[0].firstEventTimestamp as first_event_timestamp,
+    contexts_com_snowplowanalytics_snowplow_client_session_1[0].previousSessionId as previous_session_id,
+    contexts_com_snowplowanalytics_snowplow_client_session_1[0].sessionId as session_id,
+    contexts_com_snowplowanalytics_snowplow_client_session_1[0].sessionIndex as session_index,
+    contexts_com_snowplowanalytics_snowplow_client_session_1[0].storageMechanism as storage_mechanism,
+    contexts_com_snowplowanalytics_snowplow_client_session_1[0].userId as mobile_user_id,
+    contexts_com_snowplowanalytics_mobile_screen_1[0].id as screen_view_id,
+    contexts_com_snowplowanalytics_mobile_screen_1[0].name as screen_view_name
+
+  from prep
 )
 
 
 select
-  *
+  APP_ID,
+  PLATFORM,
+  ETL_TSTAMP,
+  COLLECTOR_TSTAMP,
+  DVCE_CREATED_TSTAMP,
+  EVENT_ID,
+  NAME_TRACKER,
+  V_TRACKER,
+  V_COLLECTOR,
+  V_ETL,
+  USER_ID,
+  USER_IPADDRESS,
+  USER_FINGERPRINT,
+  DOMAIN_USERID,
+  DOMAIN_SESSIONIDX,
+  NETWORK_USERID,
+  GEO_COUNTRY,
+  GEO_REGION,
+  GEO_CITY,
+  GEO_ZIPCODE,
+  GEO_LATITUDE,
+  GEO_LONGITUDE,
+  GEO_REGION_NAME,
+  IP_ISP,
+  IP_ORGANIZATION,
+  IP_DOMAIN,
+  IP_NETSPEED,
+  PAGE_URL,
+  PAGE_TITLE,
+  PAGE_REFERRER,
+  PAGE_URLSCHEME,
+  PAGE_URLHOST,
+  PAGE_URLPORT,
+  PAGE_URLPATH,
+  PAGE_URLQUERY,
+  PAGE_URLFRAGMENT,
+  REFR_URLSCHEME,
+  REFR_URLHOST,
+  REFR_URLPORT,
+  REFR_URLPATH,
+  REFR_URLQUERY,
+  REFR_URLFRAGMENT,
+  REFR_MEDIUM,
+  REFR_SOURCE,
+  REFR_TERM,
+  MKT_MEDIUM,
+  MKT_SOURCE,
+  MKT_TERM,
+  MKT_CONTENT,
+  MKT_CAMPAIGN,
+  SE_CATEGORY,
+  SE_ACTION,
+  SE_LABEL,
+  SE_PROPERTY,
+  SE_VALUE,
+  USERAGENT,
+  OS_NAME,
+  OS_FAMILY,
+  OS_MANUFACTURER,
+  OS_TIMEZONE,
+  BASE_CURRENCY,
+  GEO_TIMEZONE,
+  MKT_CLICKID,
+  MKT_NETWORK,
+  ETL_TAGS,
+  DVCE_SENT_TSTAMP,
+  REFR_DOMAIN_USERID,
+  REFR_DVCE_TSTAMP,
+  DOMAIN_SESSIONID,
+  DERIVED_TSTAMP,
+  EVENT_VENDOR,
+  EVENT_NAME,
+  EVENT_FORMAT,
+  EVENT_VERSION,
+  EVENT_FINGERPRINT,
+  object_construct('type', ecommerce_action_type, 'name', ecommerce_action_name) as unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1,
+  -- we do this complicated nonsense because Snowflake's `parse_json` function returns a null object if any of the fields inside the JSON are null...
+  TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id', page_view_id))) as contexts_com_snowplowanalytics_snowplow_web_page_1,
+  TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id',ecommerce_user_id,'email',ecommerce_user_email,'isGuest',ecommerce_user_is_guest))) as contexts_com_snowplowanalytics_snowplow_ecommerce_user_1,
+  TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('type',ecommerce_page_type,'language',ecommerce_page_language,'locale',ecommerce_page_locale))) as contexts_com_snowplowanalytics_snowplow_ecommerce_page_1,
+  TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('cartId',ecommerce_cart_id,'currency',ecommerce_cart_currency,'totalValue',ecommerce_cart_total_value))) as contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1,
+  contexts_com_snowplowanalytics_snowplow_ecommerce_product_1,
+  TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('step',checkout_step_number,'accountType',checkout_account_type,'billingFullAddress',checkout_billing_full_address,'billingPostcode',checkout_billing_postcode,'couponCode',checkout_coupon_code,'deliveryMethod',checkout_delivery_method,'deliveryProvider',checkout_delivery_provider,'marketingOptIn',checkout_marketing_opt_in,'paymentMethod',checkout_payment_method,'proofOfPayment',checkout_proof_of_payment,'shippingFullAddress',checkout_shipping_full_address,'shippingPostcode',checkout_shipping_postcode))) as contexts_com_snowplowanalytics_snowplow_ecommerce_checkout_step_1,
+  TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('transactionId',ecommerce_transaction_id,'currency',ecommerce_currency,'paymentMethod',ecommerce_payment_method,'revenue',ecommerce_revenue,'totalQuantity',ecommerce_total_quantity,'creditOrder',ecommerce_credit_order,'discountAmount',ecommerce_discount_amount,'discountCode',ecommerce_discount_code,'shipping',ecommerce_shipping,'tax',ecommerce_tax))) as contexts_com_snowplowanalytics_snowplow_ecommerce_transaction_1,
+  TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('eventIndex',event_index,'firstEventId',first_event_id,'firstEventTimestamp',first_event_timestamp,'previousSessionId',previous_session_id,'sessionId',session_id,'sessionIndex',session_index,'storageMechanism',storage_mechanism,'userId',mobile_user_id))) as contexts_com_snowplowanalytics_snowplow_client_session_1,
+  TO_VARIANT(ARRAY_CONSTRUCT(OBJECT_CONSTRUCT_KEEP_NULL('id',screen_view_id,'name',screen_view_name))) as contexts_com_snowplowanalytics_mobile_screen_1
 
-from prep
+from flatten

macros/bigquery/context_fields.sql

@@ -73,3 +73,41 @@
   {{ return(cart_fields) }}
 
 {% endmacro %}
+
+{% macro page_view_fields() %}
+
+  {% set page_view_fields = [
+      {'field': ('id', 'page_view_id'), 'dtype': 'string'},
+    ] %}
+
+  {{ return(page_view_fields) }}
+
+{% endmacro %}
+
+{% macro mobile_session_context_fields()  %}
+
+    {% set mobile_session_context_fields = [
+      {'field': 'event_index', 'dtype': 'numeric'},
+      {'field': 'first_event_id', 'dtype': 'string'},
+      {'field': 'first_event_timestamp', 'dtype': 'timestamp'},
+      {'field': 'previous_session_id', 'dtype': 'string'},
+      {'field': 'session_id', 'dtype': 'string'},
+      {'field': 'session_index', 'dtype': 'numeric'},
+      {'field': 'storage_mechanism', 'dtype': 'string'},
+      {'field': ('user_id', 'mobile_user_id'), 'dtype': 'string'},
+    ] %}
+
+  {{ return(mobile_session_context_fields) }}
+
+{% endmacro %}
+
+{% macro mobile_screen_view_context_fields()  %}
+
+    {% set mobile_screen_view_context_fields = [
+      {'field': 'id', 'dtype': 'string'},
+      {'field': 'name', 'dtype': 'string'},
+    ] %}
+
+  {{ return(mobile_screen_view_context_fields) }}
+
+{% endmacro %}

macros/field_extractions/get_action_context_fields.sql

@@ -0,0 +1,27 @@
+{% macro get_action_context_fields() %}
+    {{ return(adapter.dispatch('get_action_context_fields', 'snowplow_ecommerce')()) }}
+{%- endmacro -%}
+
+{% macro postgres__get_action_context_fields() %}
+    , ecommerce_action_type
+    , ecommerce_action_name
+{% endmacro %}
+
+{% macro bigquery__get_action_context_fields() %}
+    ,{{ snowplow_utils.get_optional_fields(
+        enabled=true,
+        fields=tracking_action_fields(),
+        col_prefix='unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1_',
+        relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'),
+        relation_alias=none) }}
+{% endmacro %}
+
+{% macro spark__get_action_context_fields() %}
+    , unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1.type::string as ecommerce_action_type
+    , unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1.name::string as ecommerce_action_name
+{% endmacro %}
+
+{% macro snowflake__get_action_context_fields() %}
+    , unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1:type::varchar as ecommerce_action_type
+    , unstruct_event_com_snowplowanalytics_snowplow_ecommerce_snowplow_ecommerce_action_1:name::varchar as ecommerce_action_name
+{% endmacro %}

macros/field_extractions/get_cart_context_fields.sql

@@ -0,0 +1,55 @@
+{% macro get_cart_context_fields() %}
+    {{ return(adapter.dispatch('get_cart_context_fields', 'snowplow_ecommerce')()) }}
+{%- endmacro -%}
+
+{% macro postgres__get_cart_context_fields() %}
+    {% if var('snowplow__disable_ecommerce_carts', false) %}
+        , cast(NULL as {{ type_string() }}) as cart_id
+        , cast(NULL as {{ type_string() }}) as cart_currency
+        , cast(NULL as decimal(9,2)) as cart_total_value
+    {% else %}
+        , cart__cart_id as cart_id
+        , cart__currency as cart_currency
+        , cart__total_value as cart_total_value
+    {% endif %}
+{% endmacro %}
+
+{% macro bigquery__get_cart_context_fields() %}
+
+    {% if var('snowplow__disable_ecommerce_carts', false) %}
+        , cast(NULL as {{ type_string() }}) as cart_id
+        , cast(NULL as {{ type_string() }}) as cart_currency
+        , cast(NULL as decimal(9,2)) as cart_total_value
+    {% else %}
+        , {{ snowplow_utils.get_optional_fields(
+            enabled=true,
+            fields=cart_fields(),
+            col_prefix='contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1_',
+            relation=source('atomic', 'events') if project_name != 'snowplow_ecommerce_integration_tests' else ref('snowplow_ecommerce_events_stg'),
+            relation_alias=none) }}
+    {% endif %}
+{% endmacro %}
+
+{% macro spark__get_cart_context_fields() %}
+    {% if var('snowplow__disable_ecommerce_carts', false) %}
+        , cast(NULL as {{ type_string() }}) as cart_id
+        , cast(NULL as {{ type_string() }}) as cart_currency
+        , cast(NULL as decimal(9,2)) as cart_total_value
+    {% else %}
+        , contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].cart_id::string as cart_id
+        , contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].currency::string as cart_currency
+        , contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0].total_value::decimal(9,2) as cart_total_value
+    {% endif %}
+{% endmacro %}
+
+{% macro snowflake__get_cart_context_fields() %}
+    {% if var('snowplow__disable_ecommerce_carts', false) %}
+        , cast(NULL as {{ type_string() }}) as cart_id
+        , cast(NULL as {{ type_string() }}) as cart_currency
+        , cast(NULL as decimal(9,2)) as cart_total_value
+    {% else %}
+        , contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0]:cartId::varchar as cart_id
+        , contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0]:currency::varchar as cart_currency
+        , contexts_com_snowplowanalytics_snowplow_ecommerce_cart_1[0]:totalValue::decimal(9,2) as cart_total_value
+    {% endif %}
+{% endmacro %}