diff --git a/.test/great_expectations/expectations/web/v1/base_redshift.json b/.test/great_expectations/expectations/web/v1/base_redshift.json index 9e87a41d..a0502c40 100644 --- a/.test/great_expectations/expectations/web/v1/base_redshift.json +++ b/.test/great_expectations/expectations/web/v1/base_redshift.json @@ -286,7 +286,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0" + "redshift_model_version": "1.3.0" }, "great_expectations.__version__": "0.12.0" } diff --git a/.test/great_expectations/expectations/web/v1/metadata.json b/.test/great_expectations/expectations/web/v1/metadata.json index 4ee04917..a99b70ef 100644 --- a/.test/great_expectations/expectations/web/v1/metadata.json +++ b/.test/great_expectations/expectations/web/v1/metadata.json @@ -103,7 +103,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json b/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json index 640f9cba..0e387c02 100644 --- a/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json +++ b/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json @@ -27,7 +27,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/.test/great_expectations/expectations/web/v1/page_views.json b/.test/great_expectations/expectations/web/v1/page_views.json index a2580d77..ae30c0c7 100644 --- a/.test/great_expectations/expectations/web/v1/page_views.json +++ b/.test/great_expectations/expectations/web/v1/page_views.json @@ -225,7 +225,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/.test/great_expectations/expectations/web/v1/sessions.json b/.test/great_expectations/expectations/web/v1/sessions.json index c2c01d00..d652caa5 100644 --- a/.test/great_expectations/expectations/web/v1/sessions.json +++ b/.test/great_expectations/expectations/web/v1/sessions.json @@ -181,7 +181,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/.test/great_expectations/expectations/web/v1/users.json b/.test/great_expectations/expectations/web/v1/users.json index d949d4df..ee6051a1 100644 --- a/.test/great_expectations/expectations/web/v1/users.json +++ b/.test/great_expectations/expectations/web/v1/users.json @@ -117,7 +117,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/CHANGELOG b/CHANGELOG index d09facad..907d8d18 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,42 +1,47 @@ +Redshift Web Version 1.3.0 (2022-06-06) +-------------------------- +Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!) +Redshift Web: Fix column lengths in manifest tables (#131) + Snowflake Web Version 1.0.1 (2022-02-24) --------------------------------------- -Snowflake Web: Update column check stored procedure (Close #125) -Snowflake Web: Fix varchar length for pseudonymized fields (Close #122) -Snowflake Web: Remove start_date variable from users module (Close #123) -Snowflake Web: Update copyright notices (Close #124) -Snowflake Web: Fix logic in users_sessions_this_run to account for sparse data (Close #120) -Snowflake Web: Fix varchar length for yauaa columns (Close #97) -Snowflake Web: Fix se_label column length in events_staged (Close #109) +Snowflake Web: Update column check stored procedure (#125) +Snowflake Web: Fix varchar length for pseudonymized fields (#122) +Snowflake Web: Remove start_date variable from users module (#123) +Snowflake Web: Update copyright notices (#124) +Snowflake Web: Fix logic in users_sessions_this_run to account for sparse data (#120) +Snowflake Web: Fix varchar length for yauaa columns (#97) +Snowflake Web: Fix se_label column length in events_staged (#109) Snowflake Mobile Version 1.1.0 (2021-06-16) --------------------------------------- -Snowflake Mobile: Update update_manifest to throw error message (Close #104) -Snowflake Mobile: Update commit_table procedure to throw error messages (Close #103) -Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #102) +Snowflake Mobile: Update update_manifest to throw error message (#104) +Snowflake Mobile: Update commit_table procedure to throw error messages (#103) +Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (#102) BigQuery Mobile Version 1.1.0 (2021-06-15) --------------------------------------- -BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (Close #100) -BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #99) +BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (#100) +BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (#99) Redshift Mobile Version 1.1.0 (2021-05-17) --------------------------------------- -Redshift Mobile: Increase session_id character limit in manifest (Close #93) -Redshift Mobile: Add simplified configs (Close #89) -Redshift Mobile: Set app errors module to disabled by default (Close #88) -Redshift Mobile: Add model_tstamp to derived tables (Close #82) -Redshift Mobile: Move app errors columns to end of table (Close #83) -Redshift Mobile: Remove CTE from user-aggs (Close #80) +Redshift Mobile: Increase session_id character limit in manifest (#93) +Redshift Mobile: Add simplified configs (#89) +Redshift Mobile: Set app errors module to disabled by default (#88) +Redshift Mobile: Add model_tstamp to derived tables (#82) +Redshift Mobile: Move app errors columns to end of table (#83) +Redshift Mobile: Remove CTE from user-aggs (#80) Snowflake Mobile Version 1.0.0 (2021-05-06) --------------------------------------- -Add Snowflake mobile model v1 (Close #85) +Add Snowflake mobile model v1 (#85) BigQuery Mobile Version 1.0.0 (2021-04-26) --------------------------------------- -Update licence (Close #79) -Fix credential loading in scripts (close #72) -Mobile: Add BigQuery mobile model v1 (close #73) +Update licence (#79) +Fix credential loading in scripts (#72) +Mobile: Add BigQuery mobile model v1 (#73) Redshift Mobile Version 1.0.0 (2021-03-25) --------------------------------------- @@ -51,8 +56,8 @@ Update Architecture diagram to match snowplow/snowplow (#62) BigQuery Web Version 1.0.2 (2021-03-11) --------------------------------------- -BigQuery: Handle schema evolution in core contexts (close #52) -Update SQL Runner download link to point to Github releases (close #51) +BigQuery: Handle schema evolution in core contexts (#52) +Update SQL Runner download link to point to Github releases (#51) BigQuery Web Version 1.0.1 (2021-02-24) --------------------------------------- diff --git a/web/v1/redshift/CHANGELOG b/web/v1/redshift/CHANGELOG index 0b8a02b3..a77401e8 100644 --- a/web/v1/redshift/CHANGELOG +++ b/web/v1/redshift/CHANGELOG @@ -1,3 +1,8 @@ +Version 1.3.0 (2022-06-06) +-------------------------- +Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!) +Redshift Web: Fix column lengths in manifest tables (#131) + Version 1.2.0 (2021-01-25) --------------------------- Redshift: Add config (#12) diff --git a/web/v1/redshift/README.md b/web/v1/redshift/README.md index 49eccb0a..067bd43d 100644 --- a/web/v1/redshift/README.md +++ b/web/v1/redshift/README.md @@ -167,7 +167,7 @@ Detail on configuring the users module's playbook can be found [in the relevant ## A note on duplicates -This version of the model (1.2.0) excludes duplicated event_ids and page_view_ids. Ideally in the future it will provide standard options for handling them, however because customisations of this model involve queries on federated atomic tables, the safest option for a first version is to exclude them. +This version of the model (>1.1.0) excludes duplicated event_ids and page_view_ids. Ideally in the future it will provide standard options for handling them, however because customisations of this model involve queries on federated atomic tables, the safest option for a first version is to exclude them. If there is a need to handle duplicates, this can be done by adding a custom module to the base level of aggregation - take good care to manage the possibility of introducing duplicates downstream if doing so. diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl index 04cf575d..aade872c 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl index 60b94254..eb50423c 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl index 2525b7b5..9a2720e7 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl index 2d799348..fd7540a2 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :input_schema: atomic :scratch_schema: scratch :output_schema: derived diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl index 92577c44..289c52fb 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl index 80e4e1e2..7e51bf02 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl index 45e34289..3a72ab6c 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :input_schema: atomic :scratch_schema: scratch :output_schema: derived diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl index 757ac59a..a2b38a21 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl index e68bdf9a..68a82468 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl index 5318448e..57524ea9 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl index 65570dff..63349656 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl index 35173ddc..5dd78f3c 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl index b425ad2e..d5dc5b2b 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl index 52887aac..afb897c1 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl index e15868b2..efb4139d 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: ""