diff --git a/.test/great_expectations/expectations/web/v1/base_redshift.json b/.test/great_expectations/expectations/web/v1/base_redshift.json index 9e87a41d..a0502c40 100644 --- a/.test/great_expectations/expectations/web/v1/base_redshift.json +++ b/.test/great_expectations/expectations/web/v1/base_redshift.json @@ -286,7 +286,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0" + "redshift_model_version": "1.3.0" }, "great_expectations.__version__": "0.12.0" } diff --git a/.test/great_expectations/expectations/web/v1/metadata.json b/.test/great_expectations/expectations/web/v1/metadata.json index 4ee04917..a99b70ef 100644 --- a/.test/great_expectations/expectations/web/v1/metadata.json +++ b/.test/great_expectations/expectations/web/v1/metadata.json @@ -103,7 +103,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json b/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json index 640f9cba..0e387c02 100644 --- a/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json +++ b/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json @@ -27,7 +27,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/.test/great_expectations/expectations/web/v1/page_views.json b/.test/great_expectations/expectations/web/v1/page_views.json index a2580d77..ae30c0c7 100644 --- a/.test/great_expectations/expectations/web/v1/page_views.json +++ b/.test/great_expectations/expectations/web/v1/page_views.json @@ -225,7 +225,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/.test/great_expectations/expectations/web/v1/sessions.json b/.test/great_expectations/expectations/web/v1/sessions.json index c2c01d00..d652caa5 100644 --- a/.test/great_expectations/expectations/web/v1/sessions.json +++ b/.test/great_expectations/expectations/web/v1/sessions.json @@ -181,7 +181,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/.test/great_expectations/expectations/web/v1/users.json b/.test/great_expectations/expectations/web/v1/users.json index d949d4df..ee6051a1 100644 --- a/.test/great_expectations/expectations/web/v1/users.json +++ b/.test/great_expectations/expectations/web/v1/users.json @@ -117,7 +117,7 @@ "meta": { "versions": { "test_suite_version": "1.1.1", - "redshift_model_version": "1.2.0", + "redshift_model_version": "1.3.0", "bigquery_model_version": "1.0.3", "snowflake_model_version": "1.0.1" }, diff --git a/CHANGELOG b/CHANGELOG index d09facad..907d8d18 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,42 +1,47 @@ +Redshift Web Version 1.3.0 (2022-06-06) +-------------------------- +Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!) +Redshift Web: Fix column lengths in manifest tables (#131) + Snowflake Web Version 1.0.1 (2022-02-24) --------------------------------------- -Snowflake Web: Update column check stored procedure (Close #125) -Snowflake Web: Fix varchar length for pseudonymized fields (Close #122) -Snowflake Web: Remove start_date variable from users module (Close #123) -Snowflake Web: Update copyright notices (Close #124) -Snowflake Web: Fix logic in users_sessions_this_run to account for sparse data (Close #120) -Snowflake Web: Fix varchar length for yauaa columns (Close #97) -Snowflake Web: Fix se_label column length in events_staged (Close #109) +Snowflake Web: Update column check stored procedure (#125) +Snowflake Web: Fix varchar length for pseudonymized fields (#122) +Snowflake Web: Remove start_date variable from users module (#123) +Snowflake Web: Update copyright notices (#124) +Snowflake Web: Fix logic in users_sessions_this_run to account for sparse data (#120) +Snowflake Web: Fix varchar length for yauaa columns (#97) +Snowflake Web: Fix se_label column length in events_staged (#109) Snowflake Mobile Version 1.1.0 (2021-06-16) --------------------------------------- -Snowflake Mobile: Update update_manifest to throw error message (Close #104) -Snowflake Mobile: Update commit_table procedure to throw error messages (Close #103) -Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #102) +Snowflake Mobile: Update update_manifest to throw error message (#104) +Snowflake Mobile: Update commit_table procedure to throw error messages (#103) +Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (#102) BigQuery Mobile Version 1.1.0 (2021-06-15) --------------------------------------- -BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (Close #100) -BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #99) +BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (#100) +BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (#99) Redshift Mobile Version 1.1.0 (2021-05-17) --------------------------------------- -Redshift Mobile: Increase session_id character limit in manifest (Close #93) -Redshift Mobile: Add simplified configs (Close #89) -Redshift Mobile: Set app errors module to disabled by default (Close #88) -Redshift Mobile: Add model_tstamp to derived tables (Close #82) -Redshift Mobile: Move app errors columns to end of table (Close #83) -Redshift Mobile: Remove CTE from user-aggs (Close #80) +Redshift Mobile: Increase session_id character limit in manifest (#93) +Redshift Mobile: Add simplified configs (#89) +Redshift Mobile: Set app errors module to disabled by default (#88) +Redshift Mobile: Add model_tstamp to derived tables (#82) +Redshift Mobile: Move app errors columns to end of table (#83) +Redshift Mobile: Remove CTE from user-aggs (#80) Snowflake Mobile Version 1.0.0 (2021-05-06) --------------------------------------- -Add Snowflake mobile model v1 (Close #85) +Add Snowflake mobile model v1 (#85) BigQuery Mobile Version 1.0.0 (2021-04-26) --------------------------------------- -Update licence (Close #79) -Fix credential loading in scripts (close #72) -Mobile: Add BigQuery mobile model v1 (close #73) +Update licence (#79) +Fix credential loading in scripts (#72) +Mobile: Add BigQuery mobile model v1 (#73) Redshift Mobile Version 1.0.0 (2021-03-25) --------------------------------------- @@ -51,8 +56,8 @@ Update Architecture diagram to match snowplow/snowplow (#62) BigQuery Web Version 1.0.2 (2021-03-11) --------------------------------------- -BigQuery: Handle schema evolution in core contexts (close #52) -Update SQL Runner download link to point to Github releases (close #51) +BigQuery: Handle schema evolution in core contexts (#52) +Update SQL Runner download link to point to Github releases (#51) BigQuery Web Version 1.0.1 (2021-02-24) --------------------------------------- diff --git a/web/v1/redshift/CHANGELOG b/web/v1/redshift/CHANGELOG index 0b8a02b3..a77401e8 100644 --- a/web/v1/redshift/CHANGELOG +++ b/web/v1/redshift/CHANGELOG @@ -1,3 +1,8 @@ +Version 1.3.0 (2022-06-06) +-------------------------- +Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!) +Redshift Web: Fix column lengths in manifest tables (#131) + Version 1.2.0 (2021-01-25) --------------------------- Redshift: Add config (#12) diff --git a/web/v1/redshift/README.md b/web/v1/redshift/README.md index 49eccb0a..067bd43d 100644 --- a/web/v1/redshift/README.md +++ b/web/v1/redshift/README.md @@ -167,7 +167,7 @@ Detail on configuring the users module's playbook can be found [in the relevant ## A note on duplicates -This version of the model (1.2.0) excludes duplicated event_ids and page_view_ids. Ideally in the future it will provide standard options for handling them, however because customisations of this model involve queries on federated atomic tables, the safest option for a first version is to exclude them. +This version of the model (>1.1.0) excludes duplicated event_ids and page_view_ids. Ideally in the future it will provide standard options for handling them, however because customisations of this model involve queries on federated atomic tables, the safest option for a first version is to exclude them. If there is a need to handle duplicates, this can be done by adding a custom module to the base level of aggregation - take good care to manage the possibility of introducing duplicates downstream if doing so. diff --git a/web/v1/redshift/sql-runner/configs/migrations.json b/web/v1/redshift/sql-runner/configs/migrations.json new file mode 100644 index 00000000..b4255d6f --- /dev/null +++ b/web/v1/redshift/sql-runner/configs/migrations.json @@ -0,0 +1,17 @@ +{ + "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0", + "data": { + "enabled": true, + "storage": "Default", + "playbooks": [ + { + "playbook": "standard/99-migrations/1.3.0-migration", + "dependsOn": [] + } + ], + "lockType": "hard", + "owners": [ + ] + } +} + \ No newline at end of file diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl index 04cf575d..aade872c 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl index 60b94254..eb50423c 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl index 2525b7b5..9a2720e7 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl index 2d799348..fd7540a2 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :input_schema: atomic :scratch_schema: scratch :output_schema: derived diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl index 92577c44..289c52fb 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl index 80e4e1e2..7e51bf02 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl index 45e34289..3a72ab6c 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :input_schema: atomic :scratch_schema: scratch :output_schema: derived diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl index 757ac59a..a2b38a21 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl index e68bdf9a..68a82468 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl index 5318448e..57524ea9 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl index 65570dff..63349656 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl index 35173ddc..5dd78f3c 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl index b425ad2e..d5dc5b2b 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl index 52887aac..afb897c1 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl index e15868b2..efb4139d 100644 --- a/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl +++ b/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl @@ -8,7 +8,7 @@ :password: :ssl: :variables: - :model_version: redshift/web/1.2.0 + :model_version: redshift/web/1.3.0 :scratch_schema: scratch :output_schema: derived :entropy: "" diff --git a/web/v1/redshift/sql-runner/playbooks/standard/99-migrations/1.3.0-migration.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/99-migrations/1.3.0-migration.yml.tmpl new file mode 100644 index 00000000..fa67bfee --- /dev/null +++ b/web/v1/redshift/sql-runner/playbooks/standard/99-migrations/1.3.0-migration.yml.tmpl @@ -0,0 +1,34 @@ +:targets: +- :name: + :type: redshift + :host: + :database: + :port: + :username: + :password: + :ssl: +:variables: + :scratch_schema: scratch + :output_schema: derived + :entropy: "" +:steps: +- :name: 1.3.0-migration + :queries: + - :name: base-session-id-manifest + :file: standard/99-migrations/1.3.0-migration/base-session-id-manifest.sql + :template: true +- :name: 1.3.0-migration + :queries: + - :name: sessions-userid-manifest + :file: standard/99-migrations/1.3.0-migration/sessions-userid-manifest.sql + :template: true +- :name: 1.3.0-migration + :queries: + - :name: users-manifest + :file: standard/99-migrations/1.3.0-migration/users-manifest.sql + :template: true +- :name: 1.3.0-migration + :queries: + - :name: sortkey-encoding + :file: standard/99-migrations/1.3.0-migration/sortkey-encoding.sql + :template: true \ No newline at end of file diff --git a/web/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql b/web/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql index e2dbb161..d7b79721 100644 --- a/web/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql +++ b/web/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql @@ -108,7 +108,7 @@ INSERT INTO {{.output_schema}}.base_event_id_manifest{{.entropy}} ( ); CREATE TABLE IF NOT EXISTS {{.output_schema}}.base_session_id_manifest{{.entropy}} ( - session_id VARCHAR(36), + session_id VARCHAR(128), min_tstamp TIMESTAMP ) DISTSTYLE KEY @@ -117,7 +117,7 @@ SORTKEY (min_tstamp); INSERT INTO {{.output_schema}}.base_session_id_manifest{{.entropy}} ( SELECT - 'seed'::VARCHAR(36), + 'seed'::VARCHAR(128), '{{.start_date}}'::TIMESTAMP WHERE diff --git a/web/v1/redshift/sql-runner/sql/standard/02-page-views/01-main/00-setup-page-views.sql b/web/v1/redshift/sql-runner/sql/standard/02-page-views/01-main/00-setup-page-views.sql index a49469e8..43b13009 100644 --- a/web/v1/redshift/sql-runner/sql/standard/02-page-views/01-main/00-setup-page-views.sql +++ b/web/v1/redshift/sql-runner/sql/standard/02-page-views/01-main/00-setup-page-views.sql @@ -106,7 +106,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.page_views{{.entropy}} ( dvce_created_tstamp TIMESTAMP ENCODE ZSTD, collector_tstamp TIMESTAMP ENCODE ZSTD, derived_tstamp TIMESTAMP ENCODE ZSTD, - start_tstamp TIMESTAMP ENCODE ZSTD, + start_tstamp TIMESTAMP ENCODE RAW, end_tstamp TIMESTAMP ENCODE ZSTD, engaged_time_in_s INT ENCODE ZSTD, diff --git a/web/v1/redshift/sql-runner/sql/standard/03-sessions/01-main/00-setup-sessions.sql b/web/v1/redshift/sql-runner/sql/standard/03-sessions/01-main/00-setup-sessions.sql index b98caaf3..bc9838e8 100644 --- a/web/v1/redshift/sql-runner/sql/standard/03-sessions/01-main/00-setup-sessions.sql +++ b/web/v1/redshift/sql-runner/sql/standard/03-sessions/01-main/00-setup-sessions.sql @@ -95,7 +95,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.sessions{{.entropy}} ( domain_sessionid VARCHAR(128) ENCODE ZSTD, domain_sessionidx INT ENCODE ZSTD, - start_tstamp TIMESTAMP ENCODE ZSTD, + start_tstamp TIMESTAMP ENCODE RAW, end_tstamp TIMESTAMP ENCODE ZSTD, -- user fields @@ -220,7 +220,7 @@ SORTKEY (start_tstamp); -- Staged manifest table as input to users step CREATE TABLE IF NOT EXISTS {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}} ( - domain_userid VARCHAR(36), + domain_userid VARCHAR(128), start_tstamp TIMESTAMP ) DISTSTYLE KEY diff --git a/web/v1/redshift/sql-runner/sql/standard/04-users/01-main/00-setup-users.sql b/web/v1/redshift/sql-runner/sql/standard/04-users/01-main/00-setup-users.sql index 567da8fd..a38cf4a6 100644 --- a/web/v1/redshift/sql-runner/sql/standard/04-users/01-main/00-setup-users.sql +++ b/web/v1/redshift/sql-runner/sql/standard/04-users/01-main/00-setup-users.sql @@ -87,7 +87,7 @@ INSERT INTO {{.scratch_schema}}.users_metadata_this_run{{.entropy}} ( ); CREATE TABLE IF NOT EXISTS {{.output_schema}}.users_manifest{{.entropy}} ( - domain_userid VARCHAR(36), + domain_userid VARCHAR(128), start_tstamp TIMESTAMP ) DISTSTYLE KEY @@ -111,7 +111,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.users{{.entropy}} ( domain_userid VARCHAR(128) ENCODE ZSTD, network_userid VARCHAR(128) ENCODE ZSTD, - start_tstamp TIMESTAMP ENCODE ZSTD, + start_tstamp TIMESTAMP ENCODE RAW, end_tstamp TIMESTAMP ENCODE ZSTD, page_views INT ENCODE ZSTD, diff --git a/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/base-session-id-manifest.sql b/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/base-session-id-manifest.sql new file mode 100644 index 00000000..e96bc98b --- /dev/null +++ b/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/base-session-id-manifest.sql @@ -0,0 +1,3 @@ +-- Extend column lengths to match their correct values +-- Since ALTER TABLE ALTER COLUMN cannot run inside a multiple command statement, we need a file each. +ALTER TABLE {{.output_schema}}.base_session_id_manifest{{.entropy}} ALTER COLUMN session_id type VARCHAR(128); diff --git a/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/sessions-userid-manifest.sql b/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/sessions-userid-manifest.sql new file mode 100644 index 00000000..ac8abd23 --- /dev/null +++ b/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/sessions-userid-manifest.sql @@ -0,0 +1,3 @@ +-- Extend column lengths to match their correct values +-- Since ALTER TABLE ALTER COLUMN cannot run inside a multiple command statement, we need a file each. +ALTER TABLE {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}} ALTER COLUMN domain_userid type VARCHAR(128); diff --git a/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/sortkey-encoding.sql b/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/sortkey-encoding.sql new file mode 100644 index 00000000..709b545c --- /dev/null +++ b/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/sortkey-encoding.sql @@ -0,0 +1,8 @@ +-- Change sortkey encodings to RAW +ALTER TABLE {{.output_schema}}.page_views{{.entropy}} ALTER COLUMN start_tstamp ENCODE RAW; + +ALTER TABLE {{.scratch_schema}}.page_views_staged{{.entropy}} ALTER COLUMN start_tstamp ENCODE RAW; + +ALTER TABLE {{.output_schema}}.sessions{{.entropy}} ALTER COLUMN start_tstamp ENCODE RAW; + +ALTER TABLE {{.output_schema}}.users{{.entropy}} ALTER COLUMN start_tstamp ENCODE RAW; \ No newline at end of file diff --git a/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/users-manifest.sql b/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/users-manifest.sql new file mode 100644 index 00000000..972b3529 --- /dev/null +++ b/web/v1/redshift/sql-runner/sql/standard/99-migrations/1.3.0-migration/users-manifest.sql @@ -0,0 +1,3 @@ +-- Extend column lengths to match their correct values +-- Since ALTER TABLE ALTER COLUMN cannot run inside a multiple command statement, we need a file each. +ALTER TABLE {{.output_schema}}.users_manifest{{.entropy}} ALTER COLUMN domain_userid type VARCHAR(128); \ No newline at end of file