From ee9156263057b5ede1ba666484c6c75819532c38 Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:52:53 +0100 Subject: [PATCH] Fix BigQuery parse_json macro usage (#371) * Use safe.parse_json instead of parse_json * Replace hard-coded parse_json with macro --- macros/parse_json.sql | 2 +- macros/upload_exposures.sql | 4 ++-- macros/upload_invocations.sql | 8 ++++---- macros/upload_model_executions.sql | 2 +- macros/upload_models.sql | 4 ++-- macros/upload_seed_executions.sql | 2 +- macros/upload_seeds.sql | 4 ++-- macros/upload_snapshot_executions.sql | 2 +- macros/upload_snapshots.sql | 4 ++-- macros/upload_sources.sql | 4 ++-- macros/upload_test_executions.sql | 2 +- macros/upload_tests.sql | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/macros/parse_json.sql b/macros/parse_json.sql index 4930c088..39ea5d8f 100644 --- a/macros/parse_json.sql +++ b/macros/parse_json.sql @@ -11,6 +11,6 @@ {%- endmacro %} {% macro bigquery__parse_json(field) -%} - parse_json({{ field }}, wide_number_mode=>'round') + safe.parse_json("""{{ field }}""", wide_number_mode=>'round') {%- endmacro %} diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index daa86fea..d8653154 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -62,7 +62,7 @@ '{{ run_started_at }}', {# run_started_at #} '{{ exposure.name | replace("'","\\'") }}', {# name #} '{{ exposure.type }}', {# type #} - parse_json('{{ tojson(exposure.owner) | replace("'","\\'") }}'), {# owner #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(exposure.owner) | replace("'","\\'")) }}, {# owner #} '{{ exposure.maturity }}', {# maturity #} '{{ exposure.original_file_path | replace('\\', '\\\\') }}', {# path #} """{{ exposure.description | replace("'","\\'") }}""", {# description #} @@ -70,7 +70,7 @@ '{{ exposure.package_name }}', {# package_name #} {{ tojson(exposure.depends_on.nodes) }}, {# depends_on_nodes #} {{ tojson(exposure.tags) }}, {# tags #} - parse_json('{{ tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index b1d6e7bc..f3dcdd6b 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -122,7 +122,7 @@ {% for env_variable in var('env_vars') %} {% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %} {% endfor %} - parse_json('''{{ tojson(env_vars_dict) }}'''), {# env_vars #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(env_vars_dict)) }}, {# env_vars #} {% else %} null, {# env_vars #} {% endif %} @@ -132,7 +132,7 @@ {% for dbt_var in var('dbt_vars') %} {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %} {% endfor %} - parse_json('''{{ tojson(dbt_vars_dict) }}'''), {# dbt_vars #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(dbt_vars_dict)) }}, {# dbt_vars #} {% else %} null, {# dbt_vars #} {% endif %} @@ -146,13 +146,13 @@ {% endif %} {% endif %} - safe.parse_json('''{{ tojson(invocation_args_dict) | replace("'", "\\'") }}'''), {# invocation_args #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(invocation_args_dict) | replace("'", "\\'")) }}, {# invocation_args #} {% set metadata_env = {} %} {% for key, value in dbt_metadata_envs.items() %} {% do metadata_env.update({key: value}) %} {% endfor %} - parse_json('''{{ tojson(metadata_env) | replace('\\', '\\\\') }}''') {# dbt_custom_envs #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(metadata_env) | replace('\\', '\\\\')) }} {# dbt_custom_envs #} ) {% endset %} diff --git a/macros/upload_model_executions.sql b/macros/upload_model_executions.sql index 14917d1a..a8ebbe89 100644 --- a/macros/upload_model_executions.sql +++ b/macros/upload_model_executions.sql @@ -131,7 +131,7 @@ '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_models.sql b/macros/upload_models.sql index b8fce69e..bc640f52 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -69,9 +69,9 @@ '{{ model.checksum.checksum }}', {# checksum #} '{{ model.config.materialized }}', {# materialization #} {{ tojson(model.tags) }}, {# tags #} - parse_json('''{{ tojson(model.config.meta) }}'''), {# meta #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.config.meta)) }}, {# meta #} '{{ model.alias }}', {# alias #} - parse_json('{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_seed_executions.sql b/macros/upload_seed_executions.sql index a7ada664..4be58b09 100644 --- a/macros/upload_seed_executions.sql +++ b/macros/upload_seed_executions.sql @@ -129,7 +129,7 @@ '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index 61683f75..8158bf84 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -62,9 +62,9 @@ '{{ seed.package_name }}', {# package_name #} '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #} '{{ seed.checksum.checksum }}', {# checksum #} - parse_json('''{{ tojson(seed.config.meta) }}'''), {# meta #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed.config.meta)) }}, {# meta #} '{{ seed.alias }}', {# alias #} - parse_json('{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index 4903f48e..3e8c03d0 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -129,7 +129,7 @@ '{{ model.node.name }}', {# name #} '{{ model.node.alias }}', {# alias #} '{{ model.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index cac08abc..477f46a6 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -69,9 +69,9 @@ '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #} '{{ snapshot.checksum.checksum }}', {# checksum #} '{{ snapshot.config.strategy }}', {# strategy #} - parse_json('''{{ tojson(snapshot.config.meta) }}'''), {# meta #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot.config.meta)) }}, {# meta #} '{{ snapshot.alias }}', {# alias #} - parse_json('{{ tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index a624ea4b..38ab6ec5 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -59,8 +59,8 @@ '{{ source.name }}', {# name #} '{{ source.identifier }}', {# identifier #} '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} - parse_json('{{ tojson(source.freshness) | replace("'","\\'") }}'), {# freshness #} - parse_json('{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# all_results #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source.freshness) | replace("'","\\'")) }}, {# freshness #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_test_executions.sql b/macros/upload_test_executions.sql index 1f0f43d7..a8747099 100644 --- a/macros/upload_test_executions.sql +++ b/macros/upload_test_executions.sql @@ -120,7 +120,7 @@ null, {# rows_affected not available in Databricks #} {{ 'null' if test.failures is none else test.failures }}, {# failures #} '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') | replace("\n", "\\n") }}', {# message #} - parse_json('{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# adapter_response #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# adapter_response #} ) {%- if not loop.last %},{%- endif %} diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index 5d7c268d..9d6a02a6 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -51,7 +51,7 @@ '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} {{ tojson(test.tags) }}, {# tags #} - parse_json('{{ tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', wide_number_mode=>'round') {# all_fields #} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_fields #} ) {%- if not loop.last %},{%- endif %} {%- endfor %}