From 36d176bfe195235e7320e9e00385b1c126202996 Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Fri, 3 Jun 2022 10:56:32 -0500 Subject: [PATCH] feature/passthrough-ids --- README.md | 3 +++ dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- .../int_jira__daily_field_history.sql | 7 ++++++- .../int_jira__pivot_daily_field_history.sql | 17 +++++++++++++++-- 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f47644c7..3b1369a8 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ vars: ### Daily Issue Field History Columns The `jira__daily_issue_field_history` model generates historical data for the columns specified by the `issue_field_history_columns` variable. By default, the only columns tracked are `status` and `sprint`, but all fields found in the `field_name` column within the Jira `FIELD` table can be included in this model. The most recent value of any tracked column is also captured in `jira__issue_enhanced`. +Additionally, if you are concerned about the mutable nature of the `field_name` being used for the custom fields. This package includes another variable `issue_field_history_columns_ids` that can be used to include the `field_id` value of your custom fields. A positive of this is that the materialized models will leverage the `field_name` for the column name. Please note if your field names are constantly changing then new columns will appear in your end model. + **If you would like to change these columns, add the following configuration to your dbt_project.yml file. Then, after adding the columns to your `dbt_project.yml` file, run the `dbt run --full-refresh` command to fully refresh any existing models.** ```yml @@ -60,6 +62,7 @@ config-version: 2 vars: jira: issue_field_history_columns: ['the', 'list', 'of', 'field', 'names'] + issue_field_history_columns_ids: ['this_id', 'that_id', 'ids_id'] ``` > Note: `sprint` and `status` will always be tracked, as they are necessary for creating common agile reports. diff --git a/dbt_project.yml b/dbt_project.yml index 01cace6b..7ee94f5e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'jira' -version: '0.8.1' +version: '0.8.2' config-version: 2 require-dbt-version: [">=1.0.0", "<2.0.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 1f6f75b9..1d09493b 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'jira_integration_tests' -version: '0.8.1' +version: '0.8.2' config-version: 2 profile: 'integration_tests' diff --git a/models/intermediate/field_history/int_jira__daily_field_history.sql b/models/intermediate/field_history/int_jira__daily_field_history.sql index 112f717d..bc92d17c 100644 --- a/models/intermediate/field_history/int_jira__daily_field_history.sql +++ b/models/intermediate/field_history/int_jira__daily_field_history.sql @@ -28,7 +28,12 @@ limit_to_relevant_fields as ( from combined_field_histories - where lower(field_id) = 'status' -- As sprint is a custom field, we filter by field name only for sprint. All others are on field_id. + -- As sprint is a custom field, we filter by field name only for sprint. All others are on field_id. + where lower(field_id) in ('status' + {%- for col in var('issue_field_history_columns_ids', []) -%} + ,'{{ (col|lower) }}' + {%- endfor -%} ) + or lower(field_name) in ('status' {%- for col in var('issue_field_history_columns', []) -%} ,'{{ (col|lower) }}' diff --git a/models/intermediate/field_history/int_jira__pivot_daily_field_history.sql b/models/intermediate/field_history/int_jira__pivot_daily_field_history.sql index 4db6f00e..3057afb1 100644 --- a/models/intermediate/field_history/int_jira__pivot_daily_field_history.sql +++ b/models/intermediate/field_history/int_jira__pivot_daily_field_history.sql @@ -32,10 +32,23 @@ pivot_out as ( max(case when lower(field_name) = 'sprint' then field_value end) as sprint -- As sprint is a custom field, we aggregate on the field_name. {% for col in var('issue_field_history_columns', []) -%} - , - max(case when lower(field_name) = '{{ col|lower }}' then field_value end) as {{ dbt_utils.slugify(col) | replace(' ', '_') | lower }} + , max(case when lower(field_name) = '{{ col|lower }}' then field_value end) as {{ dbt_utils.slugify(col) | replace(' ', '_') | lower }} {% endfor -%} + {% if var('issue_field_history_columns_ids', []) != [] %} + {% for col in var('issue_field_history_columns_ids', []) -%} + + --Map the corresponding field id that is used in the variable to the field name from the staging model + {% set name_mapping = dbt_utils.get_column_values( + table=ref('stg_jira__field'), + column='field_name', + where="field_id = '" ~ col ~ "'") + %} + + , max(case when lower(field_id) = '{{ col|lower }}' then field_value end) as {{ dbt_utils.slugify(name_mapping[0]) | replace(' ', '_') | lower }} + {% endfor -%} + {%endif %} + from daily_field_history group by 1,2