Skip to content

Commit

Permalink
Merge branch 'main' into bugfix/fix-parsing-error-for-invalid-json
Browse files Browse the repository at this point in the history
  • Loading branch information
@llifoawing
llifoawing authored Sep 20, 2024
2 parents d1c3a94 + c350293 commit fece08a
Show file tree
Hide file tree
Showing 35 changed files with 846 additions and 785 deletions.
201 changes: 106 additions & 95 deletions models/dim_dbt__current_models.sql
View file
Original file line number Diff line number Diff line change
@@ -1,97 +1,108 @@
with base as (
select *
from {{ ref('stg_dbt__models') }}
),

model_executions as (
select *
from {{ ref('stg_dbt__model_executions') }}
),

latest_models as (
/* Retrieves the models present in the most recent run */
select *
from base
where run_started_at = (select max(run_started_at) from base)
),

latest_models_runs as (
/* Retreives all successful run information for the models present in the most
recent run and ranks them based on query completion time */
select
model_executions.node_id
, model_executions.was_full_refresh
, model_executions.query_completed_at
, model_executions.total_node_runtime
, model_executions.rows_affected
{% if target.type == 'bigquery' %}
, model_executions.bytes_processed
{% endif %}
/* Row number by refresh and node ID */
, row_number() over (
partition by latest_models.node_id, model_executions.was_full_refresh
order by model_executions.query_completed_at desc /* most recent ranked first */
) as run_idx
/* Row number by node ID */
, row_number() over (
partition by latest_models.node_id
order by model_executions.query_completed_at desc /* most recent ranked first */
) as run_idx_id_only
from model_executions
inner join latest_models on model_executions.node_id = latest_models.node_id
where model_executions.status = 'success'
),

latest_model_stats as (
select
node_id
, max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at
, max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime
, max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed
{% endif %}
, max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at
, max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime
, max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed
{% endif %}
, max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at
, max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime
, max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed
{% endif %}
from latest_models_runs
where run_idx = 1
group by 1
),

final as (
select
latest_models.*
, latest_model_stats.last_full_refresh_run_completed_at
, latest_model_stats.last_full_refresh_run_total_runtime
, latest_model_stats.last_full_refresh_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_full_refresh_run_bytes_processed
{% endif %}
, latest_model_stats.last_run_completed_at
, latest_model_stats.last_run_total_runtime
, latest_model_stats.last_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_run_bytes_processed
{% endif %}
, latest_model_stats.last_incremental_run_completed_at
, latest_model_stats.last_incremental_run_total_runtime
, latest_model_stats.last_incremental_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_incremental_run_bytes_processed
{% endif %}
from latest_models
left join latest_model_stats
on latest_models.node_id = latest_model_stats.node_id
)
with
base as (

select * from {{ ref('stg_dbt__models') }}

)

, model_executions as (

select * from {{ ref('stg_dbt__model_executions') }}

)

, latest_models as (

/* Retrieves the models present in the most recent run */
select *
from base
where run_started_at = (select max(run_started_at) from base)

)

, latest_models_runs as (

/* Retreives all successful run information for the models present in the most
recent run and ranks them based on query completion time */
select
model_executions.node_id
, model_executions.was_full_refresh
, model_executions.query_completed_at
, model_executions.total_node_runtime
, model_executions.rows_affected
{% if target.type == 'bigquery' %}
, model_executions.bytes_processed
{% endif %}
/* Row number by refresh and node ID */
, row_number() over (
partition by latest_models.node_id, model_executions.was_full_refresh
order by model_executions.query_completed_at desc /* most recent ranked first */
) as run_idx
/* Row number by node ID */
, row_number() over (
partition by latest_models.node_id
order by model_executions.query_completed_at desc /* most recent ranked first */
) as run_idx_id_only
from model_executions
inner join latest_models on model_executions.node_id = latest_models.node_id
where model_executions.status = 'success'

)

, latest_model_stats as (

select
node_id
, max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at
, max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime
, max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed
{% endif %}
, max(case when run_idx_id_only = 1 then query_completed_at end) as last_run_completed_at
, max(case when run_idx_id_only = 1 then total_node_runtime end) as last_run_total_runtime
, max(case when run_idx_id_only = 1 then rows_affected end) as last_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed
{% endif %}
, max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at
, max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime
, max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected
{% if target.type == 'bigquery' %}
, max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed
{% endif %}
from latest_models_runs
where run_idx = 1
group by 1

)

, final as (

select
latest_models.*
, latest_model_stats.last_full_refresh_run_completed_at
, latest_model_stats.last_full_refresh_run_total_runtime
, latest_model_stats.last_full_refresh_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_full_refresh_run_bytes_processed
{% endif %}
, latest_model_stats.last_run_completed_at
, latest_model_stats.last_run_total_runtime
, latest_model_stats.last_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_run_bytes_processed
{% endif %}
, latest_model_stats.last_incremental_run_completed_at
, latest_model_stats.last_incremental_run_total_runtime
, latest_model_stats.last_incremental_run_rows_affected
{% if target.type == 'bigquery' %}
, latest_model_stats.last_incremental_run_bytes_processed
{% endif %}
from latest_models
left join latest_model_stats
on latest_models.node_id = latest_model_stats.node_id

)

select * from final
45 changes: 23 additions & 22 deletions models/dim_dbt__exposures.sql
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,30 @@
with base as (
with
base as (

select *
from {{ ref('stg_dbt__exposures') }}
select *
from {{ ref('stg_dbt__exposures') }}

),
)

exposures as (
, exposures as (

select
exposure_execution_id,
command_invocation_id,
node_id,
run_started_at,
name,
type,
owner,
maturity,
path,
description,
url,
package_name,
depends_on_nodes,
tags
from base
select
exposure_execution_id
, command_invocation_id
, node_id
, run_started_at
, name
, type
, owner
, maturity
, path
, description
, url
, package_name
, depends_on_nodes
, tags
from base

)
)

select * from exposures
47 changes: 24 additions & 23 deletions models/dim_dbt__models.sql
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,31 @@
with base as (
with
base as (

select *
from {{ ref('stg_dbt__models') }}
select *
from {{ ref('stg_dbt__models') }}

),
)

models as (
, models as (

select
model_execution_id,
command_invocation_id,
node_id,
run_started_at,
database,
schema,
name,
depends_on_nodes,
package_name,
path,
checksum,
materialization,
tags,
meta,
alias
from base
select
model_execution_id
, command_invocation_id
, node_id
, run_started_at
, database
, schema
, name
, depends_on_nodes
, package_name
, path
, checksum
, materialization
, tags
, meta
, alias
from base

)
)

select * from models
41 changes: 21 additions & 20 deletions models/dim_dbt__seeds.sql
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,28 @@
with base as (
with
base as (

select *
from {{ ref('stg_dbt__seeds') }}
select *
from {{ ref('stg_dbt__seeds') }}

),
)

seeds as (
, seeds as (

select
seed_execution_id,
command_invocation_id,
node_id,
run_started_at,
database,
schema,
name,
package_name,
path,
checksum,
meta,
alias
from base
select
seed_execution_id
, command_invocation_id
, node_id
, run_started_at
, database
, schema
, name
, package_name
, path
, checksum
, meta
, alias
from base

)
)

select * from seeds
Loading

0 comments on commit fece08a

Please sign in to comment.