Skip to content

Commit

Permalink
feat: add materialize_dbt_model template pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
d116626 committed Dec 20, 2023
1 parent 2edf09d commit a1249f6
Show file tree
Hide file tree
Showing 6 changed files with 43 additions and 4 deletions.
1 change: 1 addition & 0 deletions pipelines/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@
from pipelines.exemplo import * # noqa
from pipelines.iptu_inad import * # noqa
from pipelines.sigma import * # noqa
from pipelines.templates import * # noqa
2 changes: 2 additions & 0 deletions pipelines/templates/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# -*- coding: utf-8 -*-
from pipelines.templates.run_dbt_model.flows import * # noqa: F401, F403
Empty file.
36 changes: 36 additions & 0 deletions pipelines/templates/run_dbt_model/flows.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# -*- coding: utf-8 -*-
"""
MATERIALIZA MODELOS DO DBT.
"""

from copy import deepcopy

from prefect.run_configs import KubernetesRun
from prefect.storage import GCS
from prefeitura_rio.pipelines_templates.run_dbt_model.flows import (
templates__run_dbt_model__flow,
)
from prefeitura_rio.pipelines_utils.prefect import set_default_parameters
from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials

from pipelines.constants import constants

templates__run_dbt_model_smas__flow = deepcopy(templates__run_dbt_model__flow)
templates__run_dbt_model_smas__flow.state_handlers = [handler_inject_bd_credentials]

templates__run_dbt_model_smas__flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value)
templates__run_dbt_model_smas__flow.run_config = KubernetesRun(
image=constants.DOCKER_IMAGE.value,
labels=[
constants.RJ_SMFP_AGENT_LABEL.value,
],
)

templates_run_dbt_model_smas_default_parameters = {
"dataset_id": "dataset_id",
"table_id": "table_id",
}
templates__run_dbt_model_smas__flow = set_default_parameters(
templates__run_dbt_model_smas__flow,
default_parameters=templates_run_dbt_model_smas_default_parameters,
)
6 changes: 3 additions & 3 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ python = ">=3.10,<3.11"
dbt-bigquery = "^1.6.1"
google-cloud-storage = "^2.10.0"
prefect = "1.4.1"
prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "af17a1b14228ccd7542b7bad06d0174bed542e32", extras = [
prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "027793da91f479426d655a889b3377f9c4fe17a9", extras = [
"pipelines",
"pipelines-templates",
] }
Expand Down

0 comments on commit a1249f6

Please sign in to comment.