From a1249f68fe544d1385eee0eac22e90ab91313162 Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 20 Dec 2023 10:25:31 -0300 Subject: [PATCH] feat: add materialize_dbt_model template pipeline --- pipelines/flows.py | 1 + pipelines/templates/__init__.py | 2 ++ pipelines/templates/run_dbt_model/__init__.py | 0 pipelines/templates/run_dbt_model/flows.py | 36 +++++++++++++++++++ poetry.lock | 6 ++-- pyproject.toml | 2 +- 6 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 pipelines/templates/__init__.py create mode 100644 pipelines/templates/run_dbt_model/__init__.py create mode 100644 pipelines/templates/run_dbt_model/flows.py diff --git a/pipelines/flows.py b/pipelines/flows.py index cbad709..ee723fe 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -8,3 +8,4 @@ from pipelines.exemplo import * # noqa from pipelines.iptu_inad import * # noqa from pipelines.sigma import * # noqa +from pipelines.templates import * # noqa diff --git a/pipelines/templates/__init__.py b/pipelines/templates/__init__.py new file mode 100644 index 0000000..ef8b2db --- /dev/null +++ b/pipelines/templates/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from pipelines.templates.run_dbt_model.flows import * # noqa: F401, F403 diff --git a/pipelines/templates/run_dbt_model/__init__.py b/pipelines/templates/run_dbt_model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py new file mode 100644 index 0000000..c7281fb --- /dev/null +++ b/pipelines/templates/run_dbt_model/flows.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +MATERIALIZA MODELOS DO DBT. +""" + +from copy import deepcopy + +from prefect.run_configs import KubernetesRun +from prefect.storage import GCS +from prefeitura_rio.pipelines_templates.run_dbt_model.flows import ( + templates__run_dbt_model__flow, +) +from prefeitura_rio.pipelines_utils.prefect import set_default_parameters +from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials + +from pipelines.constants import constants + +templates__run_dbt_model_smas__flow = deepcopy(templates__run_dbt_model__flow) +templates__run_dbt_model_smas__flow.state_handlers = [handler_inject_bd_credentials] + +templates__run_dbt_model_smas__flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value) +templates__run_dbt_model_smas__flow.run_config = KubernetesRun( + image=constants.DOCKER_IMAGE.value, + labels=[ + constants.RJ_SMFP_AGENT_LABEL.value, + ], +) + +templates_run_dbt_model_smas_default_parameters = { + "dataset_id": "dataset_id", + "table_id": "table_id", +} +templates__run_dbt_model_smas__flow = set_default_parameters( + templates__run_dbt_model_smas__flow, + default_parameters=templates_run_dbt_model_smas_default_parameters, +) diff --git a/poetry.lock b/poetry.lock index 63bed68..596bf3f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2416,8 +2416,8 @@ sgrc = ["aiohttp (>=3.8.5,<4.0.0)", "pendulum (>=2.1.2,<3.0.0)", "pytz (>=2023.3 [package.source] type = "git" url = "https://github.com/prefeitura-rio/prefeitura-rio" -reference = "af17a1b14228ccd7542b7bad06d0174bed542e32" -resolved_reference = "af17a1b14228ccd7542b7bad06d0174bed542e32" +reference = "027793da91f479426d655a889b3377f9c4fe17a9" +resolved_reference = "027793da91f479426d655a889b3377f9c4fe17a9" [[package]] name = "proto-plus" @@ -3962,4 +3962,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "70030c35b39c93725ae9f56f20b114b33ad2ab55751f35290e18b1ffaf8f839a" +content-hash = "3fde30407d44b06f99e350fdb15106cd966fc27763fd86a5491bcf5e799cfae9" diff --git a/pyproject.toml b/pyproject.toml index 9ac2f40..6748fa6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ python = ">=3.10,<3.11" dbt-bigquery = "^1.6.1" google-cloud-storage = "^2.10.0" prefect = "1.4.1" -prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "af17a1b14228ccd7542b7bad06d0174bed542e32", extras = [ +prefeitura-rio = { git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "027793da91f479426d655a889b3377f9c4fe17a9", extras = [ "pipelines", "pipelines-templates", ] }