From 8cfa93c52ed15d388a7fbc9ed5564cf0a6869913 Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 23 Oct 2024 10:31:07 -0300 Subject: [PATCH 1/8] chore: change from kunbernetsRun to DockerRun --- pipelines/constants.py | 1 + pipelines/ergon/dump_db_ergon/flows.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pipelines/constants.py b/pipelines/constants.py index 5ffbdfc..50516db 100644 --- a/pipelines/constants.py +++ b/pipelines/constants.py @@ -17,6 +17,7 @@ class constants(Enum): # Agent labels ###################################### RJ_SMFP_AGENT_LABEL = "smfp" + RJ_LOCAL_IPLAN_AGENT_LABEL = "local-iplan" ###################################### # Other constants diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index 25d4c3b..7dcbbe4 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -5,7 +5,7 @@ from copy import deepcopy -from prefect.run_configs import KubernetesRun +from prefect.run_configs import DockerRun, KubernetesRun # noqa from prefect.storage import GCS from prefeitura_rio.pipelines_templates.dump_db.flows import flow as dump_sql_flow from prefeitura_rio.pipelines_utils.prefect import set_default_parameters @@ -21,13 +21,14 @@ dump_sql_ergon_flow.state_handlers = [handler_inject_bd_credentials, handler_initialize_sentry] dump_sql_ergon_flow.name = "SMFP: ergon - Ingerir tabelas de banco SQL" dump_sql_ergon_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value) -dump_sql_ergon_flow.run_config = KubernetesRun( +dump_sql_ergon_flow.run_config = DockerRun( image=constants.DOCKER_IMAGE.value, labels=[ constants.RJ_SMFP_AGENT_LABEL.value, ], ) + ergon_default_parameters = { "db_database": "P01.PCRJ", "db_host": "10.70.6.21", From ca0b89682bc9067b9b8fa1cb18c9770214c55ee7 Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 23 Oct 2024 10:33:45 -0300 Subject: [PATCH 2/8] chore: change from kunbernetsRun to DockerRun --- pipelines/ergon/dump_db_ergon/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index 7dcbbe4..2856714 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project......... +Database dumping flows for segovi project """ from copy import deepcopy From 636120d7741b6923c28a18279074a62d7c45c502 Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 23 Oct 2024 10:36:21 -0300 Subject: [PATCH 3/8] chore: change from kunbernetsRun to DockerRun --- pipelines/ergon/dump_db_ergon/flows.py | 2 +- pipelines/ergon/dump_db_ergon/schedules.py | 2 +- pipelines/flows.py | 2 +- pipelines/templates/run_dbt_model/flows.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index 2856714..6e1d5ac 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project +Database dumping flows for segovi project. """ from copy import deepcopy diff --git a/pipelines/ergon/dump_db_ergon/schedules.py b/pipelines/ergon/dump_db_ergon/schedules.py index 8d428d4..c89007d 100644 --- a/pipelines/ergon/dump_db_ergon/schedules.py +++ b/pipelines/ergon/dump_db_ergon/schedules.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa: E501 """ -Schedules for the database dump pipeline..... +Schedules for the database dump pipeline. """ from datetime import datetime, timedelta diff --git a/pipelines/flows.py b/pipelines/flows.py index bbfe2cb..88e7733 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Imports all flows for every project so we can register all of them... +Imports all flows for every project so we can register all of them. """ from pipelines.egpweb_metas import * # noqa from pipelines.ergon import * # noqa diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index b214bde..c7281fb 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT........... +MATERIALIZA MODELOS DO DBT. """ from copy import deepcopy From 4dacdcd1eaafcc543a32541e1b7000b71fd2d9a8 Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 23 Oct 2024 10:39:36 -0300 Subject: [PATCH 4/8] chore: change from kunbernetsRun to DockerRun --- pipelines/ergon/dump_db_ergon/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index 6e1d5ac..0a9589f 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project. +Database dumping flows for segovi project.. """ from copy import deepcopy From f78d7df63e625f8ff342b423f9ab336cb2333050 Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 23 Oct 2024 10:46:19 -0300 Subject: [PATCH 5/8] chore: change from kubernetsRun to DockerRun --- pipelines/ergon/dump_db_ergon/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index 0a9589f..c575153 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -24,7 +24,7 @@ dump_sql_ergon_flow.run_config = DockerRun( image=constants.DOCKER_IMAGE.value, labels=[ - constants.RJ_SMFP_AGENT_LABEL.value, + constants.RJ_LOCAL_IPLAN_AGENT_LABEL.value, ], ) From 64edda1dad578093a450374e8913b9ab926ad8ad Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 25 Oct 2024 14:55:30 -0300 Subject: [PATCH 6/8] chore: pump prefeitura-rio version --- pipelines/egpweb_metas/dump_db_metas/flows.py | 2 +- pipelines/ergon/dump_db_ergon/flows.py | 2 +- .../dump_db_ergon_pericia_medica/flows.py | 2 +- .../materializa_funcionarios_saude/flows.py | 2 +- .../dump_db_ergon_comlurb/flows.py | 2 +- pipelines/fincon/dump_db_fincon/flows.py | 2 +- pipelines/flows.py | 2 +- .../iptu_inad/dump_db_inadimplentes/flows.py | 2 +- .../dump_db_porte_empresa/flows.py | 2 +- .../sigma/dump_db_compras_materiais/flows.py | 2 +- .../sigma/dump_db_sancao_fornecedor/flows.py | 2 +- pipelines/templates/run_dbt_model/flows.py | 2 +- poetry.lock | 18 +++++++++--------- pyproject.toml | 2 +- 14 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pipelines/egpweb_metas/dump_db_metas/flows.py b/pipelines/egpweb_metas/dump_db_metas/flows.py index 1bcc953..f7c372a 100644 --- a/pipelines/egpweb_metas/dump_db_metas/flows.py +++ b/pipelines/egpweb_metas/dump_db_metas/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for EGPWeb project +Database dumping flows for EGPWeb project. """ from copy import deepcopy diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index c575153..04de8ec 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project.. +Database dumping flows for segovi project... """ from copy import deepcopy diff --git a/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py b/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py index 0795e3e..f02eef0 100644 --- a/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py +++ b/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project. +Database dumping flows for segovi project.. """ from copy import deepcopy diff --git a/pipelines/ergon/materializa_funcionarios_saude/flows.py b/pipelines/ergon/materializa_funcionarios_saude/flows.py index 3c65b1b..8f00717 100644 --- a/pipelines/ergon/materializa_funcionarios_saude/flows.py +++ b/pipelines/ergon/materializa_funcionarios_saude/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Materialize Active SMS Employees from Ergon.. +Materialize Active SMS Employees from Ergon... """ from copy import deepcopy diff --git a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py index ca7567a..7216488 100644 --- a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py +++ b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for smfp ergon comlurb...... +Database dumping flows for smfp ergon comlurb....... """ from copy import deepcopy diff --git a/pipelines/fincon/dump_db_fincon/flows.py b/pipelines/fincon/dump_db_fincon/flows.py index f33baa0..51708d7 100644 --- a/pipelines/fincon/dump_db_fincon/flows.py +++ b/pipelines/fincon/dump_db_fincon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for SMFP FINCON. +Database dumping flows for SMFP FINCON.. """ from copy import deepcopy diff --git a/pipelines/flows.py b/pipelines/flows.py index 88e7733..9418fd9 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Imports all flows for every project so we can register all of them. +Imports all flows for every project so we can register all of them.. """ from pipelines.egpweb_metas import * # noqa from pipelines.ergon import * # noqa diff --git a/pipelines/iptu_inad/dump_db_inadimplentes/flows.py b/pipelines/iptu_inad/dump_db_inadimplentes/flows.py index a072002..bf9fe8a 100644 --- a/pipelines/iptu_inad/dump_db_inadimplentes/flows.py +++ b/pipelines/iptu_inad/dump_db_inadimplentes/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows. +Database dumping flows.. """ from copy import deepcopy diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py index 18de2c1..7673620 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows............. +Database dumping flows.............. """ from copy import deepcopy diff --git a/pipelines/sigma/dump_db_compras_materiais/flows.py b/pipelines/sigma/dump_db_compras_materiais/flows.py index e45241c..5971813 100644 --- a/pipelines/sigma/dump_db_compras_materiais/flows.py +++ b/pipelines/sigma/dump_db_compras_materiais/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for SMFP SIGMA COMPRAS MATERIAIS. +Database dumping flows for SMFP SIGMA COMPRAS MATERIAIS.. """ from copy import deepcopy diff --git a/pipelines/sigma/dump_db_sancao_fornecedor/flows.py b/pipelines/sigma/dump_db_sancao_fornecedor/flows.py index 114f6b8..189a69c 100644 --- a/pipelines/sigma/dump_db_sancao_fornecedor/flows.py +++ b/pipelines/sigma/dump_db_sancao_fornecedor/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for SMFP SIGMA system. +Database dumping flows for SMFP SIGMA system.. """ from copy import deepcopy diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index c7281fb..b102ea8 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT. +MATERIALIZA MODELOS DO DBT.. """ from copy import deepcopy diff --git a/poetry.lock b/poetry.lock index f535ab5..1f9363b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -102,13 +102,13 @@ files = [ [[package]] name = "basedosdados" -version = "2.0.0b21" +version = "2.0.0b22" description = "Organizar e facilitar o acesso a dados brasileiros através de tabelas públicas no BigQuery." optional = false python-versions = "<4,>=3.8.1" files = [ - {file = "basedosdados-2.0.0b21-py3-none-any.whl", hash = "sha256:433b3551c8fa72b2ca13ee1a6080eca06a5c360c7b09c6da4a2f2b18046db3a2"}, - {file = "basedosdados-2.0.0b21.tar.gz", hash = "sha256:04d136dc13c9a9cc390e8092065bbd9590a08cf418bb0bcba007d2c579d20cad"}, + {file = "basedosdados-2.0.0b22-py3-none-any.whl", hash = "sha256:1de7a03eacb382fe536956cca8a28b4359d4c9c2007ce7b8e4b7993127cd1c2d"}, + {file = "basedosdados-2.0.0b22.tar.gz", hash = "sha256:87140b54c334e30dbba9328b591927df5c31f47b49d3749076b54196e833448c"}, ] [package.dependencies] @@ -2552,7 +2552,7 @@ files = [] develop = false [package.dependencies] -basedosdados = {version = "2.0.0b21", extras = ["upload"]} +basedosdados = {version = "2.0.0b22", extras = ["upload"]} croniter = {version = "^2.0.1", optional = true} cx-oracle = {version = "^8.3.0", optional = true} dask = {version = "^2023.11.0", optional = true} @@ -2580,17 +2580,17 @@ shapely = {version = "^2.0.2", optional = true} [package.extras] actions = ["ruamel-yaml (>=0.18.6,<0.19.0)"] -all = ["aiohttp (>=3.8.5,<4.0.0)", "basedosdados[upload] (==2.0.0b21)", "croniter (>=2.0.1,<3.0.0)", "cx-oracle (>=8.3.0,<9.0.0)", "dask (>=2023.11.0,<2024.0.0)", "earthengine-api (>=0.1.379,<0.2.0)", "geojsplit (>=0.1.2,<0.2.0)", "geopandas (>=0.14.1,<0.15.0)", "geopy (>=2.4.1,<3.0.0)", "google-cloud-storage (>=2.10.0,<3.0.0)", "gspread (>=5.12.4,<6.0.0)", "infisical (>=1.4.2,<1.6)", "jinja2 (>=3.1.2,<4.0.0)", "numpy (>=1.23.3,<2.0.0)", "pandas (>=2.0.3,<3.0.0)", "pendulum (>=2.1.2,<3.0.0)", "prefect (==1.4.1)", "pymongo (>=4.6.0,<5.0.0)", "pymysql (>=1.1.0,<2.0.0)", "pyodbc (>=5.0.1,<6.0.0)", "pytz (>=2023.3,<2024.0)", "redis-pal (>=1.0.0,<2.0.0)", "requests (>=2.31.0,<3.0.0)", "ruamel-yaml (>=0.18.6,<0.19.0)", "scikit-learn (>=1.1.2,<2.0.0)", "sentry-sdk (>=1.29.2,<2.0.0)", "shapely (>=2.0.2,<3.0.0)"] +all = ["aiohttp (>=3.8.5,<4.0.0)", "basedosdados[upload] (==2.0.0b22)", "croniter (>=2.0.1,<3.0.0)", "cx-oracle (>=8.3.0,<9.0.0)", "dask (>=2023.11.0,<2024.0.0)", "earthengine-api (>=0.1.379,<0.2.0)", "geojsplit (>=0.1.2,<0.2.0)", "geopandas (>=0.14.1,<0.15.0)", "geopy (>=2.4.1,<3.0.0)", "google-cloud-storage (>=2.10.0,<3.0.0)", "gspread (>=5.12.4,<6.0.0)", "infisical (>=1.4.2,<1.6)", "jinja2 (>=3.1.2,<4.0.0)", "numpy (>=1.23.3,<2.0.0)", "pandas (>=2.0.3,<3.0.0)", "pendulum (>=2.1.2,<3.0.0)", "prefect (==1.4.1)", "pymongo (>=4.6.0,<5.0.0)", "pymysql (>=1.1.0,<2.0.0)", "pyodbc (>=5.0.1,<6.0.0)", "pytz (>=2023.3,<2024.0)", "redis-pal (>=1.0.0,<2.0.0)", "requests (>=2.31.0,<3.0.0)", "ruamel-yaml (>=0.18.6,<0.19.0)", "scikit-learn (>=1.1.2,<2.0.0)", "sentry-sdk (>=1.29.2,<2.0.0)", "shapely (>=2.0.2,<3.0.0)"] metrics = ["numpy (>=1.23.3,<2.0.0)", "scikit-learn (>=1.1.2,<2.0.0)"] -pipelines = ["basedosdados[upload] (==2.0.0b21)", "dask (>=2023.11.0,<2024.0.0)", "google-cloud-storage (>=2.10.0,<3.0.0)", "infisical (>=1.4.2,<1.6)", "jinja2 (>=3.1.2,<4.0.0)", "pandas (>=2.0.3,<3.0.0)", "prefect (==1.4.1)", "sentry-sdk (>=1.29.2,<2.0.0)"] +pipelines = ["basedosdados[upload] (==2.0.0b22)", "dask (>=2023.11.0,<2024.0.0)", "google-cloud-storage (>=2.10.0,<3.0.0)", "infisical (>=1.4.2,<1.6)", "jinja2 (>=3.1.2,<4.0.0)", "pandas (>=2.0.3,<3.0.0)", "prefect (==1.4.1)", "ruamel-yaml (>=0.18.6,<0.19.0)", "sentry-sdk (>=1.29.2,<2.0.0)"] pipelines-templates = ["croniter (>=2.0.1,<3.0.0)", "cx-oracle (>=8.3.0,<9.0.0)", "earthengine-api (>=0.1.379,<0.2.0)", "geojsplit (>=0.1.2,<0.2.0)", "geopandas (>=0.14.1,<0.15.0)", "geopy (>=2.4.1,<3.0.0)", "gspread (>=5.12.4,<6.0.0)", "jinja2 (>=3.1.2,<4.0.0)", "psycopg2-binary (>=2.9.9,<3.0.0)", "pymongo (>=4.6.0,<5.0.0)", "pymysql (>=1.1.0,<2.0.0)", "pyodbc (>=5.0.1,<6.0.0)", "redis-pal (>=1.0.0,<2.0.0)", "requests (>=2.31.0,<3.0.0)", "shapely (>=2.0.2,<3.0.0)"] sgrc = ["aiohttp (>=3.8.5,<4.0.0)", "pendulum (>=2.1.2,<3.0.0)", "pytz (>=2023.3,<2024.0)", "requests (>=2.31.0,<3.0.0)"] [package.source] type = "git" url = "https://github.com/prefeitura-rio/prefeitura-rio" -reference = "ca35737d82ad16be378c68808d6d69359b753685" -resolved_reference = "ca35737d82ad16be378c68808d6d69359b753685" +reference = "43acb18c706f9bd4fe26437fd906c36eaa8f7c47" +resolved_reference = "43acb18c706f9bd4fe26437fd906c36eaa8f7c47" [[package]] name = "proto-plus" @@ -4293,4 +4293,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "13b4a3fc518fa0a7d4774a714d541da63ff5b1f6aa4df4f9d4b75079a95ca8ac" +content-hash = "f6596072acf821fb997a3c15395084968a4e9b3fc0065e1b6ae114f9826d656e" diff --git a/pyproject.toml b/pyproject.toml index 8414366..3e3018b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dbt-bigquery = "^1.6.1" google-cloud-storage = "^2.10.0" pendulum = "^3.0.0" prefect = "1.4.1" -prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "ca35737d82ad16be378c68808d6d69359b753685", extras = [ +prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "43acb18c706f9bd4fe26437fd906c36eaa8f7c47", extras = [ "pipelines", "pipelines-templates", "actions", From 309b4ec59c6c145b66c0e5dd83d1d22d98158f78 Mon Sep 17 00:00:00 2001 From: d116626 Date: Fri, 25 Oct 2024 16:46:32 -0300 Subject: [PATCH 7/8] chore: better log --- pipelines/egpweb_metas/dump_db_metas/flows.py | 2 +- pipelines/ergon/dump_db_ergon/flows.py | 2 +- pipelines/ergon/dump_db_ergon_pericia_medica/flows.py | 2 +- pipelines/ergon/materializa_funcionarios_saude/flows.py | 2 +- pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py | 2 +- pipelines/fincon/dump_db_fincon/flows.py | 2 +- pipelines/flows.py | 2 +- pipelines/iptu_inad/dump_db_inadimplentes/flows.py | 2 +- .../receita_federal_cnpj/dump_db_porte_empresa/flows.py | 2 +- pipelines/sigma/dump_db_compras_materiais/flows.py | 2 +- pipelines/sigma/dump_db_sancao_fornecedor/flows.py | 2 +- pipelines/templates/run_dbt_model/flows.py | 2 +- poetry.lock | 6 +++--- pyproject.toml | 2 +- 14 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pipelines/egpweb_metas/dump_db_metas/flows.py b/pipelines/egpweb_metas/dump_db_metas/flows.py index f7c372a..c917253 100644 --- a/pipelines/egpweb_metas/dump_db_metas/flows.py +++ b/pipelines/egpweb_metas/dump_db_metas/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for EGPWeb project. +Database dumping flows for EGPWeb project.. """ from copy import deepcopy diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index 04de8ec..3791178 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project... +Database dumping flows for segovi project.... """ from copy import deepcopy diff --git a/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py b/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py index f02eef0..5c17af4 100644 --- a/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py +++ b/pipelines/ergon/dump_db_ergon_pericia_medica/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project.. +Database dumping flows for segovi project... """ from copy import deepcopy diff --git a/pipelines/ergon/materializa_funcionarios_saude/flows.py b/pipelines/ergon/materializa_funcionarios_saude/flows.py index 8f00717..0d5bb1f 100644 --- a/pipelines/ergon/materializa_funcionarios_saude/flows.py +++ b/pipelines/ergon/materializa_funcionarios_saude/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Materialize Active SMS Employees from Ergon... +Materialize Active SMS Employees from Ergon.... """ from copy import deepcopy diff --git a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py index 7216488..94dadfd 100644 --- a/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py +++ b/pipelines/ergon_comlurb/dump_db_ergon_comlurb/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for smfp ergon comlurb....... +Database dumping flows for smfp ergon comlurb........ """ from copy import deepcopy diff --git a/pipelines/fincon/dump_db_fincon/flows.py b/pipelines/fincon/dump_db_fincon/flows.py index 51708d7..d0db668 100644 --- a/pipelines/fincon/dump_db_fincon/flows.py +++ b/pipelines/fincon/dump_db_fincon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for SMFP FINCON.. +Database dumping flows for SMFP FINCON... """ from copy import deepcopy diff --git a/pipelines/flows.py b/pipelines/flows.py index 8ca8b5e..ff4031c 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Imports all flows for every project so we can register all of them.. +Imports all flows for every project so we can register all of them... """ from pipelines.atividade_economica import * # noqa from pipelines.egpweb_metas import * # noqa diff --git a/pipelines/iptu_inad/dump_db_inadimplentes/flows.py b/pipelines/iptu_inad/dump_db_inadimplentes/flows.py index bf9fe8a..47ffc37 100644 --- a/pipelines/iptu_inad/dump_db_inadimplentes/flows.py +++ b/pipelines/iptu_inad/dump_db_inadimplentes/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows.. +Database dumping flows... """ from copy import deepcopy diff --git a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py index 7673620..67b15bb 100644 --- a/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py +++ b/pipelines/receita_federal_cnpj/dump_db_porte_empresa/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows.............. +Database dumping flows............... """ from copy import deepcopy diff --git a/pipelines/sigma/dump_db_compras_materiais/flows.py b/pipelines/sigma/dump_db_compras_materiais/flows.py index 5971813..c8676a0 100644 --- a/pipelines/sigma/dump_db_compras_materiais/flows.py +++ b/pipelines/sigma/dump_db_compras_materiais/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for SMFP SIGMA COMPRAS MATERIAIS.. +Database dumping flows for SMFP SIGMA COMPRAS MATERIAIS... """ from copy import deepcopy diff --git a/pipelines/sigma/dump_db_sancao_fornecedor/flows.py b/pipelines/sigma/dump_db_sancao_fornecedor/flows.py index 189a69c..1e8b371 100644 --- a/pipelines/sigma/dump_db_sancao_fornecedor/flows.py +++ b/pipelines/sigma/dump_db_sancao_fornecedor/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for SMFP SIGMA system.. +Database dumping flows for SMFP SIGMA system... """ from copy import deepcopy diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index af05a03..841a358 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT.. +MATERIALIZA MODELOS DO DBT... """ from copy import deepcopy diff --git a/poetry.lock b/poetry.lock index 1f9363b..17bf311 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2589,8 +2589,8 @@ sgrc = ["aiohttp (>=3.8.5,<4.0.0)", "pendulum (>=2.1.2,<3.0.0)", "pytz (>=2023.3 [package.source] type = "git" url = "https://github.com/prefeitura-rio/prefeitura-rio" -reference = "43acb18c706f9bd4fe26437fd906c36eaa8f7c47" -resolved_reference = "43acb18c706f9bd4fe26437fd906c36eaa8f7c47" +reference = "cd69b6d30b6c4b187ba4d72e4d956e8c1b4d8072" +resolved_reference = "cd69b6d30b6c4b187ba4d72e4d956e8c1b4d8072" [[package]] name = "proto-plus" @@ -4293,4 +4293,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "f6596072acf821fb997a3c15395084968a4e9b3fc0065e1b6ae114f9826d656e" +content-hash = "0b064e4845a6d69412aab603f21da69b1965e82bc4bda556dbc8a72b895cc2dd" diff --git a/pyproject.toml b/pyproject.toml index 3e3018b..7bfe05c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dbt-bigquery = "^1.6.1" google-cloud-storage = "^2.10.0" pendulum = "^3.0.0" prefect = "1.4.1" -prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "43acb18c706f9bd4fe26437fd906c36eaa8f7c47", extras = [ +prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "cd69b6d30b6c4b187ba4d72e4d956e8c1b4d8072", extras = [ "pipelines", "pipelines-templates", "actions", From 52eda4103aa460563fb9fa0ea67c99552f68129f Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 30 Oct 2024 14:52:53 -0300 Subject: [PATCH 8/8] feat: add agent_label parameter to dump_db flow --- pipelines/ergon/dump_db_ergon/flows.py | 2 +- pipelines/ergon/dump_db_ergon/schedules.py | 3 ++- poetry.lock | 6 +++--- pyproject.toml | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pipelines/ergon/dump_db_ergon/flows.py b/pipelines/ergon/dump_db_ergon/flows.py index 3791178..838c12a 100644 --- a/pipelines/ergon/dump_db_ergon/flows.py +++ b/pipelines/ergon/dump_db_ergon/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Database dumping flows for segovi project.... +Database dumping flows for segovi project..... """ from copy import deepcopy diff --git a/pipelines/ergon/dump_db_ergon/schedules.py b/pipelines/ergon/dump_db_ergon/schedules.py index c89007d..2630a3c 100644 --- a/pipelines/ergon/dump_db_ergon/schedules.py +++ b/pipelines/ergon/dump_db_ergon/schedules.py @@ -599,7 +599,7 @@ interval=timedelta(days=1), start_date=datetime(2022, 11, 9, 22, 30, tzinfo=pytz.timezone("America/Sao_Paulo")), labels=[ - constants.RJ_SMFP_AGENT_LABEL.value, + constants.RJ_LOCAL_IPLAN_AGENT_LABEL.value, ], db_database="P01.PCRJ", db_host="10.70.6.21", @@ -608,6 +608,7 @@ dataset_id="recursos_humanos_ergon", infisical_secret_path="/db-ergon-prod", table_parameters=ergon_queries, + agent_label=constants.RJ_SMFP_AGENT_LABEL.value, ) ergon_monthly_update_schedule = Schedule(clocks=untuple(ergon_clocks)) diff --git a/poetry.lock b/poetry.lock index 17bf311..f0ce0a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2589,8 +2589,8 @@ sgrc = ["aiohttp (>=3.8.5,<4.0.0)", "pendulum (>=2.1.2,<3.0.0)", "pytz (>=2023.3 [package.source] type = "git" url = "https://github.com/prefeitura-rio/prefeitura-rio" -reference = "cd69b6d30b6c4b187ba4d72e4d956e8c1b4d8072" -resolved_reference = "cd69b6d30b6c4b187ba4d72e4d956e8c1b4d8072" +reference = "f04d50d26732705f77a56db0124d5828af6a501d" +resolved_reference = "f04d50d26732705f77a56db0124d5828af6a501d" [[package]] name = "proto-plus" @@ -4293,4 +4293,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "0b064e4845a6d69412aab603f21da69b1965e82bc4bda556dbc8a72b895cc2dd" +content-hash = "23fd76f5ed0f4f45d1ae57858b422a2598aac9bee697a2db25d07ccb7ec0f658" diff --git a/pyproject.toml b/pyproject.toml index 7bfe05c..ebbd503 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dbt-bigquery = "^1.6.1" google-cloud-storage = "^2.10.0" pendulum = "^3.0.0" prefect = "1.4.1" -prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "cd69b6d30b6c4b187ba4d72e4d956e8c1b4d8072", extras = [ +prefeitura-rio = {git = "https://github.com/prefeitura-rio/prefeitura-rio", rev = "f04d50d26732705f77a56db0124d5828af6a501d", extras = [ "pipelines", "pipelines-templates", "actions",