addind some adm_licenca_urbanismo queries

.github/workflows/scripts/register_flows.py

@@ -91,11 +91,7 @@ def build_and_register(  # pylint: disable=too-many-branches
                 while attempts < max_retries:
                     attempts += 1
                     try:
-                        (
-                            flow_id,
-                            flow_version,
-                            is_new,
-                        ) = register_serialized_flow(
+                        (flow_id, flow_version, is_new,) = register_serialized_flow(
                             client=client,
                             serialized_flow=serialized_flow,
                             project_id=project_id,

pipelines/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+from pipelines.silic.dump_db_sislic.flows import *  # noqa

pipelines/constants.py

@@ -16,7 +16,7 @@ class constants(Enum):
     ######################################
     # Agent labels
     ######################################
-    # EXAMPLE_AGENT_LABEL = "example_agent"
+    RJ_SMDUE_AGENT_LABEL = "smdue"
 
     ######################################
     # Other constants

pipelines/flows.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 """
-Imports all flows for every project so we can register all of them.
+Imports all flows for every project so we can register all of them
 """
-from pipelines.exemplo import *  # noqa
+from pipelines.sislic import *  # noqa
+from pipelines.templates import *  # noqa

pipelines/sislic/dump_db_sislic/flows.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+Database dumping flows for segovi project (SISLIC)
+"""
+
+from copy import deepcopy
+
+from prefect.run_configs import KubernetesRun
+from prefect.storage import GCS
+
+from prefeitura_rio.pipelines_templates.dump_db.flows import flow as dump_sql_flow
+from prefeitura_rio.pipelines_utils.prefect import set_default_parameters
+from prefeitura_rio.pipelines_utils.state_handlers import (
+    handler_initialize_sentry,
+    handler_inject_bd_credentials,
+)
+
+from pipelines.constants import constants
+from pipelines.sislic.dump_db_sislic.schedules import (
+    update_schedule,
+)
+
+dump_sislic_flow = deepcopy(dump_sql_flow)
+dump_sislic_flow.name = "SMDUE: SISLIC - Ingerir tabelas de banco SQL"
+dump_sislic_flow.state_handlers = [handler_inject_bd_credentials, handler_initialize_sentry]
+dump_sislic_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value)
+dump_sislic_flow.run_config = KubernetesRun(
+    image=constants.DOCKER_IMAGE.value,
+    labels=[
+        constants.RJ_SMDUE_AGENT_LABEL.value,
+    ],
+)
+
+sislic_default_parameters = {
+    "db_database": "SMU_PRD",
+    "db_host": "10.2.221.101",
+    "db_port": "1433",
+    "db_type": "sql_server",
+    "infisical_secret_path": "/db_sislic",
+    "dataset_id": "adm_licenca_urbanismo",
+}
+dump_sislic_flow = set_default_parameters(
+    dump_sislic_flow, default_parameters=sislic_default_parameters
+)
+
+dump_sislic_flow.schedule = update_schedule

pipelines/sislic/dump_db_sislic/schedules.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""
+Schedules for the database dump pipeline
+"""
+
+from datetime import timedelta, datetime
+
+from prefect.schedules import Schedule
+import pytz
+
+from prefeitura_rio.pipelines_utils.io import untuple_clocks as untuple
+from prefeitura_rio.pipelines_utils.prefect import generate_dump_db_schedules
+
+from pipelines.constants import constants
+
+sislic_queries = {
+    "documento": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            SELECT
+                codDocumento,
+                numDocumento,
+                codOrgao,
+                codTipoDocumento,
+                MatricCadastrador,
+                dtAbertura,
+                dtCadastroSistema,
+                codAssunto,
+                codDocumentoOrigem,
+                descStatus,
+                ID_CLASSIFICACAO_PROCESSO,
+                ID_ORIGEM_CLASSIFICACAO_PROCESSO,
+                ID_TIPO_PROCESSO,
+                ANO_REQUERIMENTO,
+                NUMERO_REQUERIMENTO
+            FROM SMU_PRD.dbo.tbPTI_Documentos
+            """,
+    },
+    "tipo_documento": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            SELECT
+                codTipoDocumento,
+                descTipoDocumento
+            FROM SMU_PRD.dbo.tbPTI_TiposDocumentos
+            """,
+    },
+    "assunto_processo": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            SELECT
+                codAssuntoN1,
+                codAssuntoN2,
+                codAssuntoN3,
+                codAssunto,
+                Status,
+                codAssunto_ReqOnLine,
+                Exige_Endereco,
+                Exige_ProcessoConstrucao,
+                Exige_CPF_CNPJ_Requerente,
+                AssuntoProcessoRio
+            FROM SMU_PRD.dbo.tbPTI_Assunto
+            """,
+    },
+    # "cronograma_financeiro": {
+    #     "materialize_after_dump": True,
+    #     "materialization_mode": "prod",
+    #     "dump_mode": "overwrite",
+    #     "execute_query": """
+    #         Select
+    #             DISTINCT
+    #                 CD_OBRA,
+    #                 ETAPA,
+    #                 DT_INICIO_ETAPA,
+    #                 DT_FIM_ETAPA,
+    #                 PC_PERCENTUAL,
+    #                 VL_ESTIMADO
+    #         from dbo.fuSEGOVI_Cronograma_Financeiro()
+    #         """,
+    # },
+    # "localizacao": {
+    #     "materialize_after_dump": True,
+    #     "materialization_mode": "prod",
+    #     "dump_mode": "overwrite",
+    #     "execute_query": """
+    #         Select
+    #             CD_OBRA,
+    #             ENDERECO,
+    #             NM_BAIRRO,
+    #             NM_RA,
+    #             NM_AP
+    #         from dbo.fuSEGOVI_Localizacoes_obra()
+    #         """,
+    # },
+    # "cronograma_alteracao": {
+    #     "materialize_after_dump": True,
+    #     "materialization_mode": "prod",
+    #     "dump_mode": "overwrite",
+    #     "execute_query": """
+    #         Select
+    #             DISTINCT
+    #                 CD_OBRA,
+    #                 NR_PROCESSO,
+    #                 TP_ALTERACAO,
+    #                 DT_PUBL_DO,
+    #                 CD_ETAPA,
+    #                 NR_PRAZO,
+    #                 DT_VALIDADE,
+    #                 DS_OBSERVACAO
+    #         from dbo.fuSEGOVI_Alteração_de_Cronograma()
+    #         """,
+    # },
+    # "programa_fonte": {
+    #     "materialize_after_dump": True,
+    #     "materialization_mode": "prod",
+    #     "dump_mode": "overwrite",
+    #     "execute_query": """
+    #         Select
+    #             DISTINCT
+    #                 CD_OBRA,
+    #                 CD_PRG_TRAB,
+    #                 PROGRAMA_TRABALHO,
+    #                 CD_FONTE_RECURSO,
+    #                 FONTE_RECURSO,
+    #                 CD_NATUREZA_DSP,
+    #                 NATUREZA_DESPESA
+    #         from dbo.fuSEGOVI_Programa_Fonte()
+    #         """,
+    # },
+}
+
+sislic_clocks = generate_dump_db_schedules(
+    interval=timedelta(days=1),
+    start_date=datetime(2022, 12, 19, 1, 0, tzinfo=pytz.timezone("America/Sao_Paulo")),
+    labels=[
+        constants.RJ_SMDUE_AGENT_LABEL.value,
+    ],
+    db_database="SMU_PRD",
+    db_host="10.2.221.101",
+    db_port="1433",
+    db_type="sql_server",
+    dataset_id="adm_licenca_urbanismo",
+    infisical_secret_path="/db_sislic",
+    table_parameters=sislic_queries,
+)
+
+update_schedule = Schedule(clocks=untuple(sislic_clocks))

pipelines/templates/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+from pipelines.templates.run_dbt_model.flows import *  # noqa: F401, F403

pipelines/templates/run_dbt_model/flows.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+"""
+MATERIALIZA MODELOS DO DBT.
+"""
+
+from copy import deepcopy
+
+from prefect.run_configs import KubernetesRun
+from prefect.storage import GCS
+from prefeitura_rio.pipelines_templates.run_dbt_model.flows import (
+    templates__run_dbt_model__flow,
+)
+from prefeitura_rio.pipelines_utils.prefect import set_default_parameters
+from prefeitura_rio.pipelines_utils.state_handlers import handler_inject_bd_credentials
+
+from pipelines.constants import constants
+
+templates__run_dbt_model_smi__flow = deepcopy(templates__run_dbt_model__flow)
+templates__run_dbt_model_smi__flow.state_handlers = [handler_inject_bd_credentials]
+
+templates__run_dbt_model_smi__flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value)
+templates__run_dbt_model_smi__flow.run_config = KubernetesRun(
+    image=constants.DOCKER_IMAGE.value,
+    labels=[
+        constants.RJ_SMI_AGENT_LABEL.value,
+    ],
+)
+
+templates_run_dbt_model_smi_default_parameters = {
+    "dataset_id": "dataset_id",
+    "table_id": "table_id",
+}
+templates__run_dbt_model_smi__flow = set_default_parameters(
+    templates__run_dbt_model_smi__flow,
+    default_parameters=templates_run_dbt_model_smi_default_parameters,
+)