Merge branch 'main' into fixing-the-dictionarys

basedosdados · Jun 13, 2024 · 158d96c · 158d96c
2 parents aba7332 + bb250c8
commit 158d96c
Show file tree

Hide file tree

Showing 8 changed files with 10,162 additions and 5 deletions.
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -234,6 +234,9 @@ models:
     br_me_sic:
       +materialized: table
       +schema: br_me_sic
+    br_me_siconfi:
+      +materialized: table
+      +schema: br_me_siconfi
     br_mec_prouni:
       +materialized: table
       +schema: br_mec_prouni
@@ -261,6 +264,9 @@ models:
     br_ms_cnes:
       +materialized: table
       +schema: br_ms_cnes
+    br_ms_pns:
+      +materialized: table
+      +schema: br_ms_pns
     br_ms_sia:
       +materialized: table
       +schema: br_ms_sia
@@ -321,6 +327,9 @@ models:
     mundo_transfermarkt_competicoes_internacionais:
       +materialized: table
       +schema: mundo_transfermarkt_competicoes_internacionais
+    test:
+      +materialized: table
+      +schema: test
     world_ampas_oscar:
       +materialized: table
       +schema: world_ampas_oscar

diff --git a/models/br_bcb_agencia/br_bcb_agencia__agencia.sql b/models/br_bcb_agencia/br_bcb_agencia__agencia.sql
@@ -6,12 +6,12 @@
         partition_by={
             "field": "ano",
             "data_type": "int64",
-            "range": {"start": 2007, "end": 2024, "interval": 1},
+            "range": {"start": 2007, "end": 2025, "interval": 1},
         },
         pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
         post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
+            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
         ],
     )
 }}
@@ -201,6 +201,10 @@ with
             ano,
             mes
         from `basedosdados-dev.br_bcb_agencia_staging.agencia` as t
+        -- os arquivos mensais possuem cabeçalhos e rodapés que variam de posição;
+        -- Este filtro remove linhas com valores intereiramente
+        -- nulos
+        where fone = '00000nan'
     )
 
 select
@@ -225,6 +229,8 @@ select
 from wrang_data
 {% if is_incremental() %}
     where
-        date(cast(ano as int64), cast(mes as int64), 1)
-        > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
+        date(cast(ano as int64), cast(mes as int64), 1) not in (
+            select distinct (date(cast(ano as int64), cast(mes as int64), 1))
+            from {{ this }}
+        )
 {% endif %}
diff --git a/models/br_bcb_agencia/schema.yml b/models/br_bcb_agencia/schema.yml
@@ -7,13 +7,17 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns: [ano, mes, cnpj]
+          config:
+            where: __most_recent_year_month__
     columns:
       - name: ano
         description: Ano
         tests:
           - relationships:
               to: ref('br_bd_diretorios_data_tempo__data')
               field: ano
+              config:
+                where: __most_recent_year_month__
       - name: mes
         description: Mês
         tests:
@@ -25,6 +29,8 @@ models:
         tests:
           - dbt_utils.not_null_proportion:
               at_least: 0.05
+              config:
+                where: __most_recent_year_month__
       - name: id_municipio
         description: ID Município - IBGE 7 Dígitos
         tests:
@@ -33,13 +39,17 @@ models:
               field: id_municipio
           - dbt_utils.not_null_proportion:
               at_least: 0.05
+              config:
+                where: __most_recent_year_month__
       - name: data_inicio
         description: Data de criação da agência
       - name: cnpj
         description: Cadastro Nacional de Pessoa Júridica (CNPJ)
         tests:
           - dbt_utils.not_null_proportion:
               at_least: 0.05
+              config:
+                where: __most_recent_year_month__
       - name: nome_agencia
         description: Nome da Agência
       - name: instituicao
@@ -58,8 +68,12 @@ models:
           - relationships:
               to: ref('br_bd_diretorios_brasil__cep')
               field: cep
+              config:
+                where: __most_recent_year_month__
           - dbt_utils.not_null_proportion:
               at_least: 0.05
+              config:
+                where: __most_recent_year_month__
       - name: endereco
         description: endereço da agência
       - name: complemento