From 6fe3e52d951db1a472ba91a7a0f9582ad2bd0c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Wed, 17 Apr 2024 11:14:43 -0300 Subject: [PATCH 1/2] feat: add check bq project name action --- .github/workflows/check-bq-project-name.yml | 38 +++++++++++++++++++ .github/workflows/scripts/check_sql_files.py | 23 +++++++++++ ...beneficios_cidadao__novo_bolsa_familia.sql | 2 +- 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/check-bq-project-name.yml create mode 100644 .github/workflows/scripts/check_sql_files.py diff --git a/.github/workflows/check-bq-project-name.yml b/.github/workflows/check-bq-project-name.yml new file mode 100644 index 00000000..2b481962 --- /dev/null +++ b/.github/workflows/check-bq-project-name.yml @@ -0,0 +1,38 @@ +name: Check BQ project name +on: + workflow_dispatch: + pull_request: + paths: + - '**/*.sql' + +jobs: + check_bucket_name: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Get changed files + id: get_files + uses: dorny/paths-filter@v2 + with: + list-files: shell + filters: | + pr: + - added|deleted|modified: '**' + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Run Python script + run: | + for file in ${{ steps.get_files.outputs.pr_files }}; do + if [[ $file == *.sql ]]; then + echo "SQL file detected: $file" + python .github/workflows/scripts/check_sql_files.py $file + else + echo "Não é um arquivo SQL: $file" + fi + done \ No newline at end of file diff --git a/.github/workflows/scripts/check_sql_files.py b/.github/workflows/scripts/check_sql_files.py new file mode 100644 index 00000000..b30fb186 --- /dev/null +++ b/.github/workflows/scripts/check_sql_files.py @@ -0,0 +1,23 @@ +import argparse + +def check_sql_files(file): + found_staging = False + if file.endswith(".sql"): + with open(file, "r") as f: + lines = f.readlines() + for line in lines: + if "basedosdados-dev" in line: + found_staging = True + print(f"Found 'basedosdados-dev' in {file}") + break + return found_staging + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Check for 'basedosdados-dev' occurrences in SQL files.") + parser.add_argument("file", help="Path to the SQL file to check") + args = parser.parse_args() + + if check_sql_files(args.file): + exit(1) + else: + print("No occurrences of 'basedosdados-dev' found in SQL files.") \ No newline at end of file diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql index f95796b4..5cc86045 100644 --- a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql +++ b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql @@ -16,7 +16,7 @@ ], ) }} -select +select safe_cast(substr(mes_competencia, 1, 4) as int64) ano_competencia, safe_cast(substr(mes_competencia, 5, 2) as int64) mes_competencia, safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia, From 5f9ee9c13828f7272a7dab8b924bb71d6b0a8476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Wed, 17 Apr 2024 11:17:25 -0300 Subject: [PATCH 2/2] fix: fix project name --- .../br_cgu_beneficios_cidadao__novo_bolsa_familia.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql index 5cc86045..51515733 100644 --- a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql +++ b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql @@ -16,7 +16,7 @@ ], ) }} -select +select safe_cast(substr(mes_competencia, 1, 4) as int64) ano_competencia, safe_cast(substr(mes_competencia, 5, 2) as int64) mes_competencia, safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia, @@ -27,7 +27,7 @@ select safe_cast(nis as string) nis_favorecido, safe_cast(t1.nome as string) nome_favorecido, safe_cast(valor as float64) valor_parcela, -from `basedosdados-dev.br_cgu_beneficios_cidadao_staging.novo_bolsa_familia` t1 +from `basedosdados-staging.br_cgu_beneficios_cidadao_staging.novo_bolsa_familia` t1 left join `basedosdados.br_bd_diretorios_brasil.municipio` t2 on safe_cast(t1.id_municipio_siafi as int64)