From 70514f3448a8256be68e370541c88e4ba7d8e8b9 Mon Sep 17 00:00:00 2001 From: Rex Ledesma Date: Mon, 16 Oct 2023 15:19:35 -0400 Subject: [PATCH] fix(dbt): skip copying the partial parse file if it exists in the target (#17214) ## Summary & Motivation As the title. From https://docs.python.org/3/library/shutil.html#shutil.copy: > Copies the file src to the file or directory dst. src and dst should be path-like objects or strings. If dst specifies a directory, the file will be copied into dst using the base filename from src. If dst specifies a file that already exists, it will be replaced. Returns the path to the newly created file. But in windows, this information is false, as an error pops up if the file at the destination already exists. In that case, skip the copy. ## How I Tested These Changes pytest, add assertion that partial parse file is not overwritten if it already exists --- .../dagster_dbt/core/resources_v2.py | 2 +- .../core/test_resources_v2.py | 26 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/core/resources_v2.py b/python_modules/libraries/dagster-dbt/dagster_dbt/core/resources_v2.py index 1bc31f2dd37fb..5b5c85e9284a7 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/core/resources_v2.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/core/resources_v2.py @@ -231,7 +231,7 @@ def run( ) partial_parse_destination_target_path = target_path.joinpath(PARTIAL_PARSE_FILE_NAME) - if partial_parse_file_path.exists(): + if partial_parse_file_path.exists() and not partial_parse_destination_target_path.exists(): logger.info( f"Copying `{partial_parse_file_path}` to `{partial_parse_destination_target_path}`" " to take advantage of partial parsing." diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resources_v2.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resources_v2.py index 6575fd03ad7ca..07cf3669b6ef4 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resources_v2.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resources_v2.py @@ -262,10 +262,15 @@ def test_dbt_with_partial_parse() -> None: original_target_path = Path(TEST_PROJECT_DIR, "target", PARTIAL_PARSE_FILE_NAME) original_target_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(partial_parse_file_path, Path(TEST_PROJECT_DIR, "target", PARTIAL_PARSE_FILE_NAME)) + shutil.copy(partial_parse_file_path, original_target_path) # Assert that partial parsing was used. dbt_cli_compile_with_partial_parse_invocation = dbt.cli(["compile"]) + partial_parse_original_st_mtime = ( + dbt_cli_compile_with_partial_parse_invocation.target_path.joinpath(PARTIAL_PARSE_FILE_NAME) + .stat() + .st_mtime + ) assert dbt_cli_compile_with_partial_parse_invocation.is_successful() assert not any( @@ -273,6 +278,25 @@ def test_dbt_with_partial_parse() -> None: for event in dbt_cli_compile_with_partial_parse_invocation.stream_raw_events() ) + # Assert that partial parsing is continues to happen when the target directory is reused. + dbt_cli_compile_with_reused_partial_parse_invocation = dbt.cli( + ["compile"], target_path=dbt_cli_compile_with_partial_parse_invocation.target_path + ) + partial_parse_new_st_mtime = ( + dbt_cli_compile_with_reused_partial_parse_invocation.target_path.joinpath( + PARTIAL_PARSE_FILE_NAME + ) + .stat() + .st_mtime + ) + + assert partial_parse_original_st_mtime == partial_parse_new_st_mtime + assert dbt_cli_compile_with_reused_partial_parse_invocation.is_successful() + assert not any( + "Unable to do partial parsing" in event.raw_event["info"]["msg"] + for event in dbt_cli_compile_with_reused_partial_parse_invocation.stream_raw_events() + ) + def test_dbt_cli_debug_execution() -> None: @dbt_assets(manifest=manifest)