Skip to content

Commit

Permalink
fix(dbt): skip copying the partial parse file if it exists in the tar…
Browse files Browse the repository at this point in the history
…get (#17214)

## Summary & Motivation
As the title.

From https://docs.python.org/3/library/shutil.html#shutil.copy:

> Copies the file src to the file or directory dst. src and dst should
be path-like objects or strings. If dst specifies a directory, the file
will be copied into dst using the base filename from src. If dst
specifies a file that already exists, it will be replaced. Returns the
path to the newly created file.

But in windows, this information is false, as an error pops up if the
file at the destination already exists. In that case, skip the copy.

## How I Tested These Changes
pytest, add assertion that partial parse file is not overwritten if it
already exists
  • Loading branch information
rexledesma authored Oct 16, 2023
1 parent 6f998ba commit 70514f3
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def run(
)
partial_parse_destination_target_path = target_path.joinpath(PARTIAL_PARSE_FILE_NAME)

if partial_parse_file_path.exists():
if partial_parse_file_path.exists() and not partial_parse_destination_target_path.exists():
logger.info(
f"Copying `{partial_parse_file_path}` to `{partial_parse_destination_target_path}`"
" to take advantage of partial parsing."
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -262,17 +262,41 @@ def test_dbt_with_partial_parse() -> None:
original_target_path = Path(TEST_PROJECT_DIR, "target", PARTIAL_PARSE_FILE_NAME)

original_target_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(partial_parse_file_path, Path(TEST_PROJECT_DIR, "target", PARTIAL_PARSE_FILE_NAME))
shutil.copy(partial_parse_file_path, original_target_path)

# Assert that partial parsing was used.
dbt_cli_compile_with_partial_parse_invocation = dbt.cli(["compile"])
partial_parse_original_st_mtime = (
dbt_cli_compile_with_partial_parse_invocation.target_path.joinpath(PARTIAL_PARSE_FILE_NAME)
.stat()
.st_mtime
)

assert dbt_cli_compile_with_partial_parse_invocation.is_successful()
assert not any(
"Unable to do partial parsing" in event.raw_event["info"]["msg"]
for event in dbt_cli_compile_with_partial_parse_invocation.stream_raw_events()
)

# Assert that partial parsing is continues to happen when the target directory is reused.
dbt_cli_compile_with_reused_partial_parse_invocation = dbt.cli(
["compile"], target_path=dbt_cli_compile_with_partial_parse_invocation.target_path
)
partial_parse_new_st_mtime = (
dbt_cli_compile_with_reused_partial_parse_invocation.target_path.joinpath(
PARTIAL_PARSE_FILE_NAME
)
.stat()
.st_mtime
)

assert partial_parse_original_st_mtime == partial_parse_new_st_mtime
assert dbt_cli_compile_with_reused_partial_parse_invocation.is_successful()
assert not any(
"Unable to do partial parsing" in event.raw_event["info"]["msg"]
for event in dbt_cli_compile_with_reused_partial_parse_invocation.stream_raw_events()
)


def test_dbt_cli_debug_execution() -> None:
@dbt_assets(manifest=manifest)
Expand Down

0 comments on commit 70514f3

Please sign in to comment.