Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Commit

Permalink
Add as_posix to s3 push file paths in deployment step (#314)
Browse files Browse the repository at this point in the history
  • Loading branch information
markbruning authored Oct 3, 2023
1 parent 013d0b4 commit 50ad8f8
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 1 deletion.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Changed

- Changed `push_to_s3` deployment step function to write paths `as_posix()` to allow support for deploying from windows [#314](https://github.com/PrefectHQ/prefect-aws/pull/314)

### Deprecated

### Removed
Expand Down
4 changes: 3 additions & 1 deletion prefect_aws/deployments/steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ def push_to_s3(
continue
elif not local_file_path.is_dir():
remote_file_path = Path(folder) / local_file_path.relative_to(local_path)
client.upload_file(str(local_file_path), bucket, str(remote_file_path))
client.upload_file(
str(local_file_path), bucket, str(remote_file_path.as_posix())
)

return {
"bucket": bucket,
Expand Down
42 changes: 42 additions & 0 deletions tests/deploments/test_steps.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import sys
from pathlib import Path, PurePath, PurePosixPath

import boto3
Expand Down Expand Up @@ -40,6 +41,24 @@ def tmp_files(tmp_path: Path):
return tmp_path


@pytest.fixture
def tmp_files_win(tmp_path: Path):
files = [
"testfile1.txt",
"testfile2.txt",
"testfile3.txt",
r"testdir1\testfile4.txt",
r"testdir2\testfile5.txt",
]

for file in files:
filepath = tmp_path / file
filepath.parent.mkdir(parents=True, exist_ok=True)
filepath.write_text("Sample text")

return tmp_path


@pytest.fixture
def mock_aws_credentials(monkeypatch):
# Set mock environment variables for AWS credentials
Expand Down Expand Up @@ -77,6 +96,29 @@ def test_push_to_s3(s3_setup, tmp_files, mock_aws_credentials):
assert set(object_keys) == set(expected_keys)


@pytest.mark.skipif(sys.platform != "win32", reason="requires Windows")
def test_push_to_s3_as_posix(s3_setup, tmp_files_win, mock_aws_credentials):
s3, bucket_name = s3_setup
folder = "my-project"

os.chdir(tmp_files_win)

push_to_s3(bucket_name, folder)

s3_objects = s3.list_objects_v2(Bucket=bucket_name)
object_keys = [item["Key"] for item in s3_objects["Contents"]]

expected_keys = [
f"{folder}/testfile1.txt",
f"{folder}/testfile2.txt",
f"{folder}/testfile3.txt",
f"{folder}/testdir1/testfile4.txt",
f"{folder}/testdir2/testfile5.txt",
]

assert set(object_keys) == set(expected_keys)


def test_pull_from_s3(s3_setup, tmp_path, mock_aws_credentials):
s3, bucket_name = s3_setup
folder = "my-project"
Expand Down

0 comments on commit 50ad8f8

Please sign in to comment.