Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix S3 Upload Errors #338

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix S3 Upload Errors #338

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7c2adbd
Don't move to AIP to completed dir if S3 enabled
camlyall Apr 21, 2023
1ae9753
fix missing pathlib import. fix missing aip file extension.
camlyall Apr 21, 2023
7911c20
remove logging s3_path
camlyall Apr 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions a3m/client/clientScripts/a3m_store_aip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from a3m.client import metrics

from pathlib import Path

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -36,7 +37,13 @@ def _upload_file(path, bucket, key):
boto_args.update(config=config)

s3 = boto3.resource(**boto_args)
s3.meta.client.upload_file(path, bucket, key)

# TODO: The S3 path should include a pre-defined directory path for the AIP
# We could use a predefined variable in the configuration file for a global A3M Storage such as predefined/global/path/<Transfer Source/DIPS/AIPS>/
# Alternatively, we could use a predefined variable specifically for the AIP location such as predefined/aip/location/
# In any case, the S3 path must be structured in a way that makes it easy to locate and manage the AIP.
s3_path = key + path.suffix
s3.meta.client.upload_file(str(path), bucket, s3_path)


def _store_aip(job, sip_id, aip_path):
Expand All @@ -51,13 +58,13 @@ def _store_aip(job, sip_id, aip_path):
raise Exception("AIP is a directory")

logger.info("Uploading AIP...")
_upload_file(str(aip_path), settings.S3_BUCKET, sip_id)
_upload_file(aip_path, settings.S3_BUCKET, sip_id)


def call(jobs):
job = jobs[0]
with transaction.atomic():
with job.JobContext():
sip_id = job.args[1]
aip_path = job.args[2]
aip_path = Path(job.args[2])
job.set_status(_store_aip(job, sip_id, aip_path))
10 changes: 6 additions & 4 deletions a3m/client/clientScripts/verify_aip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,10 +226,12 @@ def verify_aip(job):
file=sys.stderr,
)

aip_path = Path(aip_path)
completed_dir = Path(mcpclient_settings.SHARED_DIRECTORY, "completed")
shutil.move(str(aip_path), str(completed_dir))
logger.info("AIP generated: %s", aip_path.name)
# Don't move to completed if S3 enabled
if not mcpclient_settings.S3_ENABLED:
aip_path = Path(aip_path)
completed_dir = Path(mcpclient_settings.SHARED_DIRECTORY, "completed")
shutil.move(str(aip_path), str(completed_dir))
logger.info("AIP generated: %s", aip_path.name)

return return_code

Expand Down