From 7c2adbd4facdc2fcab2b761c3976d19680de4cf4 Mon Sep 17 00:00:00 2001 From: camlyall Date: Fri, 21 Apr 2023 11:15:38 +0000 Subject: [PATCH 1/3] Don't move to AIP to completed dir if S3 enabled --- a3m/client/clientScripts/verify_aip.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/a3m/client/clientScripts/verify_aip.py b/a3m/client/clientScripts/verify_aip.py index b568cb1af..10b460582 100644 --- a/a3m/client/clientScripts/verify_aip.py +++ b/a3m/client/clientScripts/verify_aip.py @@ -226,10 +226,12 @@ def verify_aip(job): file=sys.stderr, ) - aip_path = Path(aip_path) - completed_dir = Path(mcpclient_settings.SHARED_DIRECTORY, "completed") - shutil.move(str(aip_path), str(completed_dir)) - logger.info("AIP generated: %s", aip_path.name) + # Don't move to completed if S3 enabled + if not mcpclient_settings.S3_ENABLED: + aip_path = Path(aip_path) + completed_dir = Path(mcpclient_settings.SHARED_DIRECTORY, "completed") + shutil.move(str(aip_path), str(completed_dir)) + logger.info("AIP generated: %s", aip_path.name) return return_code From 1ae97532034508cd324e74df3ba25d893f903a29 Mon Sep 17 00:00:00 2001 From: camlyall Date: Fri, 21 Apr 2023 11:16:58 +0000 Subject: [PATCH 2/3] fix missing pathlib import. fix missing aip file extension. --- a3m/client/clientScripts/a3m_store_aip.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/a3m/client/clientScripts/a3m_store_aip.py b/a3m/client/clientScripts/a3m_store_aip.py index beb005912..bd0886f84 100644 --- a/a3m/client/clientScripts/a3m_store_aip.py +++ b/a3m/client/clientScripts/a3m_store_aip.py @@ -8,6 +8,7 @@ from a3m.client import metrics +from pathlib import Path logger = logging.getLogger(__name__) @@ -36,7 +37,15 @@ def _upload_file(path, bucket, key): boto_args.update(config=config) s3 = boto3.resource(**boto_args) - s3.meta.client.upload_file(path, bucket, key) + + # TODO: The S3 path should include a pre-defined directory path for the AIP + # We could use a predefined variable in the configuration file for a global A3M Storage such as predefined/global/path// + # Alternatively, we could use a predefined variable specifically for the AIP location such as predefined/aip/location/ + # In any case, the S3 path must be structured in a way that makes it easy to locate and manage the AIP. + s3_path = key + path.suffix + + logger.info(s3_path) + s3.meta.client.upload_file(str(path), bucket, s3_path) def _store_aip(job, sip_id, aip_path): @@ -51,7 +60,7 @@ def _store_aip(job, sip_id, aip_path): raise Exception("AIP is a directory") logger.info("Uploading AIP...") - _upload_file(str(aip_path), settings.S3_BUCKET, sip_id) + _upload_file(aip_path, settings.S3_BUCKET, sip_id) def call(jobs): @@ -59,5 +68,5 @@ def call(jobs): with transaction.atomic(): with job.JobContext(): sip_id = job.args[1] - aip_path = job.args[2] + aip_path = Path(job.args[2]) job.set_status(_store_aip(job, sip_id, aip_path)) From 7911c2038410d7668943bd4121af80be0f874bf5 Mon Sep 17 00:00:00 2001 From: camlyall Date: Fri, 21 Apr 2023 11:27:42 +0000 Subject: [PATCH 3/3] remove logging s3_path --- a3m/client/clientScripts/a3m_store_aip.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/a3m/client/clientScripts/a3m_store_aip.py b/a3m/client/clientScripts/a3m_store_aip.py index bd0886f84..5a1f24a37 100644 --- a/a3m/client/clientScripts/a3m_store_aip.py +++ b/a3m/client/clientScripts/a3m_store_aip.py @@ -43,8 +43,6 @@ def _upload_file(path, bucket, key): # Alternatively, we could use a predefined variable specifically for the AIP location such as predefined/aip/location/ # In any case, the S3 path must be structured in a way that makes it easy to locate and manage the AIP. s3_path = key + path.suffix - - logger.info(s3_path) s3.meta.client.upload_file(str(path), bucket, s3_path)