From 6d00e685de110292357a48be61f4484368631564 Mon Sep 17 00:00:00 2001 From: Jozsef K Date: Thu, 3 Oct 2024 11:17:27 +0100 Subject: [PATCH] made changes --- dpypelines/pipeline/dataset_ingress_v1.py | 2 +- dpypelines/pipeline/generic_file_ingress_v1.py | 2 +- features/environment.py | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/dpypelines/pipeline/dataset_ingress_v1.py b/dpypelines/pipeline/dataset_ingress_v1.py index 9388f91..449c60e 100644 --- a/dpypelines/pipeline/dataset_ingress_v1.py +++ b/dpypelines/pipeline/dataset_ingress_v1.py @@ -102,7 +102,7 @@ def dataset_ingress_v1(files_dir: str, pipeline_config: dict): try: upload_url = os.environ.get("UPLOAD_SERVICE_URL", None) assert ( - upload_url is not None + upload_url is not None and os.environ["SKIP_DATA_UPLOAD"] is not False ), "UPLOAD_SERVICE_URL environment variable not set." logger.info("Got Upload Service URL", data={"upload_url": upload_url}) except Exception as err: diff --git a/dpypelines/pipeline/generic_file_ingress_v1.py b/dpypelines/pipeline/generic_file_ingress_v1.py index 37b4b60..a83e117 100644 --- a/dpypelines/pipeline/generic_file_ingress_v1.py +++ b/dpypelines/pipeline/generic_file_ingress_v1.py @@ -83,7 +83,7 @@ def generic_file_ingress_v1(files_dir: str, pipeline_config: dict): try: upload_url = os.environ.get("UPLOAD_SERVICE_URL", None) assert ( - upload_url is not None + upload_url is not None and os.environ["SKIP_DATA_UPLOAD"] is not False ), "UPLOAD_SERVICE_URL environment variable not set." logger.info("Got Upload Service URL", data={"upload_url": upload_url}) except Exception as err: diff --git a/features/environment.py b/features/environment.py index a6af897..c1b6fd0 100644 --- a/features/environment.py +++ b/features/environment.py @@ -32,9 +32,6 @@ def before_all(context): context.upload_service_url = os.environ.get("UPLOAD_SERVICE_URL", None) os.environ["UPLOAD_SERVICE_URL"] = "http://127.0.0.1:5001/upload-new" - context.upload_service_s3_bucket = os.environ.get("UPLOAD_SERVICE_S3_BUCKET", None) - os.environ["UPLOAD_SERVICE_S3_BUCKET"] = "my-bucket/my.tar" - context.service_token_for_upload = os.environ.get("SERVICE_TOKEN_FOR_UPLOAD", None) os.environ["SERVICE_TOKEN_FOR_UPLOAD"] = "not-a-real-token" @@ -133,9 +130,6 @@ def after_all(context): if context.upload_service_url is not None: os.environ["UPLOAD_SERVICE_URL"] = context.upload_service_url - if context.upload_service_s3_bucket is not None: - os.environ["UPLOAD_SERVICE_S3_BUCKET"] = context.upload_service_s3_bucket - if context.service_token_for_upload is not None: os.environ["SERVICE_TOKEN_FOR_UPLOAD"] = context.service_token_for_upload