made changes

dpypelines/pipeline/dataset_ingress_v1.py

@@ -102,7 +102,7 @@ def dataset_ingress_v1(files_dir: str, pipeline_config: dict):
     try:
         upload_url = os.environ.get("UPLOAD_SERVICE_URL", None)
         assert (
-            upload_url is not None
+            upload_url is not None and os.environ["SKIP_DATA_UPLOAD"] is not False
         ), "UPLOAD_SERVICE_URL environment variable not set."
         logger.info("Got Upload Service URL", data={"upload_url": upload_url})
     except Exception as err:

dpypelines/pipeline/generic_file_ingress_v1.py

@@ -83,7 +83,7 @@ def generic_file_ingress_v1(files_dir: str, pipeline_config: dict):
     try:
         upload_url = os.environ.get("UPLOAD_SERVICE_URL", None)
         assert (
-            upload_url is not None
+            upload_url is not None and os.environ["SKIP_DATA_UPLOAD"] is not False
         ), "UPLOAD_SERVICE_URL environment variable not set."
         logger.info("Got Upload Service URL", data={"upload_url": upload_url})
     except Exception as err:

features/environment.py

@@ -32,9 +32,6 @@ def before_all(context):
     context.upload_service_url = os.environ.get("UPLOAD_SERVICE_URL", None)
     os.environ["UPLOAD_SERVICE_URL"] = "http://127.0.0.1:5001/upload-new"
 
-    context.upload_service_s3_bucket = os.environ.get("UPLOAD_SERVICE_S3_BUCKET", None)
-    os.environ["UPLOAD_SERVICE_S3_BUCKET"] = "my-bucket/my.tar"
-
     context.service_token_for_upload = os.environ.get("SERVICE_TOKEN_FOR_UPLOAD", None)
     os.environ["SERVICE_TOKEN_FOR_UPLOAD"] = "not-a-real-token"
 
@@ -133,9 +130,6 @@ def after_all(context):
     if context.upload_service_url is not None:
         os.environ["UPLOAD_SERVICE_URL"] = context.upload_service_url
 
-    if context.upload_service_s3_bucket is not None:
-        os.environ["UPLOAD_SERVICE_S3_BUCKET"] = context.upload_service_s3_bucket
-
     if context.service_token_for_upload is not None:
         os.environ["SERVICE_TOKEN_FOR_UPLOAD"] = context.service_token_for_upload