From b3b47a0a3ce2f8bf4ab775bd4893a131419a7f1e Mon Sep 17 00:00:00 2001
From: Moasib-Arif <moasib_arif@yahoo.com>
Date: Thu, 31 Oct 2024 12:54:16 +0000
Subject: [PATCH] added upload json

---
 dpypelines/pipeline/dataset_ingress_v1.py     | 22 +++++++++++++++++++
 .../pipeline/generic_file_ingress_v1.py       | 14 ++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/dpypelines/pipeline/dataset_ingress_v1.py b/dpypelines/pipeline/dataset_ingress_v1.py
index 01b8ef4..c1da516 100644
--- a/dpypelines/pipeline/dataset_ingress_v1.py
+++ b/dpypelines/pipeline/dataset_ingress_v1.py
@@ -457,6 +457,28 @@ def dataset_ingress_v1(files_dir: str, pipeline_config: dict):
                         )
                         de_notifier.failure()
                         raise err
+                # If the supplementary distribution is a JSON file, upload to the Upload Service
+                elif supp_dist_path.suffix == ".json":
+                    try:
+                        upload_client.upload_new_json(supp_dist_path)
+                        logger.info(
+                            "Uploaded supplementary distribution",
+                            data={
+                                "supplementary_distribution": supp_dist_path,
+                                "upload_url": upload_url,
+                            },
+                        )
+                    except Exception as err:
+                        logger.error(
+                            "Error uploading JSON file to Upload Service",
+                            err,
+                            data={
+                                "supplementary_distribution": supp_dist_path,
+                                "upload_url": upload_url,
+                            },
+                        )
+                        de_notifier.failure()
+                        raise err
                 else:
                     raise NotImplementedError(
                         f"Uploading files of type {supp_dist_path.suffix} not supported."
diff --git a/dpypelines/pipeline/generic_file_ingress_v1.py b/dpypelines/pipeline/generic_file_ingress_v1.py
index d216d47..82373fb 100644
--- a/dpypelines/pipeline/generic_file_ingress_v1.py
+++ b/dpypelines/pipeline/generic_file_ingress_v1.py
@@ -212,6 +212,20 @@ def generic_file_ingress_v1(files_dir: str, pipeline_config: dict):
                     )
                     notifier.failure()
                     raise err
+            elif required_file_path.suffix == ".json":
+                try:
+                    upload_client.upload_new_json(required_file_path)
+                    logger.info(
+                        "File uploaded.", data={"file_path": required_file_path}
+                    )
+                except Exception as err:
+                    logger.error(
+                        "Error uploading file.",
+                        err,
+                        data={"file_path": required_file_path},
+                    )
+                    notifier.failure()
+                    raise err
             else:
                 raise NotImplementedError(
                     f"Uploading file type {required_file_path.suffix} not currently supported."