added upload json

ONSdigital · Oct 31, 2024 · b3b47a0 · b3b47a0
1 parent 7acde76
commit b3b47a0
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 0 deletions.
diff --git a/dpypelines/pipeline/dataset_ingress_v1.py b/dpypelines/pipeline/dataset_ingress_v1.py
@@ -457,6 +457,28 @@ def dataset_ingress_v1(files_dir: str, pipeline_config: dict):
                         )
                         de_notifier.failure()
                         raise err
+                # If the supplementary distribution is a JSON file, upload to the Upload Service
+                elif supp_dist_path.suffix == ".json":
+                    try:
+                        upload_client.upload_new_json(supp_dist_path)
+                        logger.info(
+                            "Uploaded supplementary distribution",
+                            data={
+                                "supplementary_distribution": supp_dist_path,
+                                "upload_url": upload_url,
+                            },
+                        )
+                    except Exception as err:
+                        logger.error(
+                            "Error uploading JSON file to Upload Service",
+                            err,
+                            data={
+                                "supplementary_distribution": supp_dist_path,
+                                "upload_url": upload_url,
+                            },
+                        )
+                        de_notifier.failure()
+                        raise err
                 else:
                     raise NotImplementedError(
                         f"Uploading files of type {supp_dist_path.suffix} not supported."

diff --git a/dpypelines/pipeline/generic_file_ingress_v1.py b/dpypelines/pipeline/generic_file_ingress_v1.py
@@ -212,6 +212,20 @@ def generic_file_ingress_v1(files_dir: str, pipeline_config: dict):
                     )
                     notifier.failure()
                     raise err
+            elif required_file_path.suffix == ".json":
+                try:
+                    upload_client.upload_new_json(required_file_path)
+                    logger.info(
+                        "File uploaded.", data={"file_path": required_file_path}
+                    )
+                except Exception as err:
+                    logger.error(
+                        "Error uploading file.",
+                        err,
+                        data={"file_path": required_file_path},
+                    )
+                    notifier.failure()
+                    raise err
             else:
                 raise NotImplementedError(
                     f"Uploading file type {required_file_path.suffix} not currently supported."