From 793aa688d8ecd7eac6f103800dfbeea552b8c190 Mon Sep 17 00:00:00 2001 From: Ilyas Ali Date: Thu, 21 Sep 2023 11:26:14 +0100 Subject: [PATCH 1/2] run id fix --- src/developer_config.yaml | 3 ++- src/utils/runlog.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/developer_config.yaml b/src/developer_config.yaml index 0c270a1d4..12918c03e 100644 --- a/src/developer_config.yaml +++ b/src/developer_config.yaml @@ -23,7 +23,7 @@ runlog_writer: log_path: "data/log" hdfs_paths: root: None - logs_foldername: "resdev/logs/run_logs" + logs_foldername: "/ons/rdbe_dev/logs/run_logs" snapshot_path: "/ons/rdbe_dev/snapshot-202012-002-fba5c4ba-fb8c-4a62-87bb-66c725eea5fd.json" postcode_masterlist: "/ons/rdbe_dev/mappers/2023/ONSPD_NOV_2022_UK.csv" postcode_path: "/ons/rdbe_dev/staging/staging_qa/postcode_validation" @@ -31,6 +31,7 @@ hdfs_paths: ultfoc_mapper_path: "/ons/rdbe_dev/mappers/2023/BERD_2022_ultfoc_anon.csv" cora_mapper_path: "/ons/rdbe_dev/mappers/2023/cora_mapper.csv" staging_output_path: "/ons/rdbe_dev/staging/staging_qa/full_responses_qa" + output_path: "/ons/rdbe_dev/outputs" cellno_path: "/ons/rdbe_dev/mappers/2023/berd_2022_cellno_coverage.csv" itl_path: "/ons/rdbe_dev/mappers/2023/itl.csv" history_path: "/ons/rdbe_dev/BERD_V7_Anonymised" diff --git a/src/utils/runlog.py b/src/utils/runlog.py index 093e096c6..f2ef3c5c2 100644 --- a/src/utils/runlog.py +++ b/src/utils/runlog.py @@ -38,7 +38,7 @@ def __init__( def _make_main_path(self): """Creating a local runlog folder if it doesn't exist""" if self.environment == "hdfs": - logs_folder = f"/user/{self.user}/{self.logs_folder}" + logs_folder = self.logs_folder elif self.environment == "network": logs_folder = self.logs_folder return logs_folder From 1e071461aa97a5c9ffcb27f557084fdbf71ad12c Mon Sep 17 00:00:00 2001 From: Ilyas Ali Date: Thu, 21 Sep 2023 15:02:32 +0100 Subject: [PATCH 2/2] run id fix for latest_id --- src/utils/runlog.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/utils/runlog.py b/src/utils/runlog.py index f2ef3c5c2..3a52deae5 100644 --- a/src/utils/runlog.py +++ b/src/utils/runlog.py @@ -36,11 +36,8 @@ def __init__( self.timestamp = self._generate_time() def _make_main_path(self): - """Creating a local runlog folder if it doesn't exist""" - if self.environment == "hdfs": - logs_folder = self.logs_folder - elif self.environment == "network": - logs_folder = self.logs_folder + """Returns the log folder from the config""" + logs_folder = self.logs_folder return logs_folder def _create_folder(self):