Merge pull request #265 from ONSdigital/RDRP-867_mapping_qa

Rdrp 867 mapping qa CSV output

src/_version.py

@@ -1 +1 @@
-__version__ = "1.0.3"
+__version__ = "1.0.4"

src/mapping/mapping_main.py

@@ -1,5 +1,7 @@
 """The main file for the mapping module."""
 import logging
+from datetime import datetime
+from typing import Callable
 
 from src.mapping import mapping_helpers as hlp
 from src.mapping.pg_conversion import run_pg_conversion
@@ -14,17 +16,10 @@ def run_mapping(
     full_responses,
     ni_full_responses,
     config: dict,
+    write_csv: Callable,
+    run_id: int,
 ):
 
-    # Check the environment switch
-    # network_or_hdfs = config["global"]["network_or_hdfs"]
-
-    # if network_or_hdfs == "network":
-    #     from src.utils import local_file_mods as mods
-
-    # elif network_or_hdfs == "hdfs":
-    #     from src.utils import hdfs_mods as mods
-
     # Load ultfoc (Foreign Ownership) mapper
     ultfoc_mapper = stage_hlp.load_validate_mapper(
         "ultfoc_mapper_path",
@@ -86,5 +81,19 @@ def run_mapping(
             is_northern_ireland=True,
         )
 
+    # output QA files
+    qa_path = config["mapping_paths"]["qa_path"]
+
+    if config["global"]["output_mapping_qa"]:
+        MappingMainLogger.info("Outputting Mapping QA files.")
+        tdate = datetime.now().strftime("%y-%m-%d")
+        survey_year = config["years"]["survey_year"]
+        full_responses_filename = (
+            f"{survey_year}_full_responses_mapped_{tdate}_v{run_id}.csv"
+        )
+
+        write_csv(f"{qa_path}/{full_responses_filename}", full_responses)  # Changed
+    MappingMainLogger.info("Finished Mapping QA calculation.")
+
     # return mapped_df
     return (full_responses, ni_full_responses, itl_mapper, cellno_df)

src/pipeline.py

@@ -117,6 +117,8 @@ def run_pipeline(user_config_path, dev_config_path):
         full_responses,
         ni_df,
         config,
+        mods.rd_write_csv,
+        run_id,
     )
     MainLogger.info("Finished Mapping...")
 

src/user_config.yaml

@@ -18,6 +18,7 @@ global:
   output_full_responses: False
   output_ni_full_responses: False
   output_imputation_qa: False
+  output_mapping_qa: False
   output_auto_outliers: False
   output_outlier_qa : False
   output_estimation_qa: False
@@ -26,10 +27,10 @@ global:
   output_long_form: False
   output_short_form: False
   output_gb_sas: True
-  output_ni_sas: True
-  output_tau: True
-  output_intram_by_pg_gb: True
-  output_intram_by_pg_uk: True
+  output_ni_sas: False
+  output_tau: False
+  output_intram_by_pg_gb: False
+  output_intram_by_pg_uk: False
   output_intram_gb_itl: False
   output_intram_uk_itl: False
   output_intram_by_civil_defence: False