diff --git a/src/_version.py b/src/_version.py index 976498ab9..92192eed4 100644 --- a/src/_version.py +++ b/src/_version.py @@ -1 +1 @@ -__version__ = "1.0.3" +__version__ = "1.0.4" diff --git a/src/mapping/mapping_main.py b/src/mapping/mapping_main.py index 60bf691c8..049b2965e 100644 --- a/src/mapping/mapping_main.py +++ b/src/mapping/mapping_main.py @@ -1,5 +1,7 @@ """The main file for the mapping module.""" import logging +from datetime import datetime +from typing import Callable from src.mapping import mapping_helpers as hlp from src.mapping.pg_conversion import run_pg_conversion @@ -14,17 +16,10 @@ def run_mapping( full_responses, ni_full_responses, config: dict, + write_csv: Callable, + run_id: int, ): - # Check the environment switch - # network_or_hdfs = config["global"]["network_or_hdfs"] - - # if network_or_hdfs == "network": - # from src.utils import local_file_mods as mods - - # elif network_or_hdfs == "hdfs": - # from src.utils import hdfs_mods as mods - # Load ultfoc (Foreign Ownership) mapper ultfoc_mapper = stage_hlp.load_validate_mapper( "ultfoc_mapper_path", @@ -86,5 +81,19 @@ def run_mapping( is_northern_ireland=True, ) + # output QA files + qa_path = config["mapping_paths"]["qa_path"] + + if config["global"]["output_mapping_qa"]: + MappingMainLogger.info("Outputting Mapping QA files.") + tdate = datetime.now().strftime("%y-%m-%d") + survey_year = config["years"]["survey_year"] + full_responses_filename = ( + f"{survey_year}_full_responses_mapped_{tdate}_v{run_id}.csv" + ) + + write_csv(f"{qa_path}/{full_responses_filename}", full_responses) # Changed + MappingMainLogger.info("Finished Mapping QA calculation.") + # return mapped_df return (full_responses, ni_full_responses, itl_mapper, cellno_df) diff --git a/src/pipeline.py b/src/pipeline.py index 9ee386b25..3278897bf 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -117,6 +117,8 @@ def run_pipeline(user_config_path, dev_config_path): full_responses, ni_df, config, + mods.rd_write_csv, + run_id, ) MainLogger.info("Finished Mapping...") diff --git a/src/user_config.yaml b/src/user_config.yaml index e36e0bcd0..d974c36af 100644 --- a/src/user_config.yaml +++ b/src/user_config.yaml @@ -18,6 +18,7 @@ global: output_full_responses: False output_ni_full_responses: False output_imputation_qa: False + output_mapping_qa: False output_auto_outliers: False output_outlier_qa : False output_estimation_qa: False @@ -26,10 +27,10 @@ global: output_long_form: False output_short_form: False output_gb_sas: True - output_ni_sas: True - output_tau: True - output_intram_by_pg_gb: True - output_intram_by_pg_uk: True + output_ni_sas: False + output_tau: False + output_intram_by_pg_gb: False + output_intram_by_pg_uk: False output_intram_gb_itl: False output_intram_uk_itl: False output_intram_by_civil_defence: False