From eb68350c45973ee6816a7b0381b44c752a1f6768 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Mon, 29 Jan 2024 15:32:23 +0000
Subject: [PATCH 01/15] update to html template

Adds more detailed information on the input files required.
---
 dMLPA-flask/templates/index.html | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)
diff --git a/dMLPA-flask/templates/index.html b/dMLPA-flask/templates/index.html
index 9fa76b3..a6481f4 100644
--- a/dMLPA-flask/templates/index.html
+++ b/dMLPA-flask/templates/index.html
@@ -24,7 +24,7 @@
 <body>
 
     <div class="container">
-        <h1>dMLPA-flask Web App</h1>
+        <h1>dMLPA Results Generator</h1>
     </div>
 
     <div class="container">
@@ -55,12 +55,21 @@ <h4 class="alert-heading">File Errors:</h4>
 
         {% if backend_state == "initial" %}
         <div class="alert alert-dark" role="alert">
-            <h4 class="alert-heading">Welcome to dMLPA-flask</h4>
-            <p>Please select the files for your analysis below, then click
-                the "Run dMLPA-flask" button.</p>
+            <h4 class="alert-heading">Welcome to dMLPA Results Generator</h4>
+            <p> This web-interface can be used to generate a pdf report of the results obtained from
+                dMLPA analysis. <br>
+                To do this the following 2 inputs are required:<br>
+                <ul>
+                    <li>Probe Information File (xls/xlsx)- This is an excel document containing
+                        information on the genomic location of all probes present in the digitalMLPA
+                        probemix.</li>
+                    <li>Patient Files (xls/xlsx) - These are inidividual excel files generated
+                        by the digitalMLPA filtering script</li>
+                    </ul>
+                Please select the appropriate files below and then click the "Run dMLPA-flask" button.</p>
         </div>
         <div class="container">
-            <h4>dMLPA-flask File Upload</h4>
+            <h4>File Upload</h4>
             <form action=" {{ url_for('dMLPA-flask.form') }}  " method="post" enctype="multipart/form-data">
                 {{ form.hidden_tag() }}
                 <div class="row">
@@ -79,13 +88,13 @@ <h4>dMLPA-flask File Upload</h4>
 
         {%elif backend_state == "started" %}
         <div class="alert alert-success" role="alert">
-            <h4 class="alert-heading">dMLPA-flask Analysis Complete</h4>
-            <p>You submitted the following single file: </p>
+            <h4 class="alert-heading">dMLPA Analysis Complete</h4>
+            <p>You submitted the following probe file: </p>
             <p style="color:blue;"> {{ single_file_name }} </p>
-            <p>You submitted the following multiple files:</p>
+            <p>You submitted the following patient files:</p>
             <p style="color:blue;"> {{ multiple_array_of_files_names }} </p>
             <hr>
-            <h4>The following output can be downloaded to your computer using the button below:</h4>
+            <h4>To download the pdf reports to your computer clink the download button below.</h4>
             <p style="color:blue;"> {{ report_name }} </p>
             <br>
             <a class="btn btn-success" href=" {{url_for('dMLPA-flask.download')}}  ">Download</a>

From c6b1c99c8cc3597a0e56ec5ec4f339a48e1c1cb3 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Mon, 29 Jan 2024 15:33:24 +0000
Subject: [PATCH 02/15] Fix to launch.json

connection port is changed from string to int.
---
 .vscode/launch.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.vscode/launch.json b/.vscode/launch.json
index 91ae1bf..7fc9df2 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -7,7 +7,7 @@
             "request": "attach",
             "connect": {
                 "host": "localhost",
-                "port": "5678"
+                "port": 5678
             },
             "pathMappings": [
                 {

From efb5e3a806ecd618e7392e426e1b9054baa85502 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Wed, 31 Jan 2024 11:58:08 +0000
Subject: [PATCH 03/15] Update to input and output directories

Split upload directory for probe file and patient files.
Created and output directory
---
 Dockerfile.development | 16 +++++++++++-----
 dMLPA-flask/app.py     | 21 ++++++++++++---------
 docker-compose.yml     |  4 +++-
 3 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/Dockerfile.development b/Dockerfile.development
index ea2fbf6..1d10c9c 100644
--- a/Dockerfile.development
+++ b/Dockerfile.development
@@ -12,19 +12,25 @@ ENV FLASK_APP=/dMLPA-flask/app.py
 ENV PYTHONPATH=/dMLPA-flask/dMLPA-flask
 ENV SESSION_FILE_DIR /var/local/dMLPA-flask/flask_sessions
 ENV SCRIPT_NAME=dMLPA-flask
-ENV UPLOAD_FOLDER /var/local/dMLPA-flask/uploads
-ENV FLASK_DEBUG=1 
+ENV UPLOAD_FOLDER_1 /var/local/dMLPA-flask/probe_file
+ENV UPLOAD_FOLDER_2 /var/local/dMLPA-flask/inputs
+ENV OUTPUT_FOLDER /var/local/dMLPA-flask/outputs
+ENV FLASK_DEBUG=1
 # only for development
 
 # add and install requirements
 COPY ./requirements.txt .
 RUN pip3 install -r requirements.txt
-USER 0 
+USER 0
 RUN mkdir -p /var/local/dMLPA-flask/logs/
-RUN mkdir -p /var/local/dMLPA-flask/uploads/
+RUN mkdir -p /var/local/dMLPA-flask/probe_file
+RUN mkdir -p /var/local/dMLPA-flask/inputs
+RUN mkdir -p /var/local/dMLPA-flask/outputs
 RUN mkdir -p /var/local/dMLPA-flask/flask_sessions/
 RUN chmod 777 /var/local/dMLPA-flask/logs/
-RUN chmod 777 /var/local/dMLPA-flask/uploads/
+RUN chmod 777 /var/local/dMLPA-flask/probe_file
+RUN chmod 777 /var/local/dMLPA-flask/inputs
+RUN chmod 777 /var/local/dMLPA-flask/outputs
 RUN chmod 777 /var/local/dMLPA-flask/flask_sessions/
 
 USER $CONTAINER_USER_ID
diff --git a/dMLPA-flask/app.py b/dMLPA-flask/app.py
index 4c3bce0..c680fe7 100644
--- a/dMLPA-flask/app.py
+++ b/dMLPA-flask/app.py
@@ -39,7 +39,9 @@
 
 
 app = Flask(__name__)
-app.config["UPLOAD_FOLDER"] = os.environ["UPLOAD_FOLDER"]
+app.config["UPLOAD_FOLDER_1"] = os.environ["UPLOAD_FOLDER_1"]
+app.config["UPLOAD_FOLDER_2"] = os.environ["UPLOAD_FOLDER_2"]
+app.config["OUTPUT_FOLDER"] = os.environ["OUTPUT_FOLDER"]
 app.config["SECRET_KEY"] = "catchmeifyoucan"
 app.config["SESSION_TYPE"] = "filesystem"
 app.config["SESSION_PERMANENT"] = True
@@ -141,7 +143,8 @@ def form(backend_state="initial"):
         # Use FileHandler() to log to a file
 
         session["timestr"] = datetime.now().strftime("%Y%m%d-%H%M%S")
-        os.mkdir(os.path.join(app.config["UPLOAD_FOLDER"], session["timestr"]))
+        os.mkdir(os.path.join(app.config["UPLOAD_FOLDER_1"], session["timestr"]))
+        os.mkdir(os.path.join(app.config["UPLOAD_FOLDER_2"], session["timestr"]))
         file_handler = logging.FileHandler(
             f"/var/local/dMLPA-flask/logs/dMLPA-flask_error.log"
         )
@@ -170,18 +173,18 @@ def form(backend_state="initial"):
 
         # Create the paths to the uploaded files
         input_single_file_tmp_path = os.path.join(
-            app.config["UPLOAD_FOLDER"], session["timestr"], input_single_file_basename
+            app.config["UPLOAD_FOLDER_1"], session["timestr"], input_single_file_basename
         )
 
         input_files_tmp_paths = [
-            os.path.join(app.config["UPLOAD_FOLDER"], session["timestr"], basename)
+            os.path.join(app.config["UPLOAD_FOLDER_2"], session["timestr"], basename)
             for basename in input_files_basenames
         ]
 
         zipped_output, input_errors, input_ok_flag = run_backend(
             input_single_file_tmp_path,
             input_files_tmp_paths,
-            app.config["UPLOAD_FOLDER"],
+            app.config["OUTPUT_FOLDER"],
         )
         # Save the zipped output path to the session
         session["zipped_output"] = zipped_output
@@ -225,7 +228,7 @@ def upload(self, file):
             secure_file_name = secure_filename(file_name)
             file.save(
                 os.path.join(
-                    app.config["UPLOAD_FOLDER"],
+                    app.config["UPLOAD_FOLDER_1"],
                     session["timestr"],
                     secure_file_name,
                 )
@@ -233,7 +236,7 @@ def upload(self, file):
 
             return str(
                 os.path.join(
-                    app.config["UPLOAD_FOLDER"],
+                    app.config["UPLOAD_FOLDER_1"],
                     session["timestr"],
                     secure_file_name,
                 )
@@ -257,7 +260,7 @@ def upload(self, files):
                 secure_file_name = secure_filename(file_name)
                 file.save(
                     os.path.join(
-                        app.config["UPLOAD_FOLDER"],
+                        app.config["UPLOAD_FOLDER_2"],
                         session["timestr"],
                         secure_file_name,
                     )
@@ -265,7 +268,7 @@ def upload(self, files):
                 file_names.append(
                     str(
                         os.path.join(
-                            app.config["UPLOAD_FOLDER"],
+                            app.config["UPLOAD_FOLDER_2"],
                             session["timestr"],
                             secure_file_name,
                         )
diff --git a/docker-compose.yml b/docker-compose.yml
index 98699e7..717d384 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,7 +9,9 @@ services:
     volumes:
       - .:/dMLPA-flask/
       - /var/local/dMLPA-flask/logs/:/var/local/dMLPA-flask/logs/
-      - /var/local/dMLPA-flask/uploads/:/var/local/dMLPA-flask/uploads/
+      - /var/local/dMLPA-flask/probe_file/:/var/local/dMLPA-flask/probe_file/
+      - /var/local/dMLPA-flask/inputs/:/var/local/dMLPA-flask/inputs/
+      - /var/local/dMLPA-flask/outputs/:/var/local/dMLPA-flask/outputs/
 
     ports:
       - 5000:5000

From 37f7e8a4e807b96217221c923053040e57dd2f89 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Mon, 4 Mar 2024 15:09:47 +0000
Subject: [PATCH 04/15] update output folder path

---
 dMLPA-flask/app.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/dMLPA-flask/app.py b/dMLPA-flask/app.py
index c680fe7..7aee98e 100644
--- a/dMLPA-flask/app.py
+++ b/dMLPA-flask/app.py
@@ -145,6 +145,7 @@ def form(backend_state="initial"):
         session["timestr"] = datetime.now().strftime("%Y%m%d-%H%M%S")
         os.mkdir(os.path.join(app.config["UPLOAD_FOLDER_1"], session["timestr"]))
         os.mkdir(os.path.join(app.config["UPLOAD_FOLDER_2"], session["timestr"]))
+        os.mkdir(os.path.join(app.config["OUTPUT_FOLDER"], session["timestr"]))
         file_handler = logging.FileHandler(
             f"/var/local/dMLPA-flask/logs/dMLPA-flask_error.log"
         )
@@ -181,10 +182,13 @@ def form(backend_state="initial"):
             for basename in input_files_basenames
         ]
 
+        # Create the path to the output folder
+        output_path = os.path.join(app.config["OUTPUT_FOLDER"], session["timestr"])
+
         zipped_output, input_errors, input_ok_flag = run_backend(
             input_single_file_tmp_path,
             input_files_tmp_paths,
-            app.config["OUTPUT_FOLDER"],
+            output_path,
         )
         # Save the zipped output path to the session
         session["zipped_output"] = zipped_output

From e6c062c8429de2ce8964249ada40334798a3b9b4 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Mon, 4 Mar 2024 16:57:45 +0000
Subject: [PATCH 05/15] Progress up to point of code refactor

This includes:
- creation of html report template
- parsing test data from results/excel file
- converting copy number analysis script to python from original R script in https://github.com/moka-guys/dMLPA-reporting
- Generating minimal pdf reports (from html template) per results/excel file
- Updating backend.py to create zip of output files rather than of input files
---
 Dockerfile.development            |   1 +
 dMLPA-flask/backend.py            |  67 ++++++++-
 dMLPA-flask/cna.py                | 233 ++++++++++++++++++++++++++++++
 dMLPA-flask/qc_data.py            |  40 +++++
 dMLPA-flask/templates/report.html | 144 ++++++++++++++++++
 5 files changed, 484 insertions(+), 1 deletion(-)
 create mode 100644 dMLPA-flask/cna.py
 create mode 100644 dMLPA-flask/qc_data.py
 create mode 100644 dMLPA-flask/templates/report.html

diff --git a/Dockerfile.development b/Dockerfile.development
index 1d10c9c..32a6153 100644
--- a/Dockerfile.development
+++ b/Dockerfile.development
@@ -36,6 +36,7 @@ RUN chmod 777 /var/local/dMLPA-flask/flask_sessions/
 USER $CONTAINER_USER_ID
 
 RUN pip install debugpy
+RUN apt-get update && apt-get install -y wkhtmltopdf
 
 # add app
 COPY . .
diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index dc628f3..c2466a0 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -1,6 +1,11 @@
 import os
+import fnmatch
 import zipfile
 from datetime import datetime
+from jinja2 import Environment, PackageLoader
+import pdfkit
+from qc_data import parse_run_data
+from cna import cna
 
 img_versioned = os.getenv(
     "IMG_VERSIONED"
@@ -15,12 +20,72 @@ def run_backend(single_file, array_of_files, output_directory):
     timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
     zip_name = os.path.join(output_directory, f"output_files_{timestamp}.zip")
 
+    # Creating empty list of files to zip
+    files_to_zip = []
+
     # Ensure all paths are absolute
     single_file = os.path.abspath(single_file)
     array_of_files = [os.path.abspath(file) for file in array_of_files]
 
+    # Generating patient reports
+    for file in array_of_files:
+
+        # Getting file basename
+        file_name = os.path.basename(file)
+
+        # Parsing test information from file name
+        # TODO do I want this to be within backend.py or somewhere else
+        # TODO might need reworking after confirming file name convention
+        file_name_elements = file_name.split("_")
+        # # patient identifier might not need to be included.
+        # patient_identifier = ", ".join([file_name_elements[5], file_name_elements[4]])
+        # r_code = file_name_elements[1]
+        pan_number = file_name_elements[2]
+
+        # Creating ouput file name and path
+        # TODO consider if all files have this filtered results header - check erins script
+        report = file_name.replace("_filtered_results.xlsx", ".pdf").replace("_filtered_results.xls", ".pdf")
+        report_path = os.path.join(output_directory, report)
+
+        # Getting Probe File Name
+        probe_file = os.path.basename(single_file)
+
+        # Get QC data
+        table_html, qc_status, sample_type = parse_run_data(file, file_name)
+
+        # Analyse data for CNVs
+        gene_list, cnv_summary = cna(file, file_name, single_file)
+
+        # Converting gene_list to string
+        genes = ", ".join(gene_list)
+
+        # Rendering template and saving as pdf
+
+        env = Environment(loader=PackageLoader("dMLPA-flask", "templates"))
+        template = env.get_template("report.html")
+        data_values = {
+            "file_name": report,
+            "patient_identifier": patient_identifier,
+            "sample_type": sample_type,
+            # "r_code": r_code,
+            "pan_number": pan_number,
+            "genes": genes,
+            "patient_file": file_name,
+            "probe_file": probe_file,
+            "qc_status": qc_status,
+            "cnv_summary": cnv_summary,
+            "html_table": table_html
+        }
+
+        html_report = template.render(data_values)
+
+        pdfkit.from_string(html_report, report_path)
+
+        # Add report to list of files to be zipped
+        files_to_zip.append(report_path)
+
     # Prepare the files to be zipped
-    files_to_zip: list[str] = [single_file] + array_of_files
+    # files_to_zip: list[str] = [single_file] + array_of_files
 
     # USER WARNINGS: Anything added to the list 'input_errors' will be passed from the backend to
     # the frontend and displayed in a warning to the user. Used to give immediate feedback to the
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
new file mode 100644
index 0000000..af0a70a
--- /dev/null
+++ b/dMLPA-flask/cna.py
@@ -0,0 +1,233 @@
+import pandas as pd
+import numpy as np
+
+# # Test files to test script
+# xls = "/home/epayne1/Documents/GSTT/test_input/BP01-11-dMLPA_R210v1_Pan5025_185897_SURNAME_Firstname11_filtered_results.xlsx"
+# probe = "/home/epayne1/Documents/GSTT/probe_file/PIF_D001-C1_Hereditary_Cancer_Panel_1-v02_modB1probe.xlsx"
+# file_name = "BP01-11-dMLPA_R210v1_Pan5025_185897_SURNAME_Firstname11_filtered_results.xlsx"
+
+def cna(file, file_basename, probe_file):
+    """
+    TODO Add function docstring
+    """
+
+    # Import files into panda df
+    probe_df = pd.read_excel(probe_file, sheet_name="probe_info")
+    results_raw_df = pd.read_excel(file, sheet_name="Gene filtered results")
+
+    # Renaming columns so they match between both dataframes
+    probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
+    results_raw_df.rename(columns={"Normal copy number": "Normal Copy Number"}, inplace=True)
+
+    # Controlling datatype in results_raw_df
+    # results_raw_df["Exon"] = results_raw_df["Exon"].astype(int)
+    # TODO Consider this. Some are "up" or "down" so can't be converted to int
+
+    # Separating chr, start and end co-ordinates into separate columns
+    probe_df[["Chr", "Start Position",  "End Position"]] = probe_df["Mapview (hg38)a"].str.split(r"[:\-]", expand=True)
+
+    # Manipulting data in Chr. 
+    probe_df["Chr"] = probe_df["Chr"].str.replace("chr", "").str.lstrip("0")
+
+    # TODO Having trouble converting data type to int
+    # results_raw_df[["Start Position", "End Position"]] = results_raw_df[["Start Position", "End Position"]].astype(int)
+
+    # Finding shared columns
+    shared_columns = list(set(probe_df.columns) & set(results_raw_df.columns))
+
+    # Renaming results column
+    # TODO Potentially refactor this/ rename header
+    current_column_header = file_basename.replace("_filtered_results.xlsx", "")
+    results_raw_df.rename(columns={current_column_header:"Mean Dosage Quotient"}, inplace=True)
+
+    # defining select columns to include from each df
+    columns_probe_df = [
+      "Probe number",
+      "Chr",
+      "Start Position",
+      "End Position",
+      "Position",
+      "Warnings"
+    ]
+
+    columns_results_df = [
+        "Mean Dosage Quotient"
+    ]
+
+    # merging dfs
+
+    # results_probes = pd.merge(
+    #     probe_df[shared_columns + columns_probe_df],
+    #     results_raw_df[shared_columns + columns_results_df],
+    #     on=shared_columns,
+    #     how="left",
+    #     suffixes = ("_probe", "_results")
+    # )
+
+    results_probes_df = results_raw_df[shared_columns + columns_results_df].merge(
+        probe_df[columns_probe_df],
+        how = "left",
+        on = "Probe number"
+    )
+    # print(results_probes_df)
+
+    # # TESTING Script:Exploring unique row entries
+    # # rows that are in results raw but not in merged pdf.
+    # unique_rows_results_df = results_raw_df[~results_raw_df["Probe number"].isin(results_probes_df["Probe number"])]
+
+    # # rows that are in merged pdf but no in results raw.
+    # unique_rows_merge_df = results_probes_df[~results_probes_df["Probe number"].isin(results_raw_df["Probe number"])]
+
+    # # rows that are in results but not in the probe file
+    # unique_rows_not_in_probe = results_raw_df[~results_raw_df["Probe number"].isin(probe_df["Probe number"])]
+    # # unique_rows_probes_df = probe_df[~probe_df["Probe number"].isin(results_raw_df["Probe number"])]
+    # number_unique_results = len(unique_rows_results_df)
+    # number_unique_merge = len(unique_rows_merge_df)
+    # number_unique_not_in_probe = len(unique_rows_not_in_probe)
+    # # number_unique_in_probe = len(unique_rows_probes_df)
+
+    # print(number_unique_merge)
+    # if number_unique_merge != 0:
+    #     print(unique_rows_merge_df)
+
+    # print(number_unique_results)
+    # if number_unique_results != 0:
+    #     print(unique_rows_results_df)
+
+    # print(number_unique_not_in_probe)
+    # if number_unique_not_in_probe != 0:
+    #     print(unique_rows_not_in_probe)
+
+
+    # Defining the order of the columns
+    # TODO Might not be needed it's just to help me visualize
+
+    desired_column_order = [
+        "Probe order",
+        "Probe number",
+        "Reference probe",
+        "Chr",
+        "Gene",
+        "Exon",
+        "Start Position",
+        "End Position",
+        "Normal Copy Number",
+        "Mean Dosage Quotient",
+        "Probe type",
+        "Additional information",
+        "Position",
+        "Warnings",
+    ]
+
+    # restructuring df based on column order
+    results_probes_df = results_probes_df[desired_column_order]
+
+    # Filtering results by probetype
+    results_clinical = results_probes_df[results_probes_df["Probe type"] != "CTRL"]
+
+    # TODO Delete in final code. This is only to test outputs.
+    # print(results_clinical)
+    # results_clinical.to_excel("merged_excel.xlsx")
+
+    # Obtaining list of genes tested for
+    genes_list = results_clinical["Gene"].unique()
+    print(genes_list)
+
+    cncategories = {
+        "Ref to Scientist 1*": [0, 0.4],
+        "HetDel": [0.4, 0.6],  # theoretical value 0.5, represents CN 1
+        "Ref to Scientist 2*": [0.6, 0.8],
+        # "normal": [0.8, 1.2],  # theoretical value 1, represents CN 2
+        "Ref to Scientist 3*": [1.2, 1.4],
+        "HetDup": [1.4, 1.6],  # theoretical value 1.5, represents CN 3
+        "Ref to Scientist 4*": [1.6, float("Inf")]
+    }
+
+    # Creating empty dataframe for all identified copy number variations
+    cnv_summary = pd.DataFrame()
+
+    # Creating dictionary list for details on the cnv
+    cnv_detail = {}
+
+    # TODO This is mostly from chatGPT - Need to understand what it's actually doing
+    for gene in genes_list:
+
+        # Get results per gene
+        results_gene =  results_clinical[results_clinical["Gene"] == gene]
+
+        # iterate through CN categories and identify matching probe entries
+        for direction, parameters in cncategories.items():
+            print(direction, gene) # TODO Replace with logging
+            cnv_rows = np.where((parameters[0] <= results_gene["Mean Dosage Quotient"]) &
+                    (results_gene["Mean Dosage Quotient"] < parameters[1]))[0]
+
+            # If no probe entries match CN category skipping next steps
+            if len(cnv_rows) == 0:
+                continue
+
+            # If probe entries match CN category, identify uniform CN segments
+            else:
+                # gather information for appropriate probe entries
+                cnv_data = results_gene.iloc[cnv_rows]
+                print(cnv_data)
+                # Assess based on probe order if probes are contiguous
+                cnv_groups = np.cumsum(np.concatenate(([1], np.diff(cnv_data["Probe order"]) != 1)))
+                print(cnv_groups)
+
+                # Splitting results per contiguous CN segment if multiple
+                cnv_list = {cnv_index: group for cnv_index, group in cnv_data.groupby(cnv_groups)}
+                print(cnv_list)
+
+                # Processing data of individual CN segments
+                for cnv_index, cnv_lines in cnv_list.items():
+
+                    # Obtainins summary data of contiguous CN segments
+                    start_exon = cnv_lines["Exon"].min()
+                    end_exon = cnv_lines["Exon"].max()
+                    start_pos = cnv_lines["Start Position"].min()
+                    end_pos = cnv_lines["End Position"].max()
+
+                    # Calculate average dosage quotient across CN segment and standard deviation
+                    copy_change_mean = round(cnv_lines["Mean Dosage Quotient"].mean(), 3)
+                    copy_change_sd = round(cnv_lines["Mean Dosage Quotient"].std(), 3)
+
+                # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
+                    copy_number_est = round(np.mean(cnv_lines["Normal Copy Number"] * cnv_lines["Mean Dosage Quotient"]))
+
+                    # summarize CN data into a df entry
+                    cnv_aggregate = pd.DataFrame({
+                        "Gene": [gene],
+                        "Variant": [direction],
+                        "Exon(s)": f"{start_exon} - {end_exon}" if start_exon != end_exon else str(start_exon),
+                        "Chr": cnv_lines["Chr"].unique(),
+                        "Probe Co-ordinates": f"{start_pos}-{end_pos}",
+                        "Mean Dosage Quotient": f"{copy_change_mean}+/-{copy_change_sd}",
+                        "Est. Copy Number": [copy_number_est],
+                        "Supporting Probes": [len(cnv_lines)]
+                    })
+
+                    # Adding df entry into existing CNA_summary dataframe
+                    # TODO concat method is not ideal apparently. Consider alternative
+                    cnv_summary = pd.concat([cnv_summary,cnv_aggregate], ignore_index=True)
+
+                    # organize summary by genomic co-ordinates
+                    cnv_summary = cnv_summary.sort_values(by="Probe Co-ordinates")
+
+                    # print(cnv_summary)
+
+                    # # Saving probe details of CN segment
+                    # # TODO Need to figure out how to build this to then be able to parse to front end.
+                    # cnv_context = results_gene[results_gene["Probe number"]].isin(cnv_lines["Probe number"])
+                    # cnv_context["marked_cnv"] = cnv_context["Probe number"].isin(cnv_lines["Probe number"])
+                    # cna_detail[len(cna_summary)] = cnv_context    # unique_rows_results_df = results_raw_df[~results_raw_df["Probe number"].isin(results_probes["Probe number"])]
+
+
+                    # print(cna_detail)
+
+                # Generating cnv_summary as html
+                cnv_summary_html = cnv_summary.to_html(classes="table", index=False)
+
+    # returning output data to backend.py
+    return genes_list, cnv_summary_html
+
+# cna(xls, file_name, probe)
diff --git a/dMLPA-flask/qc_data.py b/dMLPA-flask/qc_data.py
new file mode 100644
index 0000000..b3204eb
--- /dev/null
+++ b/dMLPA-flask/qc_data.py
@@ -0,0 +1,40 @@
+import os
+from datetime import datetime
+import pandas as pd
+
+"""
+TODO:
+Parse
+- Patient Name = Split file by name by _ and then access 4 and 5
+- Rcode = acces 1
+- Pan Number = access 2
+- List of genes = MAYBE PARSE FROM ELSEWHERE
+- Date/RunID
+- QC passed or Failed = From final row of the QC section
+
+"""
+
+# xls = "/home/epayne1/Documents/GSTT/test_input/BP01-01-dMLPA_R211v1_Pan5026_200930_SURNAME_Firstname1_filtered_results.xlsx"
+# file_name = "BP01-01-dMLPA_R211v1_Pan5026_200930_SURNAME_Firstname1"
+
+def parse_run_data(file, file_basename):
+    """
+    
+    """
+    # Parse Sample Info sheet from excel into dataframe
+    sample_info_df = pd.read_excel(file, sheet_name="Sample info")
+
+    #Saving dataframe to html format
+    #TODO Format table style Including getting rid of row index
+    sample_info_html = sample_info_df.to_html(classes="table", index=False)
+
+    # Parsing test information from spreadshet
+    column_header = file_basename.replace("_filtered_results.xlsx", "")
+    qc_status = sample_info_df.loc[6, column_header].upper()
+
+    # Parsing sample type
+    sample_type = sample_info_df.loc[1, column_header]
+
+    return sample_info_html, qc_status, sample_type
+
+# print(parse_run_data(xls, file_name))
\ No newline at end of file
diff --git a/dMLPA-flask/templates/report.html b/dMLPA-flask/templates/report.html
new file mode 100644
index 0000000..4e0ac78
--- /dev/null
+++ b/dMLPA-flask/templates/report.html
@@ -0,0 +1,144 @@
+<!DOCTYPE html>
+<html lang="en-UK">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>dMLPA Reporting Template</title>
+    <style>  /* TODO Consider transferring to css style layout instead and importing */
+        /* Style for printing to PDF */
+        @page {
+            size: A4;
+            margin: 1.5cm;
+        } /* Make sure header and footer work */
+
+        /* Global Style */
+        body {
+            font-family: Arial;
+            padding: 10px;/* TODO Fix padding. It is only applying on left */
+            font-size: 12px;
+        }
+
+        body .large{
+            font-family: Arial;
+            font-size: 16px;
+        }
+
+        /* Header Styles TODO only appearing on the first page */
+        header {
+            display: flex;
+        }
+
+        header .header-element1{
+            flex: 1;
+        }
+
+        header .header-element2{
+            flex: 1;
+            position: absolute;
+            right: 0;
+        }
+
+        /* Body Styles */
+        .container {
+            display: flex;
+        }
+        .element:first-child{
+            font-family: Arial;
+            font-size: 12px;
+        }
+        .element:last-child {
+            position: absolute;
+            right: 0;
+            font-family: Arial;
+            font-size: 28px;
+            font-style: bold;
+        }
+        /* Page Break Element*/
+        .page-break {
+            page-break-before: always;
+        }
+    </style>
+</head>
+
+<body>
+
+<header>
+    <div class="header-element1"> dMLPA report generator v.<!-- TODO Insert Jinja2 --></div>
+    <div class="header-element2"> {{ file_name }}</div>
+</header>
+<hr><br>
+
+<h1>dMLPA Copy Number Analysis</h1>
+<p><i>South East Genomics Laboratory Hub</i></p>
+
+<br>
+<div class="large">
+    <p><b>Patient identifier:</b><i>{{ surname }}, {{ given_name }}</i><p> <!-- TODO Consider layout reformatting. Want font larger-->
+</div>
+
+<div class="container">
+    <div class="element">
+        <h4>Test Request</h4>
+            <p><b>Sample Type: </b><i>{{ sample_type }}</i></p> <!-- TODO Potentially delete-->
+            <p><b>Number: </b><i>{{ pan_number }}</i></p>
+            <p><b>Genes: </b><i>{{ genes }}</i></p>
+            <br>
+            <p><b>Run ID: </b><i><!-- TODO To be parsed after confirming sample naming convention --></i></p>
+            <p><b>File-Name: </b><i>{{ patient_file }}</i></p>
+            <p><b>Probe-file: </b><i>{{ probe_file }}</i><!-- TODO Insert Probe file used --> </p>
+    </div>
+    <div class="element">
+        {% if qc_status == "PASSED" %}
+            <p style="color:green"> QC {{ qc_status }}</p>
+        {% else %}
+            <p style="color:red"> QC {{ qc_status }} </p>
+        {% endif %}
+    </div>
+</div>
+<hr>
+
+<div class="body">
+    <h2>Called CNV's</h2>
+        <p> {{ cnv_summary }}</p>
+        {% if "*Ref to Scientist 1" is in cnv_summary %}
+            <p>Ref to Scientist 1: The mean dosage quotient for one or more probes falls
+                below the range expected for a copy number loss. Scientist review required.
+            </p>
+        {% elif "*Ref to Scientist 2" is in cnv_summary %}
+            <p>Ref to Scientist 2: The mean dosage quotient for one or more probes falls
+                in between the expected ranges for copy number loss and copy number neutral.
+                Scientist review required.
+            </p>
+        {% elif "*Ref to Scientist 3" is in cnv_summary %}
+            <p>Ref to Scientist 3: The mean dosage quotient for one or more probes falls
+                in between the expected ranges for copy number neutral and copy number gain.
+                Scientist review required.
+            </p>
+        {% elif "*Ref to Scientist 4" is in cnv_summary %}
+            <p>Ref to Scientist 4: The mean dosage quotient for one or more probes falls
+                above the range expected for a copy number gain. Scientist review required.
+            </p>
+        {% endif %}
+</div>
+
+<div class="body">
+    <h3>CNV details</h3>
+        <p>INSERT PLOT DIAGRAM</p>
+        <p>INSERT PROBE TABLE</p>
+</div>
+
+<div class="page-break"></div>
+
+<div>
+    <h2>Appendix</h2>
+    <br>
+    <h3>QC Data</h3>
+        <p> {{ html_table }}</p>
+    <br>
+    <h3>Raw Data</h3>
+        <p>INSERT TABLES</p> <!-- TODO This will require a for loop -->
+</div>
+
+</body>
+</html>

From 9edc6364e7af9ca8569da90b70e69ef395470786 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Wed, 27 Mar 2024 19:53:52 +0000
Subject: [PATCH 06/15] Refactoring script and adding raw summary data tables

---
 dMLPA-flask/backend.py            | 118 ++++++++++---
 dMLPA-flask/cna.py                | 276 ++++++++++++++++++++++++++++--
 dMLPA-flask/qc_data.py            | 154 ++++++++++++++---
 dMLPA-flask/templates/report.html |  37 ++--
 4 files changed, 488 insertions(+), 97 deletions(-)

diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index c2466a0..b84b3b3 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -1,11 +1,11 @@
 import os
-import fnmatch
 import zipfile
 from datetime import datetime
 from jinja2 import Environment, PackageLoader
 import pdfkit
-from qc_data import parse_run_data
-from cna import cna
+import pandas as pd
+from qc_data import get_pan_numer, df_to_html, get_qc_status, summary_data
+from cna import split_df_column, clinical_df, copy_number_analysis
 
 img_versioned = os.getenv(
     "IMG_VERSIONED"
@@ -27,54 +27,120 @@ def run_backend(single_file, array_of_files, output_directory):
     single_file = os.path.abspath(single_file)
     array_of_files = [os.path.abspath(file) for file in array_of_files]
 
-    # Generating patient reports
+    # Iterating over files to generate reports
     for file in array_of_files:
 
-        # Getting file basename
+        # Getting basenames
         file_name = os.path.basename(file)
-
-        # Parsing test information from file name
-        # TODO do I want this to be within backend.py or somewhere else
-        # TODO might need reworking after confirming file name convention
-        file_name_elements = file_name.split("_")
-        # # patient identifier might not need to be included.
-        # patient_identifier = ", ".join([file_name_elements[5], file_name_elements[4]])
-        # r_code = file_name_elements[1]
-        pan_number = file_name_elements[2]
+        probe_file = os.path.basename(single_file)
 
         # Creating ouput file name and path
-        # TODO consider if all files have this filtered results header - check erins script
+        # TODO might need changing depending on file_naming convention.
         report = file_name.replace("_filtered_results.xlsx", ".pdf").replace("_filtered_results.xls", ".pdf")
         report_path = os.path.join(output_directory, report)
 
-        # Getting Probe File Name
-        probe_file = os.path.basename(single_file)
-
-        # Get QC data
-        table_html, qc_status, sample_type = parse_run_data(file, file_name)
+        # Reading data in dataframes
+        metadata_df = pd.read_excel(file, sheet_name="Sample info")
+        raw_results_df = pd.read_excel(file, sheet_name="Gene filtered results")
+        probe_df = pd.read_excel(single_file, sheet_name="probe_info")
+
+        # Defining new column names
+        probe_column_headers = [
+            "chr",
+            "start position",
+            "end position"
+        ]
+
+        # hard manipulation/formatting of df's
+        probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
+        probe_df = split_df_column(probe_df, "Mapview (hg38)a", probe_column_headers)
+        probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
+        raw_results_df.rename(columns={raw_results_df.columns[-1]:"mean dosage quotient"}, inplace=True)
+
+        # Definining unique columns needed from df's
+        columns_results_df = [
+            "mean dosage quotient"
+        ]
+
+        columns_probe_df = [
+            "probe number",
+            "chr",
+            "start position",
+            "end position",
+        ]
+
+        # creating results_clinical dataframe
+        results_clinical_df = clinical_df(raw_results_df, probe_df, columns_results_df, columns_probe_df)
+
+        # get list of genes
+        gene_list = results_clinical_df["gene"].unique()
+
+        # perform copy number analysis
+        cnv_summary_df = copy_number_analysis(results_clinical_df, gene_list)
+        summary_html = df_to_html(cnv_summary_df)
+
+        # defining review messages
+        report_notes = {
+            "Ref to Scientist 1": (
+                "Ref to Scientist 1: The mean dosage quotient for one or "\
+                "more probes falls below the range expected for a copy "\
+                "number loss. Scientist review required."
+            ),
+            "Ref to Scientist 2": (
+                "Ref to Scientist 2: The mean dosage quotient for one or "\
+                "more probes falls in between the expected ranges for "\
+                "copy number loss and copy number neutral. Scientist review "\
+                "required"
+            ),
+            "Ref to Scientist 3": (
+                "Ref to Scientist 3: The mean dosage quotient for one or "\
+                "more probes falls in between the expected ranges for copy "\
+                "number neutral and copy number gain. Scientist review "\
+                "required."
+            ),
+            "Ref to Scientist 4": (
+                "Ref to Scientist 4: The mean dosage quotient for one or "\
+                "more probes falls above the range expected for a copy "\
+                "number gain. Scientist review required."
+            )
+        }
 
-        # Analyse data for CNVs
-        gene_list, cnv_summary = cna(file, file_name, single_file)
+        # getting panel number and qc data from metadata
+        # TODO consider if i shouldn't have this futher down rather than at beginning.
+        pan_number = get_pan_numer(metadata_df)
+        qc_status = get_qc_status(metadata_df)
+        qc_data = df_to_html(metadata_df)
 
         # Converting gene_list to string
         genes = ", ".join(gene_list)
 
+        # obtaining summary data per gene for appendix
+        summary_data_df = summary_data(results_clinical_df)
+        print(summary_data_df)
+
+        summary_tables = {}
+        for gene in gene_list:
+            summary_data_gene =  summary_data_df[summary_data_df["gene"] == gene]
+            summary_data_gene.columns = summary_data_gene.columns.str.replace("_", " ").str.title()
+            summary_tables[gene] = df_to_html(summary_data_gene)
+            # summary_tables[gene] = gene_summary_data(gene, summary_data_df)c
+
         # Rendering template and saving as pdf
 
         env = Environment(loader=PackageLoader("dMLPA-flask", "templates"))
         template = env.get_template("report.html")
         data_values = {
             "file_name": report,
-            "patient_identifier": patient_identifier,
-            "sample_type": sample_type,
             # "r_code": r_code,
             "pan_number": pan_number,
             "genes": genes,
             "patient_file": file_name,
             "probe_file": probe_file,
             "qc_status": qc_status,
-            "cnv_summary": cnv_summary,
-            "html_table": table_html
+            "cnv_summary": summary_html,
+            "report_notes": report_notes,
+            "qc_table": qc_data,
+            "summary_tables": summary_tables
         }
 
         html_report = template.render(data_values)
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
index af0a70a..673839f 100644
--- a/dMLPA-flask/cna.py
+++ b/dMLPA-flask/cna.py
@@ -1,14 +1,246 @@
+"""
+Module for conducting copy number analysis on pan filtered digital
+MLPA output data.
+
+This module provides functions for merging the probe data with
+the results data (which is necessary to return the desired output
+data), manipulating the relevant dataframes, creating copy number
+summary and detailed copy number reports.
+
+Functions:
+- TODO UPDATE
+"""
+
 import pandas as pd
 import numpy as np
 
-# # Test files to test script
-# xls = "/home/epayne1/Documents/GSTT/test_input/BP01-11-dMLPA_R210v1_Pan5025_185897_SURNAME_Firstname11_filtered_results.xlsx"
-# probe = "/home/epayne1/Documents/GSTT/probe_file/PIF_D001-C1_Hereditary_Cancer_Panel_1-v02_modB1probe.xlsx"
-# file_name = "BP01-11-dMLPA_R210v1_Pan5025_185897_SURNAME_Firstname11_filtered_results.xlsx"
+def shared_columns(df1, df2):
+    """Identify shared dataframe columns"""
+
+    # streamline column formatting
+    df1.columns = df1.columns.str.replace("_", " ").str.title()
+    df2.columns = df2.columns.str.replace("_", " ").str.title()
+
+    # identifying shared columns
+    shared_columns = list(set(df1) & set(df2))
+
+    return shared_columns
+
+def split_df_column(df, column, headers):
+    """
+    Split dataframe columns based on set regular expression
+
+    Arguments:
+    - df (Pandas Dataframe)
+    - column (str): name of existing column
+    - headers (list): list of new column headers
+
+    Return:
+    - df (Pandas Dataframe): updated dataframe
+    """
+    df[headers] = df[column].str.split(r"[:\-]", expand=True)
+    return df
+
+def merge_df(df1, df2, df1_columns, df2_columns):
+    """Left join databases based on shared columns
+
+    Arguments:
+    - df1 (Pandas Dataframe): 'left' database in the merge.
+    Data of this database will be maintained for the 'shared
+    columns'
+    - df2 (Pandas Dataframe): 'right' database. Only Data in
+    the unique df2_columns will be maintained.
+    - df1_columns (list): Specific unique columns from df1 that
+    must be maintained
+    - df2_columns (list): Specific unique columns from df2 that
+    must be maintained
+
+    Return:
+    - merged_df (Pandas Dataframe)
+
+    """
+
+    # streamline column formatting
+    df1.columns = df1.columns.str.replace("_", " ").str.lower()
+    df2.columns = df2.columns.str.replace("_", " ").str.lower()
+
+    # identifying shared columns
+    shared_columns = list(set(df1) & set(df2))
+    # print(shared_columns)
+
+    # left merge
+    merged_df = df1[shared_columns + df1_columns].merge(
+        df2[df2_columns],
+        how = "left",
+        on = "probe number" # can be changed based on desired column
+    )
+
+    return merged_df
+
+def clinical_df(df1, df2, df1_columns, df2_columns):
+    """
+    Create dataframe of clinical relevant MLPA probes
+
+    Calls merge_df (function) to left-join probe and results
+    dataframe.
+
+    Arguments:
+    - df1 (Pandas Dataframe): 'left' database in the merge.
+    Data of this database will be maintained for the 'shared
+    columns'
+    - df2 (Pandas Dataframe): 'right' database. Only Data in
+    the unique df2_columns will be maintained.
+    - df1_columns (list): Specific unique columns from df1 that
+    must be maintained
+    - df2_columns (list): Specific unique columns from df2 that
+    must be maintained
+
+    Return:
+    - clinical_df (Pandas Dataframe)
+    """
+
+    merged_df =  merge_df(df1, df2, df1_columns, df2_columns)
+
+    # specifying column order
+    # TODO Change as required
+    desired_column_order = [
+        "probe order",
+        "probe number",
+        "reference probe",
+        "chr",
+        "gene",
+        "exon",
+        "start position",
+        "end position",
+        "normal copy number",
+        "mean dosage quotient",
+        "probe type",
+        "additional information",
+    ]
+
+    # restructuring df based on column order
+    merged_df = merged_df[desired_column_order]
+
+    # Filtering results by probetype
+    results_clinical_df = merged_df[merged_df["probe type"] != "CTRL"]
+
+    return results_clinical_df
+
+def copy_number_analysis(df, gene_list):
+    """
+    TODO function string needed.
+    Currently set up to analyse for all genes in run.
+    Might need to be broken down more in future
+
+    """
+
+    # defining copy number categories
+    # TODO How does code handle MDQ that is 3 dp in the excel
+    cncategories = {
+        "Ref to Scientist 1*": [0, 0.39],
+        "HetDel": [0.4, 0.6],  # theoretical value 0.5, represents CN 1
+        "Ref to Scientist 2*": [0.61, 0.79],
+        # "normal": [0.8, 1.2],  # theoretical value 1, represents CN 2
+        "Ref to Scientist 3*": [1.21, 1.39],
+        "HetDup": [1.4, 1.6],  # theoretical value 1.5, represents CN 3
+        "Ref to Scientist 4*": [1.61, float("Inf")]
+    }
+
+    # Empty dataframe for all identified copy number variations
+    cnv_summary = pd.DataFrame()
+
+    # TODO This is mostly from chatGPT - Need to understand what it's actually doing
+    for gene in gene_list:
+
+        # Get results per gene
+        results_gene =  df[df["gene"] == gene]
+
+        # iterate through CN categories and identify matching probe entries
+        for direction, parameters in cncategories.items():
+            # print(direction, gene) # TODO Replace with logging
+            cnv_rows = np.where((parameters[0] <= results_gene["mean dosage quotient"]) &
+                    (results_gene["mean dosage quotient"] <= parameters[1]))[0]
+
+            # If no probe entries match CN category skipping next steps
+            if len(cnv_rows) == 0:
+                continue
+
+            # If probe entries match CN category, identify uniform CN segments
+            else:
+                # gather information for appropriate probe entries
+                cnv_data = results_gene.iloc[cnv_rows]
+                # print(cnv_data)
+                # Assess based on probe order if probes are contiguous
+                cnv_groups = np.cumsum(np.concatenate(([1], np.diff(cnv_data["probe order"]) != 1)))
+                # print(cnv_groups)
+
+                # Splitting results per contiguous CN segment if multiple
+                cnv_list = {cnv_index: group for cnv_index, group in cnv_data.groupby(cnv_groups)}
+                # print(cnv_list)
+
+                # Processing data of individual CN segments
+                for cnv_index, cnv_lines in cnv_list.items():
+
+                    # Obtainins summary data of contiguous CN segments
+                    start_exon = cnv_lines["exon"].min()
+                    end_exon = cnv_lines["exon"].max()
+                    start_pos = cnv_lines["start position"].min()
+                    end_pos = cnv_lines["end position"].max()
+
+                    # Calculate average dosage quotient across CN segment and standard deviation
+                    copy_change_mean = round(cnv_lines["mean dosage quotient"].mean(), 3)
+                    copy_change_sd = round(cnv_lines["mean dosage quotient"].std(), 3)
+
+                    # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
+                    if "Ref to Scientist" in direction:
+                        copy_number_est = "-"
+                    else:
+                        copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["mean dosage quotient"]))
+
+                    # # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
+                    # if len(cnv_aggregate["Variant"].unique()) == 1 and cnv_aggregate["Variant"].str.contains("Ref to Scientist").any():
+                    #     cnv_aggregate["Copy Number Est."] = "-"
+                    # else: 
+                    #     cnv_aggregate["Copy Number Est."] = round(cnv_aggregate["Normal Copy Number"].astype(float) * cnv_aggregate["Mean Dosage Quotient"].astype(float))
+
+                    # # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
+                    # copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["mean dosage quotient"]))
+
+                    # summarize CN data into a df entry
+                    cnv_aggregate = pd.DataFrame({
+                        "Gene": [gene],
+                        "Variant": [direction],
+                        "Exon(s)": f"{start_exon} - {end_exon}" if start_exon != end_exon else str(start_exon),
+                        "Chr": cnv_lines["chr"].unique(),
+                        "Probe Co-ordinates": f"{start_pos}-{end_pos}",
+                        "Normal Copy Number": int(cnv_lines["normal copy number"].unique()), #Problem if there is more than one potential copy number?
+                        "Mean Dosage Quotient": f"{copy_change_mean}+/-{copy_change_sd}",
+                        "Est. Copy Number": [copy_number_est],
+                        "Supporting Probes": [len(cnv_lines)]
+                    })
+
+                    # Adding df entry into existing CNA_summary dataframe
+                    # TODO concat method is not ideal apparently. Consider alternative
+                    cnv_summary = pd.concat([cnv_summary,cnv_aggregate], ignore_index=True)
+
+                    # organize summary by genomic co-ordinates
+                    cnv_summary = cnv_summary.sort_values(by="Probe Co-ordinates")
+                    # print(cnv_summary)
+
+    # returning output data to backend.py
+    return cnv_summary
+
+"""UNUSED FUNCTIONS"""
+
+def gene_list(df):
+    # Might be excessive to functionalise this
+    genes_list = df["gene"].unique()
+
+    return genes_list
 
 def cna(file, file_basename, probe_file):
     """
-    TODO Add function docstring
+    Original function
     """
 
     # Import files into panda df
@@ -17,7 +249,6 @@ def cna(file, file_basename, probe_file):
 
     # Renaming columns so they match between both dataframes
     probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
-    results_raw_df.rename(columns={"Normal copy number": "Normal Copy Number"}, inplace=True)
 
     # Controlling datatype in results_raw_df
     # results_raw_df["Exon"] = results_raw_df["Exon"].astype(int)
@@ -38,20 +269,20 @@ def cna(file, file_basename, probe_file):
     # Renaming results column
     # TODO Potentially refactor this/ rename header
     current_column_header = file_basename.replace("_filtered_results.xlsx", "")
-    results_raw_df.rename(columns={current_column_header:"Mean Dosage Quotient"}, inplace=True)
+    results_raw_df.rename(columns={current_column_header:"mean dosage quotient"}, inplace=True)
 
     # defining select columns to include from each df
     columns_probe_df = [
-      "Probe number",
-      "Chr",
-      "Start Position",
-      "End Position",
-      "Position",
-      "Warnings"
+      "probe number",
+      "chr",
+      "start position",
+      "end position",
+      "position",
+      "warnings"
     ]
 
     columns_results_df = [
-        "Mean Dosage Quotient"
+        "mean dosage quotient"
     ]
 
     # merging dfs
@@ -103,7 +334,7 @@ def cna(file, file_basename, probe_file):
     # TODO Might not be needed it's just to help me visualize
 
     desired_column_order = [
-        "Probe order",
+        "Probe Order",
         "Probe number",
         "Reference probe",
         "Chr",
@@ -131,7 +362,7 @@ def cna(file, file_basename, probe_file):
 
     # Obtaining list of genes tested for
     genes_list = results_clinical["Gene"].unique()
-    print(genes_list)
+    # print(genes_list)
 
     cncategories = {
         "Ref to Scientist 1*": [0, 0.4],
@@ -228,6 +459,15 @@ def cna(file, file_basename, probe_file):
                 cnv_summary_html = cnv_summary.to_html(classes="table", index=False)
 
     # returning output data to backend.py
-    return genes_list, cnv_summary_html
+    return genes_list, cnv_summary_html, results_clinical
+
+# OLD CODE FOR MANUAL TESTING
+# genes_list, cnv_summary_html, results_clinical = cna(xls, file_name, probe)
+
+# summary_data_df = summary_data(results_clinical)
+
+# summary_tables = {}
+# for gene in genes_list:
+#     summary_tables[gene] = gene_summary_data(gene, summary_data_df)
 
-# cna(xls, file_name, probe)
+# print(summary_tables)
\ No newline at end of file
diff --git a/dMLPA-flask/qc_data.py b/dMLPA-flask/qc_data.py
index b3204eb..15e8a9a 100644
--- a/dMLPA-flask/qc_data.py
+++ b/dMLPA-flask/qc_data.py
@@ -1,40 +1,140 @@
-import os
-from datetime import datetime
-import pandas as pd
-
 """
-TODO:
-Parse
-- Patient Name = Split file by name by _ and then access 4 and 5
-- Rcode = acces 1
-- Pan Number = access 2
-- List of genes = MAYBE PARSE FROM ELSEWHERE
-- Date/RunID
-- QC passed or Failed = From final row of the QC section
+Module for pulling relevant run and raw results data from dMLPA
+reporting files.
+
+This module provides functions for extracting virtual pan number,
+obtaining QC status, converting DataFrame tables to specific HTML
+format, and generating summary results data.
 
+Functions:
+- get_pan_number(df): Return virtual pan number used for analysis.
+- get_qc_status(df): Return QC pass or fail status.
+- df_to_html(df): Convert DataFrame table to HTML without index.
+- summary_data (df): Create a summary table of key probe data.
 """
 
-# xls = "/home/epayne1/Documents/GSTT/test_input/BP01-01-dMLPA_R211v1_Pan5026_200930_SURNAME_Firstname1_filtered_results.xlsx"
-# file_name = "BP01-01-dMLPA_R211v1_Pan5026_200930_SURNAME_Firstname1"
+import re
 
-def parse_run_data(file, file_basename):
+def get_pan_numer(df):
     """
-    
+    Return virtual pan number used for analysis
+
+    Arguments:
+    - df(Pandas Dataframe): assumed to be a dataframe containing the
+    Pan(el) number within the second column header (samplename)
+
+    Return Value:
+    - pan_number(Str): GSTT virtual gene pan(el) number used to filter
+    raw NGS output
     """
-    # Parse Sample Info sheet from excel into dataframe
-    sample_info_df = pd.read_excel(file, sheet_name="Sample info")
 
-    #Saving dataframe to html format
+    # TODO:
+    # - confirm samplenaming convetion prior to release
+    # - Logging.info[panel number] and logging.error[pan number not found]
+    # - Print error onto report.
+
+    # Get samplename from df
+    samplename = df.columns[1]
+
+    # Searching for Pan number by regular expression
+    panel_number_match = re.search(r'Pan[0-9]+', samplename)
+
+    if panel_number_match:
+        # Extracting the matched PAN number
+        pan_number = panel_number_match.group(0)
+        return pan_number
+    else:
+        return None
+
+def get_qc_status(df):
+    """
+    Return qc pass or fail status
+
+    Arguments:
+    - df(Pandas Dataframe): assumed to be a dataframe containing the run
+    qc status at a specific location.
+
+    Return Value:
+    - qc_status(Str):
+    """
+    qc_status = df.loc[6, df.columns[1]].upper()
+
+    return qc_status
+
+def df_to_html(df):
+    """Convert dataframe table to html without index"""
     #TODO Format table style Including getting rid of row index
-    sample_info_html = sample_info_df.to_html(classes="table", index=False)
 
-    # Parsing test information from spreadshet
-    column_header = file_basename.replace("_filtered_results.xlsx", "")
-    qc_status = sample_info_df.loc[6, column_header].upper()
+    #Saving dataframe to html format
+    html_format = df.to_html(classes="table", index=False)
+
+    return html_format
+
+def summary_data(df):
+    """Create summary table of key probe data"""
+
+    # Merging start and end co-ordinates into one column
+    df["co-ordinates"] = df["start position"] + "-" + df["end position"]
+
+    # defining columns desired in table from existing clinical_data.
+    columns_to_keep = [
+        "probe number",
+        "gene",
+        "exon",
+        "chr",
+        "co-ordinates",
+        "normal copy number",
+        "mean dosage quotient"
+    ]
+
+    # creating copy of selected columns.
+    summary_data_df = df[columns_to_keep].copy()
+
+    # calculating copy number estimate
+    summary_data_df["copy number est."] = (
+        summary_data_df["normal copy number"] *
+        summary_data_df["mean dosage quotient"]
+    ).round()
+
+    return summary_data_df
+
+
+"""UNUSED FUNCTIONS"""
+
+def gene_summary_data(gene, df):
+    """Create summary of probe data per gene"""
+
+    summary_data_gene =  df[df["gene"] == gene]
+
+    summary_data_gene.columns = summary_data_gene.columns.str.replace("_", " ").str.title()
+
+    gene_summary_html = df_to_html(summary_data_gene)
+
+    return gene_summary_html
+
+# # function to determine estimate but I don't think this will be needed.
+# def copy_number_estimate(row):
+#     copy_number_est = round(row["Normal Copy Number"] * row["Mean Dosage Quotient"])
+#     return copy_number_est
+
+# def parse_run_data(file, file_basename):
+    # """Old function not in use"""
+
+#     # Parse Sample Info sheet from excel into dataframe
+#     sample_info_df = pd.read_excel(file, sheet_name="Sample info")
+
+#     #Saving dataframe to html format
+#     sample_info_html = sample_info_df.to_html(classes="table", index=False)
+
+#     # Parsing test information from spreadsheet
+#     column_header = file_basename.replace("_filtered_results.xlsx", "")
+#     qc_status = sample_info_df.loc[6, column_header].upper()
 
-    # Parsing sample type
-    sample_type = sample_info_df.loc[1, column_header]
+#     column_header = sample_info_df.columns[1]
+#     panel_number = re.search(r"Pan[0-9]+", column_header).group(0)
+#     qc_status = sample_info_df.loc[6, sample_info_df.columns[1]]
 
-    return sample_info_html, qc_status, sample_type
+#     print(panel_number)
+#     print(qc_status)
 
-# print(parse_run_data(xls, file_name))
\ No newline at end of file
+#     return sample_info_html, qc_status
diff --git a/dMLPA-flask/templates/report.html b/dMLPA-flask/templates/report.html
index 4e0ac78..b2a386c 100644
--- a/dMLPA-flask/templates/report.html
+++ b/dMLPA-flask/templates/report.html
@@ -73,15 +73,11 @@ <h1>dMLPA Copy Number Analysis</h1>
 <p><i>South East Genomics Laboratory Hub</i></p>
 
 <br>
-<div class="large">
-    <p><b>Patient identifier:</b><i>{{ surname }}, {{ given_name }}</i><p> <!-- TODO Consider layout reformatting. Want font larger-->
-</div>
 
 <div class="container">
     <div class="element">
         <h4>Test Request</h4>
-            <p><b>Sample Type: </b><i>{{ sample_type }}</i></p> <!-- TODO Potentially delete-->
-            <p><b>Number: </b><i>{{ pan_number }}</i></p>
+            <p><b>Pan Number: </b><i>{{ pan_number }}</i></p>
             <p><b>Genes: </b><i>{{ genes }}</i></p>
             <br>
             <p><b>Run ID: </b><i><!-- TODO To be parsed after confirming sample naming convention --></i></p>
@@ -101,25 +97,11 @@ <h4>Test Request</h4>
 <div class="body">
     <h2>Called CNV's</h2>
         <p> {{ cnv_summary }}</p>
-        {% if "*Ref to Scientist 1" is in cnv_summary %}
-            <p>Ref to Scientist 1: The mean dosage quotient for one or more probes falls
-                below the range expected for a copy number loss. Scientist review required.
-            </p>
-        {% elif "*Ref to Scientist 2" is in cnv_summary %}
-            <p>Ref to Scientist 2: The mean dosage quotient for one or more probes falls
-                in between the expected ranges for copy number loss and copy number neutral.
-                Scientist review required.
-            </p>
-        {% elif "*Ref to Scientist 3" is in cnv_summary %}
-            <p>Ref to Scientist 3: The mean dosage quotient for one or more probes falls
-                in between the expected ranges for copy number neutral and copy number gain.
-                Scientist review required.
-            </p>
-        {% elif "*Ref to Scientist 4" is in cnv_summary %}
-            <p>Ref to Scientist 4: The mean dosage quotient for one or more probes falls
-                above the range expected for a copy number gain. Scientist review required.
-            </p>
-        {% endif %}
+        {% for key, value in report_notes.items() %}
+            {% if key in cnv_summary %}
+                <p> {{ value}} </p>
+            {% endif %}
+        {% endfor %}
 </div>
 
 <div class="body">
@@ -134,10 +116,13 @@ <h3>CNV details</h3>
     <h2>Appendix</h2>
     <br>
     <h3>QC Data</h3>
-        <p> {{ html_table }}</p>
+        <p> {{ qc_table }}</p>
     <br>
     <h3>Raw Data</h3>
-        <p>INSERT TABLES</p> <!-- TODO This will require a for loop -->
+        {% for gene, summary_table in summary_tables.items() %}
+        <p>{{ gene }}</p>
+        <p>{{ summary_table }}</p>
+        {% endfor %}
 </div>
 
 </body>

From 00e2480d147fbfbc94a993eb919181e36e7a6161 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Mon, 8 Apr 2024 13:18:20 +0100
Subject: [PATCH 07/15] Adds draft of copy number graph

Also corrects mistake of using "mean dosage quotient" as apposed
to "dosage quotient" throughout.
---
 dMLPA-flask/backend.py            |  17 ++--
 dMLPA-flask/cna.py                |  18 ++--
 dMLPA-flask/plotting.py           | 142 ++++++++++++++++++++++++++++++
 dMLPA-flask/qc_data.py            |   4 +-
 dMLPA-flask/templates/report.html |   4 +-
 5 files changed, 166 insertions(+), 19 deletions(-)
 create mode 100644 dMLPA-flask/plotting.py

diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index b84b3b3..bb0c62f 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -55,11 +55,13 @@ def run_backend(single_file, array_of_files, output_directory):
         probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
         probe_df = split_df_column(probe_df, "Mapview (hg38)a", probe_column_headers)
         probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
-        raw_results_df.rename(columns={raw_results_df.columns[-1]:"mean dosage quotient"}, inplace=True)
+        raw_results_df.rename(columns={raw_results_df.columns[-1]:"dosage quotient"}, inplace=True)
+
+        print(raw_results_df)
 
         # Definining unique columns needed from df's
         columns_results_df = [
-            "mean dosage quotient"
+            "dosage quotient"
         ]
 
         columns_probe_df = [
@@ -82,28 +84,31 @@ def run_backend(single_file, array_of_files, output_directory):
         # defining review messages
         report_notes = {
             "Ref to Scientist 1": (
-                "Ref to Scientist 1: The mean dosage quotient for one or "\
+                "Ref to Scientist 1: The dosage quotient for one or "\
                 "more probes falls below the range expected for a copy "\
                 "number loss. Scientist review required."
             ),
             "Ref to Scientist 2": (
-                "Ref to Scientist 2: The mean dosage quotient for one or "\
+                "Ref to Scientist 2: The dosage quotient for one or "\
                 "more probes falls in between the expected ranges for "\
                 "copy number loss and copy number neutral. Scientist review "\
                 "required"
             ),
             "Ref to Scientist 3": (
-                "Ref to Scientist 3: The mean dosage quotient for one or "\
+                "Ref to Scientist 3: The dosage quotient for one or "\
                 "more probes falls in between the expected ranges for copy "\
                 "number neutral and copy number gain. Scientist review "\
                 "required."
             ),
             "Ref to Scientist 4": (
-                "Ref to Scientist 4: The mean dosage quotient for one or "\
+                "Ref to Scientist 4: The dosage quotient for one or "\
                 "more probes falls above the range expected for a copy "\
                 "number gain. Scientist review required."
             )
         }
+        # creating cnv plots and cnv summary data
+        cnv_data = {}
+        
 
         # getting panel number and qc data from metadata
         # TODO consider if i shouldn't have this futher down rather than at beginning.
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
index 673839f..6f699ef 100644
--- a/dMLPA-flask/cna.py
+++ b/dMLPA-flask/cna.py
@@ -113,7 +113,7 @@ def clinical_df(df1, df2, df1_columns, df2_columns):
         "start position",
         "end position",
         "normal copy number",
-        "mean dosage quotient",
+        "dosage quotient",
         "probe type",
         "additional information",
     ]
@@ -158,8 +158,8 @@ def copy_number_analysis(df, gene_list):
         # iterate through CN categories and identify matching probe entries
         for direction, parameters in cncategories.items():
             # print(direction, gene) # TODO Replace with logging
-            cnv_rows = np.where((parameters[0] <= results_gene["mean dosage quotient"]) &
-                    (results_gene["mean dosage quotient"] <= parameters[1]))[0]
+            cnv_rows = np.where((parameters[0] <= results_gene["dosage quotient"]) &
+                    (results_gene["dosage quotient"] <= parameters[1]))[0]
 
             # If no probe entries match CN category skipping next steps
             if len(cnv_rows) == 0:
@@ -176,7 +176,7 @@ def copy_number_analysis(df, gene_list):
 
                 # Splitting results per contiguous CN segment if multiple
                 cnv_list = {cnv_index: group for cnv_index, group in cnv_data.groupby(cnv_groups)}
-                # print(cnv_list)
+                print(cnv_list)
 
                 # Processing data of individual CN segments
                 for cnv_index, cnv_lines in cnv_list.items():
@@ -188,14 +188,14 @@ def copy_number_analysis(df, gene_list):
                     end_pos = cnv_lines["end position"].max()
 
                     # Calculate average dosage quotient across CN segment and standard deviation
-                    copy_change_mean = round(cnv_lines["mean dosage quotient"].mean(), 3)
-                    copy_change_sd = round(cnv_lines["mean dosage quotient"].std(), 3)
+                    copy_change_mean = round(cnv_lines["dosage quotient"].mean(), 3)
+                    copy_change_sd = round(cnv_lines["dosage quotient"].std(), 3)
 
-                    # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
+                    # Calculating Estimated Copy number for CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
                     if "Ref to Scientist" in direction:
                         copy_number_est = "-"
                     else:
-                        copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["mean dosage quotient"]))
+                        copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["dosage quotient"]))
 
                     # # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
                     # if len(cnv_aggregate["Variant"].unique()) == 1 and cnv_aggregate["Variant"].str.contains("Ref to Scientist").any():
@@ -204,7 +204,7 @@ def copy_number_analysis(df, gene_list):
                     #     cnv_aggregate["Copy Number Est."] = round(cnv_aggregate["Normal Copy Number"].astype(float) * cnv_aggregate["Mean Dosage Quotient"].astype(float))
 
                     # # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
-                    # copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["mean dosage quotient"]))
+                    # copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["dosage quotient"]))
 
                     # summarize CN data into a df entry
                     cnv_aggregate = pd.DataFrame({
diff --git a/dMLPA-flask/plotting.py b/dMLPA-flask/plotting.py
new file mode 100644
index 0000000..1869a6c
--- /dev/null
+++ b/dMLPA-flask/plotting.py
@@ -0,0 +1,142 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+from cna import shared_columns, split_df_column, merge_df, clinical_df, gene_list, copy_number_analysis
+
+
+def cnv_plot(df, gene, extend=1000):
+    # create copy of dataframe
+    cnv_copy = df.copy()
+
+    # pull specific data from columns
+    # gene = cnv_copy["gene"].unique
+    # chrom = cnv_copy["chr"].unique
+    start = min(cnv_copy["start position"]) + extend
+    end = max(cnv_copy["end position"]) + extend
+
+    # determining y-axis limit for DQ plot
+    ylim_ax1 = (0, max(2.0, 1.25*cnv_copy["dosage quotient"].max()))
+
+    #setting up plot layout
+    fig = plt.figure(figsize=(8,6))
+    gs = fig.add_gridspec(nrows=2, ncols=1, height_ratios=[3,1])
+
+    ax1 = plt.subplot(gs[0])
+    ax1.set_xlim(start,end)
+    ax1.set_ylim(*ylim_ax1)
+    plt.ylabel("Dosage Quotient")
+    plt.setp(ax1.get_xticklabels(), visible=False)
+    plt.title(gene)
+
+    ax2 = plt.subplot(gs[1], sharex = ax1)
+    ax2.set_xlim(start,end)
+    ax2.set_ylim(0,1)
+    plt.setp(ax2.get_yticklabels(), visible=False)
+
+    #n.b. this ensures that we are getting the full integer x-axis rather than scientific notation.
+    # but this is from chatGPT and i need to question if this is the best way to do this. 
+    ax2.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: '{:d}'.format(int(x))))
+
+    # adding shading to define ranges for normal(green)
+    # hetdel (red), hetdup (blue)
+    normal_dq_range = (0.8, 1.2)
+    ax1.axhspan(normal_dq_range[0], normal_dq_range[1], facecolor = "#00FF0044", edgecolor = None)
+    het_del_dq_range = (0.4,0.6)
+    ax1.axhspan(het_del_dq_range[0], het_del_dq_range[1], facecolor = "#FF000044", edgecolor = None)
+    het_dup_dq_range = (1.4,1.6)
+    ax1.axhspan(het_dup_dq_range[0], het_dup_dq_range[1], facecolor = "#0000FF44", edgecolor = None)
+
+    # plot DQ value
+    ax1.plot(cnv_copy["start position"], cnv_copy["dosage quotient"], marker = "x", linestyle = "", color = "black")
+
+    plt.show()
+
+    return fig
+
+# Test files to test script
+xls = "/home/epayne1/Documents/GSTT/test_input/BP01-11-dMLPA_R210v1_Pan5025_185897_SURNAME_Firstname11_filtered_results.xlsx"
+probe = "/home/epayne1/Documents/GSTT/probe_file/PIF_D001-C1_Hereditary_Cancer_Panel_1-v02_modB1probe.xlsx"
+
+# Reading data in dataframes
+metadata_df = pd.read_excel(xls, sheet_name="Sample info")
+raw_results_df = pd.read_excel(xls, sheet_name="Gene filtered results")
+probe_df = pd.read_excel(probe, sheet_name="probe_info")
+
+# Defining new column names
+probe_column_headers = [
+    "chr",
+    "start position",
+    "end position"
+]
+
+# hard manipulation/formatting of df's
+probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
+probe_df = split_df_column(probe_df, "Mapview (hg38)a", probe_column_headers)
+probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
+raw_results_df.rename(columns={raw_results_df.columns[-1]:"dosage quotient"}, inplace=True)
+
+# Definining unique columns needed from df's
+columns_results_df = [
+    "dosage quotient"
+]
+
+columns_probe_df = [
+    "probe number",
+    "chr",
+    "start position",
+    "end position",
+]
+
+# creating results_clinical dataframe
+results_clinical_df = clinical_df(raw_results_df, probe_df, columns_results_df, columns_probe_df)
+results_clinical_df["start position"] = results_clinical_df["start position"].astype(int)
+results_clinical_df["end position"] = results_clinical_df["end position"].astype(int)
+print(results_clinical_df)
+results_clinical_df.to_excel("merged_excel.xlsx")
+
+# get list of genes
+gene_list = ["MSH6"]
+
+## perform copy number analysis
+# cnv_summary_df = copy_number_analysis(results_clinical_df, gene_list)
+# print(cnv_summary_df)
+
+# perform copy number analysis
+cnv_summary_df = copy_number_analysis(results_clinical_df, gene_list)
+
+for gene in gene_list:
+    results_gene = results_clinical_df[results_clinical_df["gene"] == gene]
+    print(results_gene)
+
+    cnv_plot(results_gene, gene)
+
+    # plt.switch_backend('TkAgg')
+
+    # # defining x and y axis ranges
+    # xmin = results_gene["start position"].min()
+    # xmax = results_gene["end position"].max()
+    # ymin = 0
+
+    # if results_gene["mean dosage quotient"].max() > 2.5:
+    #     ymax = 1.25 * results_gene["mean dosage quotient"].max()
+    # else:
+    #     ymax = 2.5
+
+    # #plotting per gene
+    # plt.figure(figsize=(12,6))
+
+    # ax1 = plt.subplot(211)
+    # # plt.axis([xmin, xmax, ymin, ymax])
+    # plt.plot(results_clinical_df["start position"], results_clinical_df["mean dosage quotient"], "rx-")
+    # plt.xlabel("Probe start co-ordinate")
+    # plt.ylabel("Mean Dosage Quotient")
+    # plt.setp(ax1.get_xticklabels(), visible=False)
+    # plt.title(gene)
+
+    # ax2 = plt.subplot(212, sharex=ax1)
+    # plt.plot(results_clinical_df["start position"], results_clinical_df["exon"])
+    # plt.setp(ax2.get_xticklabels(), fontsize=6)
+
+    # plt.show()
+
diff --git a/dMLPA-flask/qc_data.py b/dMLPA-flask/qc_data.py
index 15e8a9a..adefb7c 100644
--- a/dMLPA-flask/qc_data.py
+++ b/dMLPA-flask/qc_data.py
@@ -84,7 +84,7 @@ def summary_data(df):
         "chr",
         "co-ordinates",
         "normal copy number",
-        "mean dosage quotient"
+        "dosage quotient"
     ]
 
     # creating copy of selected columns.
@@ -93,7 +93,7 @@ def summary_data(df):
     # calculating copy number estimate
     summary_data_df["copy number est."] = (
         summary_data_df["normal copy number"] *
-        summary_data_df["mean dosage quotient"]
+        summary_data_df["dosage quotient"]
     ).round()
 
     return summary_data_df
diff --git a/dMLPA-flask/templates/report.html b/dMLPA-flask/templates/report.html
index b2a386c..f019137 100644
--- a/dMLPA-flask/templates/report.html
+++ b/dMLPA-flask/templates/report.html
@@ -106,8 +106,6 @@ <h2>Called CNV's</h2>
 
 <div class="body">
     <h3>CNV details</h3>
-        <p>INSERT PLOT DIAGRAM</p>
-        <p>INSERT PROBE TABLE</p>
 </div>
 
 <div class="page-break"></div>
@@ -117,11 +115,13 @@ <h2>Appendix</h2>
     <br>
     <h3>QC Data</h3>
         <p> {{ qc_table }}</p>
+        <div class="page-break"></div>
     <br>
     <h3>Raw Data</h3>
         {% for gene, summary_table in summary_tables.items() %}
         <p>{{ gene }}</p>
         <p>{{ summary_table }}</p>
+        <div class="page-break"></div>
         {% endfor %}
 </div>
 

From fe7c194233e0185611974136eb1474f88d7063d5 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Thu, 2 May 2024 15:47:41 +0100
Subject: [PATCH 08/15] Adds final reporting elements into report

Including:
- One matplotlib generated graph per gene with identified CNV's
- One summary table of probes associated with CNV's per gene.
---
 dMLPA-flask/backend.py            |  55 ++++--
 dMLPA-flask/cna.py                | 298 ++++++------------------------
 dMLPA-flask/notes.py              | 159 ++++++++++++++++
 dMLPA-flask/plotting.py           | 142 --------------
 dMLPA-flask/qc_data.py            |  88 ++++-----
 dMLPA-flask/templates/report.html |  14 +-
 requirements.txt                  |   5 +-
 7 files changed, 306 insertions(+), 455 deletions(-)
 create mode 100644 dMLPA-flask/notes.py
 delete mode 100644 dMLPA-flask/plotting.py

diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index bb0c62f..e7f9573 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -4,8 +4,8 @@
 from jinja2 import Environment, PackageLoader
 import pdfkit
 import pandas as pd
-from qc_data import get_pan_numer, df_to_html, get_qc_status, summary_data
-from cna import split_df_column, clinical_df, copy_number_analysis
+from qc_data import get_test_codes, df_to_html, get_qc_status, summary_data
+from cna import split_df_column, clinical_df, copy_number_analysis, cnv_plot
 
 img_versioned = os.getenv(
     "IMG_VERSIONED"
@@ -73,12 +73,17 @@ def run_backend(single_file, array_of_files, output_directory):
 
         # creating results_clinical dataframe
         results_clinical_df = clinical_df(raw_results_df, probe_df, columns_results_df, columns_probe_df)
+        # results_clinical_df.to_excel("merged_excel.xlsx")
+
+        # controlling dataypes in clinical dataframe
+        results_clinical_df["start position"] = results_clinical_df["start position"].astype(int)
+        results_clinical_df["end position"] = results_clinical_df["end position"].astype(int)
 
         # get list of genes
         gene_list = results_clinical_df["gene"].unique()
 
         # perform copy number analysis
-        cnv_summary_df = copy_number_analysis(results_clinical_df, gene_list)
+        cnv_summary_df, marked_df = copy_number_analysis(results_clinical_df, gene_list)
         summary_html = df_to_html(cnv_summary_df)
 
         # defining review messages
@@ -106,29 +111,42 @@ def run_backend(single_file, array_of_files, output_directory):
                 "number gain. Scientist review required."
             )
         }
-        # creating cnv plots and cnv summary data
-        cnv_data = {}
-        
-
-        # getting panel number and qc data from metadata
-        # TODO consider if i shouldn't have this futher down rather than at beginning.
-        pan_number = get_pan_numer(metadata_df)
-        qc_status = get_qc_status(metadata_df)
-        qc_data = df_to_html(metadata_df)
-
-        # Converting gene_list to string
-        genes = ", ".join(gene_list)
 
         # obtaining summary data per gene for appendix
-        summary_data_df = summary_data(results_clinical_df)
+        summary_data_df = summary_data(marked_df)
         print(summary_data_df)
 
         summary_tables = {}
+        cnv_detail = {}
         for gene in gene_list:
+            # summary data by gene
             summary_data_gene =  summary_data_df[summary_data_df["gene"] == gene]
             summary_data_gene.columns = summary_data_gene.columns.str.replace("_", " ").str.title()
             summary_tables[gene] = df_to_html(summary_data_gene)
-            # summary_tables[gene] = gene_summary_data(gene, summary_data_df)c
+
+            # cnv_detail by impacted gene
+            impacted_genes = cnv_summary_df["Gene"].unique()
+            if gene not in impacted_genes:
+                continue
+            else:
+                results_gene = marked_df[marked_df["gene"] == gene]
+                gene_plot = cnv_plot(results_gene, gene)
+                impacted_probes = summary_data_gene.dropna(subset=["Variant"])
+                impacted_probes = summary_data_gene.loc[summary_data_gene["Variant"] != ""]
+                impacted_probes = (impacted_probes.sort_values(by="Start Position"))
+                impacted_probes_html = df_to_html(impacted_probes)
+
+                # saving to cnv_detail dict.
+                cnv_detail[gene] = [gene_plot, impacted_probes_html]
+
+        # getting panel number and qc data from metadata
+        # TODO consider if i shouldn't have this futher down rather than at beginning.
+        pan_number, r_code = get_test_codes(metadata_df)
+        qc_status = get_qc_status(metadata_df)
+        qc_data = df_to_html(metadata_df)
+
+        # Converting gene_list to string
+        genes = ", ".join(gene_list)
 
         # Rendering template and saving as pdf
 
@@ -136,13 +154,14 @@ def run_backend(single_file, array_of_files, output_directory):
         template = env.get_template("report.html")
         data_values = {
             "file_name": report,
-            # "r_code": r_code,
+            "r_code": r_code,
             "pan_number": pan_number,
             "genes": genes,
             "patient_file": file_name,
             "probe_file": probe_file,
             "qc_status": qc_status,
             "cnv_summary": summary_html,
+            "cnv_detail": cnv_detail,
             "report_notes": report_notes,
             "qc_table": qc_data,
             "summary_tables": summary_tables
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
index 6f699ef..f29c42d 100644
--- a/dMLPA-flask/cna.py
+++ b/dMLPA-flask/cna.py
@@ -13,6 +13,11 @@
 
 import pandas as pd
 import numpy as np
+import io
+import base64
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
 
 def shared_columns(df1, df2):
     """Identify shared dataframe columns"""
@@ -149,6 +154,11 @@ def copy_number_analysis(df, gene_list):
     # Empty dataframe for all identified copy number variations
     cnv_summary = pd.DataFrame()
 
+    
+    # Create 'variant' column with object dtype
+    df['variant'] = ""  # Initialize with NaN values, which are compatible with object dtype
+    df["variant"] = df["variant"].astype(object) #controls expected type and prevents FutureError observed in manual testing
+
     # TODO This is mostly from chatGPT - Need to understand what it's actually doing
     for gene in gene_list:
 
@@ -176,10 +186,19 @@ def copy_number_analysis(df, gene_list):
 
                 # Splitting results per contiguous CN segment if multiple
                 cnv_list = {cnv_index: group for cnv_index, group in cnv_data.groupby(cnv_groups)}
+
                 print(cnv_list)
 
                 # Processing data of individual CN segments
                 for cnv_index, cnv_lines in cnv_list.items():
+                    cnv_lines = pd.DataFrame(cnv_lines)
+                    # print(cnv_lines)
+
+                    # Update the input DataFrame by adding a new column containing the CN category
+                    probe_numbers = cnv_lines["probe number"]
+                    for probe_number in probe_numbers:
+                        df.loc[df["probe number"] == probe_number, 'variant'] = direction
+
 
                     # Obtainins summary data of contiguous CN segments
                     start_exon = cnv_lines["exon"].min()
@@ -192,19 +211,12 @@ def copy_number_analysis(df, gene_list):
                     copy_change_sd = round(cnv_lines["dosage quotient"].std(), 3)
 
                     # Calculating Estimated Copy number for CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
+                    # TODO Does datatype need to be considered before calculation?
                     if "Ref to Scientist" in direction:
                         copy_number_est = "-"
                     else:
                         copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["dosage quotient"]))
 
-                    # # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
-                    # if len(cnv_aggregate["Variant"].unique()) == 1 and cnv_aggregate["Variant"].str.contains("Ref to Scientist").any():
-                    #     cnv_aggregate["Copy Number Est."] = "-"
-                    # else: 
-                    #     cnv_aggregate["Copy Number Est."] = round(cnv_aggregate["Normal Copy Number"].astype(float) * cnv_aggregate["Mean Dosage Quotient"].astype(float))
-
-                    # # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
-                    # copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["dosage quotient"]))
 
                     # summarize CN data into a df entry
                     cnv_aggregate = pd.DataFrame({
@@ -228,246 +240,58 @@ def copy_number_analysis(df, gene_list):
                     # print(cnv_summary)
 
     # returning output data to backend.py
-    return cnv_summary
-
-"""UNUSED FUNCTIONS"""
-
-def gene_list(df):
-    # Might be excessive to functionalise this
-    genes_list = df["gene"].unique()
-
-    return genes_list
-
-def cna(file, file_basename, probe_file):
-    """
-    Original function
-    """
-
-    # Import files into panda df
-    probe_df = pd.read_excel(probe_file, sheet_name="probe_info")
-    results_raw_df = pd.read_excel(file, sheet_name="Gene filtered results")
-
-    # Renaming columns so they match between both dataframes
-    probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
-
-    # Controlling datatype in results_raw_df
-    # results_raw_df["Exon"] = results_raw_df["Exon"].astype(int)
-    # TODO Consider this. Some are "up" or "down" so can't be converted to int
-
-    # Separating chr, start and end co-ordinates into separate columns
-    probe_df[["Chr", "Start Position",  "End Position"]] = probe_df["Mapview (hg38)a"].str.split(r"[:\-]", expand=True)
-
-    # Manipulting data in Chr. 
-    probe_df["Chr"] = probe_df["Chr"].str.replace("chr", "").str.lstrip("0")
-
-    # TODO Having trouble converting data type to int
-    # results_raw_df[["Start Position", "End Position"]] = results_raw_df[["Start Position", "End Position"]].astype(int)
-
-    # Finding shared columns
-    shared_columns = list(set(probe_df.columns) & set(results_raw_df.columns))
-
-    # Renaming results column
-    # TODO Potentially refactor this/ rename header
-    current_column_header = file_basename.replace("_filtered_results.xlsx", "")
-    results_raw_df.rename(columns={current_column_header:"mean dosage quotient"}, inplace=True)
-
-    # defining select columns to include from each df
-    columns_probe_df = [
-      "probe number",
-      "chr",
-      "start position",
-      "end position",
-      "position",
-      "warnings"
-    ]
-
-    columns_results_df = [
-        "mean dosage quotient"
-    ]
-
-    # merging dfs
-
-    # results_probes = pd.merge(
-    #     probe_df[shared_columns + columns_probe_df],
-    #     results_raw_df[shared_columns + columns_results_df],
-    #     on=shared_columns,
-    #     how="left",
-    #     suffixes = ("_probe", "_results")
-    # )
+    return cnv_summary, df
 
-    results_probes_df = results_raw_df[shared_columns + columns_results_df].merge(
-        probe_df[columns_probe_df],
-        how = "left",
-        on = "Probe number"
-    )
-    # print(results_probes_df)
+def cnv_plot(df, gene, extend=1000):
+    # create copy of dataframe
+    cnv_copy = df.copy()
 
-    # # TESTING Script:Exploring unique row entries
-    # # rows that are in results raw but not in merged pdf.
-    # unique_rows_results_df = results_raw_df[~results_raw_df["Probe number"].isin(results_probes_df["Probe number"])]
+    # pull specific data from columns
+    start = min(cnv_copy["start position"]) - extend
+    end = max(cnv_copy["end position"]) + extend
 
-    # # rows that are in merged pdf but no in results raw.
-    # unique_rows_merge_df = results_probes_df[~results_probes_df["Probe number"].isin(results_raw_df["Probe number"])]
+    # determining y-axis limit for DQ plot
+    ylim = (0, max(2.0, 1.25*cnv_copy["dosage quotient"].max()))
 
-    # # rows that are in results but not in the probe file
-    # unique_rows_not_in_probe = results_raw_df[~results_raw_df["Probe number"].isin(probe_df["Probe number"])]
-    # # unique_rows_probes_df = probe_df[~probe_df["Probe number"].isin(results_raw_df["Probe number"])]
-    # number_unique_results = len(unique_rows_results_df)
-    # number_unique_merge = len(unique_rows_merge_df)
-    # number_unique_not_in_probe = len(unique_rows_not_in_probe)
-    # # number_unique_in_probe = len(unique_rows_probes_df)
+    #setting up plot layout
+    plt.figure(figsize=(10,4))
 
-    # print(number_unique_merge)
-    # if number_unique_merge != 0:
-    #     print(unique_rows_merge_df)
+    # setting axis
+    plt.xlim(start,end)
+    plt.ylim(*ylim)
+    plt.tick_params(axis='both', labelsize=8)
 
-    # print(number_unique_results)
-    # if number_unique_results != 0:
-    #     print(unique_rows_results_df)
+    # setting plot labels
+    plt.ylabel("Dosage Quotient", fontdict={'family': 'Sans', 'size': 8})
+    plt.xlabel("Co-ordinates", fontdict={'family': 'Sans', 'size': 8})
+    plt.title(gene, fontdict={'family': 'Sans', 'size': 10})
 
-    # print(number_unique_not_in_probe)
-    # if number_unique_not_in_probe != 0:
-    #     print(unique_rows_not_in_probe)
+    # formatting x-axis ticks
+    # but this is from chatGPT and i need to question if this is the best way to do this.
+    plt.gca().xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: '{:d}'.format(int(x))))
 
+    # adding shading to define ranges for normal, hetdel and hetdup
+    # TODO Could be done from config.
+    normal_dq_range = (0.8, 1.2)
+    plt.axhspan(normal_dq_range[0], normal_dq_range[1], facecolor = "#00FF0044", edgecolor = None)
+    het_del_dq_range = (0.4,0.6)
+    plt.axhspan(het_del_dq_range[0], het_del_dq_range[1], facecolor = "#FF000044", edgecolor = None)
+    het_dup_dq_range = (1.4,1.6)
+    plt.axhspan(het_dup_dq_range[0], het_dup_dq_range[1], facecolor = "#318CE744", edgecolor = None)
 
-    # Defining the order of the columns
-    # TODO Might not be needed it's just to help me visualize
-
-    desired_column_order = [
-        "Probe Order",
-        "Probe number",
-        "Reference probe",
-        "Chr",
-        "Gene",
-        "Exon",
-        "Start Position",
-        "End Position",
-        "Normal Copy Number",
-        "Mean Dosage Quotient",
-        "Probe type",
-        "Additional information",
-        "Position",
-        "Warnings",
-    ]
-
-    # restructuring df based on column order
-    results_probes_df = results_probes_df[desired_column_order]
+    # plot DQ value
+    plt.plot(cnv_copy["start position"], cnv_copy["dosage quotient"], marker = "x", linestyle = "", color = "black")
 
-    # Filtering results by probetype
-    results_clinical = results_probes_df[results_probes_df["Probe type"] != "CTRL"]
-
-    # TODO Delete in final code. This is only to test outputs.
-    # print(results_clinical)
-    # results_clinical.to_excel("merged_excel.xlsx")
-
-    # Obtaining list of genes tested for
-    genes_list = results_clinical["Gene"].unique()
-    # print(genes_list)
-
-    cncategories = {
-        "Ref to Scientist 1*": [0, 0.4],
-        "HetDel": [0.4, 0.6],  # theoretical value 0.5, represents CN 1
-        "Ref to Scientist 2*": [0.6, 0.8],
-        # "normal": [0.8, 1.2],  # theoretical value 1, represents CN 2
-        "Ref to Scientist 3*": [1.2, 1.4],
-        "HetDup": [1.4, 1.6],  # theoretical value 1.5, represents CN 3
-        "Ref to Scientist 4*": [1.6, float("Inf")]
-    }
-
-    # Creating empty dataframe for all identified copy number variations
-    cnv_summary = pd.DataFrame()
-
-    # Creating dictionary list for details on the cnv
-    cnv_detail = {}
-
-    # TODO This is mostly from chatGPT - Need to understand what it's actually doing
-    for gene in genes_list:
-
-        # Get results per gene
-        results_gene =  results_clinical[results_clinical["Gene"] == gene]
-
-        # iterate through CN categories and identify matching probe entries
-        for direction, parameters in cncategories.items():
-            print(direction, gene) # TODO Replace with logging
-            cnv_rows = np.where((parameters[0] <= results_gene["Mean Dosage Quotient"]) &
-                    (results_gene["Mean Dosage Quotient"] < parameters[1]))[0]
-
-            # If no probe entries match CN category skipping next steps
-            if len(cnv_rows) == 0:
-                continue
-
-            # If probe entries match CN category, identify uniform CN segments
-            else:
-                # gather information for appropriate probe entries
-                cnv_data = results_gene.iloc[cnv_rows]
-                print(cnv_data)
-                # Assess based on probe order if probes are contiguous
-                cnv_groups = np.cumsum(np.concatenate(([1], np.diff(cnv_data["Probe order"]) != 1)))
-                print(cnv_groups)
-
-                # Splitting results per contiguous CN segment if multiple
-                cnv_list = {cnv_index: group for cnv_index, group in cnv_data.groupby(cnv_groups)}
-                print(cnv_list)
-
-                # Processing data of individual CN segments
-                for cnv_index, cnv_lines in cnv_list.items():
-
-                    # Obtainins summary data of contiguous CN segments
-                    start_exon = cnv_lines["Exon"].min()
-                    end_exon = cnv_lines["Exon"].max()
-                    start_pos = cnv_lines["Start Position"].min()
-                    end_pos = cnv_lines["End Position"].max()
-
-                    # Calculate average dosage quotient across CN segment and standard deviation
-                    copy_change_mean = round(cnv_lines["Mean Dosage Quotient"].mean(), 3)
-                    copy_change_sd = round(cnv_lines["Mean Dosage Quotient"].std(), 3)
-
-                # Calculating Estimated Copy number fo CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
-                    copy_number_est = round(np.mean(cnv_lines["Normal Copy Number"] * cnv_lines["Mean Dosage Quotient"]))
-
-                    # summarize CN data into a df entry
-                    cnv_aggregate = pd.DataFrame({
-                        "Gene": [gene],
-                        "Variant": [direction],
-                        "Exon(s)": f"{start_exon} - {end_exon}" if start_exon != end_exon else str(start_exon),
-                        "Chr": cnv_lines["Chr"].unique(),
-                        "Probe Co-ordinates": f"{start_pos}-{end_pos}",
-                        "Mean Dosage Quotient": f"{copy_change_mean}+/-{copy_change_sd}",
-                        "Est. Copy Number": [copy_number_est],
-                        "Supporting Probes": [len(cnv_lines)]
-                    })
-
-                    # Adding df entry into existing CNA_summary dataframe
-                    # TODO concat method is not ideal apparently. Consider alternative
-                    cnv_summary = pd.concat([cnv_summary,cnv_aggregate], ignore_index=True)
-
-                    # organize summary by genomic co-ordinates
-                    cnv_summary = cnv_summary.sort_values(by="Probe Co-ordinates")
-
-                    # print(cnv_summary)
-
-                    # # Saving probe details of CN segment
-                    # # TODO Need to figure out how to build this to then be able to parse to front end.
-                    # cnv_context = results_gene[results_gene["Probe number"]].isin(cnv_lines["Probe number"])
-                    # cnv_context["marked_cnv"] = cnv_context["Probe number"].isin(cnv_lines["Probe number"])
-                    # cna_detail[len(cna_summary)] = cnv_context    # unique_rows_results_df = results_raw_df[~results_raw_df["Probe number"].isin(results_probes["Probe number"])]
-
-
-                    # print(cna_detail)
-
-                # Generating cnv_summary as html
-                cnv_summary_html = cnv_summary.to_html(classes="table", index=False)
-
-    # returning output data to backend.py
-    return genes_list, cnv_summary_html, results_clinical
+    # plt.show()
 
-# OLD CODE FOR MANUAL TESTING
-# genes_list, cnv_summary_html, results_clinical = cna(xls, file_name, probe)
+    # Save plot as image
+    img = io.BytesIO()
+    plt.savefig(img, format="svg")
+    img.seek(0)
 
-# summary_data_df = summary_data(results_clinical)
+    # Encode SVG image as Base64
+    img_base64 = base64.b64encode(img.getvalue()).decode()
 
-# summary_tables = {}
-# for gene in genes_list:
-#     summary_tables[gene] = gene_summary_data(gene, summary_data_df)
+    plt.close()
 
-# print(summary_tables)
\ No newline at end of file
+    return img_base64
\ No newline at end of file
diff --git a/dMLPA-flask/notes.py b/dMLPA-flask/notes.py
new file mode 100644
index 0000000..5167818
--- /dev/null
+++ b/dMLPA-flask/notes.py
@@ -0,0 +1,159 @@
+import pandas as pd
+import numpy as np
+from cna import split_df_column, merge_df, clinical_df, copy_number_analysis, cnv_plot
+from qc_data import summary_data, df_to_html
+
+from jinja2 import Environment, PackageLoader
+import pdfkit
+
+# Test files to test script
+xls = "/home/epayne1/Documents/GSTT/test_input/BP01-11-dMLPA_R210v1_Pan5025_185897_SURNAME_Firstname11_filtered_results.xlsx"
+probe = "/home/epayne1/Documents/GSTT/probe_file/PIF_D001-C1_Hereditary_Cancer_Panel_1-v02_modB1probe.xlsx"
+
+# Reading data in dataframes
+metadata_df = pd.read_excel(xls, sheet_name="Sample info")
+raw_results_df = pd.read_excel(xls, sheet_name="Gene filtered results")
+probe_df = pd.read_excel(probe, sheet_name="probe_info")
+
+# Defining new column names
+probe_column_headers = [
+    "chr",
+    "start position",
+    "end position"
+]
+
+# hard manipulation/formatting of df's
+probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
+probe_df = split_df_column(probe_df, "Mapview (hg38)a", probe_column_headers)
+probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
+raw_results_df.rename(columns={raw_results_df.columns[-1]:"dosage quotient"}, inplace=True)
+
+# Definining unique columns needed from df's
+columns_results_df = [
+    "dosage quotient"
+]
+
+columns_probe_df = [
+    "probe number",
+    "chr",
+    "start position",
+    "end position",
+]
+
+# creating results_clinical dataframe
+results_clinical_df = clinical_df(raw_results_df, probe_df, columns_results_df, columns_probe_df)
+results_clinical_df["start position"] = results_clinical_df["start position"].astype(int)
+results_clinical_df["end position"] = results_clinical_df["end position"].astype(int)
+# results_clinical_df.to_excel("merged_excel.xlsx")
+
+# get list of genes
+gene_list = results_clinical_df["gene"].unique()
+
+# perform copy number analysis
+cnv_summary_df, marked_df = copy_number_analysis(results_clinical_df, gene_list)
+# marked_df.to_excel("marked_df.xlsx")
+
+# obtaining summary data per gene for appendix
+summary_data_df = summary_data(marked_df)
+# print(summary_data_df)
+
+summary_tables = {}
+cnv_detail = {}
+for gene in gene_list:
+    # summary data by gene
+    summary_data_gene =  summary_data_df[summary_data_df["gene"] == gene]
+    print(summary_data_gene)
+    summary_data_gene.columns = summary_data_gene.columns.str.replace("_", " ").str.title()
+    summary_tables[gene] = df_to_html(summary_data_gene)
+    # summary_tables[gene] = gene_summary_data(gene, summary_data_df)
+
+    # cnv_detail by impacted gene
+    impacted_genes = cnv_summary_df["Gene"].unique()
+    if gene not in impacted_genes:
+        continue
+    else:
+        results_gene = marked_df[marked_df["gene"] == gene]
+        gene_plot = cnv_plot(results_gene, gene)
+        impacted_probes = summary_data_gene.dropna(subset=["Variant"])
+        impacted_probes = summary_data_gene.loc[summary_data_gene["Variant"] != ""]
+        impacted_probes = (impacted_probes.sort_values(by="Start Position"))
+        print(impacted_probes)
+
+        impacted_probes_html = df_to_html(impacted_probes)
+
+        # saving to cnv_detail dict.
+        cnv_detail[gene] = [gene_plot, impacted_probes_html]
+
+gene_plots = {}
+
+for gene in gene_list:
+    results_gene = results_clinical_df[results_clinical_df["gene"] == gene]
+    gene_plot = cnv_plot(results_gene, gene)
+    gene_plots[gene] = gene_plot
+
+# env = Environment(loader=PackageLoader("dMLPA-flask", "templates"))
+# template = env.get_template("report.html")
+# data_values = {
+#     "file_name": "report.pdf",
+#     # "r_code": r_code,
+#     "pan_number": "pan_number",
+#     "genes": gene_list,
+#     "patient_file": xls,
+#     "probe_file": probe,
+#     "qc_status": "qc_status",
+#     "cnv_summary": "summary_html",
+#     "report_notes": "report_notes",
+#     "qc_table": "passed",
+#     "summary_tables": "summary_tables",
+#     "gene_plots": gene_plots
+# }
+
+# html_report = template.render(data_values)
+
+# pdfkit.from_string(html_report)
+
+""" NOTES ON PLOT ANNOTATION """
+
+def get_text_positions(text, x_data, y_data, txt_width, txt_height):
+    a = zip(y_data, x_data)
+    text_positions = list(y_data)
+    for index, (y, x) in enumerate(a):
+        local_text_positions = [i for i in a if i[0] > (y - txt_height)
+                            and (abs(i[1] - x) < txt_width * 2) and i != (y,x)]
+        if local_text_positions:
+            sorted_ltp = sorted(local_text_positions)
+            if abs(sorted_ltp[0][0] - y) < txt_height: #True == collision
+                differ = np.diff(sorted_ltp, axis=0)
+                a[index] = (sorted_ltp[-1][0] + txt_height, a[index][1])
+                text_positions[index] = sorted_ltp[-1][0] + txt_height*1.01
+                for k, (j, m) in enumerate(differ):
+                    #j is the vertical distance between words
+                    if j > txt_height * 2: #if True then room to fit a word in
+                        a[index] = (sorted_ltp[k][0] + txt_height, a[index][1])
+                        text_positions[index] = sorted_ltp[k][0] + txt_height
+                        break
+    return text_positions
+
+def text_plotter(text, x_data, y_data, text_positions, txt_width,txt_height):
+    for z,x,y,t in zip(text, x_data, y_data, text_positions):
+        plt.annotate(str(z), xy=(x-txt_width/2, t), size=12)
+        if y != t:
+            plt.arrow(x, t,0,y-t, color='red',alpha=0.3, width=txt_width*0.1,
+                head_width=txt_width, head_length=txt_height*0.5,
+                zorder=0,length_includes_head=True)
+
+# Pulled from function
+    # # Annotate probes within het_del_dq_range or het_dup_dq_range with probe numbers
+    # for index, row in cnv_copy.iterrows():
+    #     dq_value = row["dosage quotient"]
+    #     position = row["start position"]
+    #     probe_number = row["probe number"]
+
+    #     if dq_value < normal_dq_range[-1] or dq_value > normal_dq_range[1]:
+    #         plt.annotate(probe_number, xy=(position, dq_value),
+    #                      xytext=(position + 99, dq_value + 0.5),  # Adjust xytext position
+    #                      rotation=89,
+    #                     #  textcoords="offset points",  # Offset in points
+    #                     #  ha='center', va='center',  # Horizontal and vertical alignment
+    #                     #  bbox=dict(boxstyle='round,pad=-1.3', fc='yellow', alpha=0.5),  # Bounding box style
+    #                      arrowprops=dict(arrowstyle='->', connectionstyle='arc,rad=-1', color='black'))  # Arrow properties
\ No newline at end of file
diff --git a/dMLPA-flask/plotting.py b/dMLPA-flask/plotting.py
deleted file mode 100644
index 1869a6c..0000000
--- a/dMLPA-flask/plotting.py
+++ /dev/null
@@ -1,142 +0,0 @@
-import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-import matplotlib.ticker as ticker
-from cna import shared_columns, split_df_column, merge_df, clinical_df, gene_list, copy_number_analysis
-
-
-def cnv_plot(df, gene, extend=1000):
-    # create copy of dataframe
-    cnv_copy = df.copy()
-
-    # pull specific data from columns
-    # gene = cnv_copy["gene"].unique
-    # chrom = cnv_copy["chr"].unique
-    start = min(cnv_copy["start position"]) + extend
-    end = max(cnv_copy["end position"]) + extend
-
-    # determining y-axis limit for DQ plot
-    ylim_ax1 = (0, max(2.0, 1.25*cnv_copy["dosage quotient"].max()))
-
-    #setting up plot layout
-    fig = plt.figure(figsize=(8,6))
-    gs = fig.add_gridspec(nrows=2, ncols=1, height_ratios=[3,1])
-
-    ax1 = plt.subplot(gs[0])
-    ax1.set_xlim(start,end)
-    ax1.set_ylim(*ylim_ax1)
-    plt.ylabel("Dosage Quotient")
-    plt.setp(ax1.get_xticklabels(), visible=False)
-    plt.title(gene)
-
-    ax2 = plt.subplot(gs[1], sharex = ax1)
-    ax2.set_xlim(start,end)
-    ax2.set_ylim(0,1)
-    plt.setp(ax2.get_yticklabels(), visible=False)
-
-    #n.b. this ensures that we are getting the full integer x-axis rather than scientific notation.
-    # but this is from chatGPT and i need to question if this is the best way to do this. 
-    ax2.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: '{:d}'.format(int(x))))
-
-    # adding shading to define ranges for normal(green)
-    # hetdel (red), hetdup (blue)
-    normal_dq_range = (0.8, 1.2)
-    ax1.axhspan(normal_dq_range[0], normal_dq_range[1], facecolor = "#00FF0044", edgecolor = None)
-    het_del_dq_range = (0.4,0.6)
-    ax1.axhspan(het_del_dq_range[0], het_del_dq_range[1], facecolor = "#FF000044", edgecolor = None)
-    het_dup_dq_range = (1.4,1.6)
-    ax1.axhspan(het_dup_dq_range[0], het_dup_dq_range[1], facecolor = "#0000FF44", edgecolor = None)
-
-    # plot DQ value
-    ax1.plot(cnv_copy["start position"], cnv_copy["dosage quotient"], marker = "x", linestyle = "", color = "black")
-
-    plt.show()
-
-    return fig
-
-# Test files to test script
-xls = "/home/epayne1/Documents/GSTT/test_input/BP01-11-dMLPA_R210v1_Pan5025_185897_SURNAME_Firstname11_filtered_results.xlsx"
-probe = "/home/epayne1/Documents/GSTT/probe_file/PIF_D001-C1_Hereditary_Cancer_Panel_1-v02_modB1probe.xlsx"
-
-# Reading data in dataframes
-metadata_df = pd.read_excel(xls, sheet_name="Sample info")
-raw_results_df = pd.read_excel(xls, sheet_name="Gene filtered results")
-probe_df = pd.read_excel(probe, sheet_name="probe_info")
-
-# Defining new column names
-probe_column_headers = [
-    "chr",
-    "start position",
-    "end position"
-]
-
-# hard manipulation/formatting of df's
-probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
-probe_df = split_df_column(probe_df, "Mapview (hg38)a", probe_column_headers)
-probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
-raw_results_df.rename(columns={raw_results_df.columns[-1]:"dosage quotient"}, inplace=True)
-
-# Definining unique columns needed from df's
-columns_results_df = [
-    "dosage quotient"
-]
-
-columns_probe_df = [
-    "probe number",
-    "chr",
-    "start position",
-    "end position",
-]
-
-# creating results_clinical dataframe
-results_clinical_df = clinical_df(raw_results_df, probe_df, columns_results_df, columns_probe_df)
-results_clinical_df["start position"] = results_clinical_df["start position"].astype(int)
-results_clinical_df["end position"] = results_clinical_df["end position"].astype(int)
-print(results_clinical_df)
-results_clinical_df.to_excel("merged_excel.xlsx")
-
-# get list of genes
-gene_list = ["MSH6"]
-
-## perform copy number analysis
-# cnv_summary_df = copy_number_analysis(results_clinical_df, gene_list)
-# print(cnv_summary_df)
-
-# perform copy number analysis
-cnv_summary_df = copy_number_analysis(results_clinical_df, gene_list)
-
-for gene in gene_list:
-    results_gene = results_clinical_df[results_clinical_df["gene"] == gene]
-    print(results_gene)
-
-    cnv_plot(results_gene, gene)
-
-    # plt.switch_backend('TkAgg')
-
-    # # defining x and y axis ranges
-    # xmin = results_gene["start position"].min()
-    # xmax = results_gene["end position"].max()
-    # ymin = 0
-
-    # if results_gene["mean dosage quotient"].max() > 2.5:
-    #     ymax = 1.25 * results_gene["mean dosage quotient"].max()
-    # else:
-    #     ymax = 2.5
-
-    # #plotting per gene
-    # plt.figure(figsize=(12,6))
-
-    # ax1 = plt.subplot(211)
-    # # plt.axis([xmin, xmax, ymin, ymax])
-    # plt.plot(results_clinical_df["start position"], results_clinical_df["mean dosage quotient"], "rx-")
-    # plt.xlabel("Probe start co-ordinate")
-    # plt.ylabel("Mean Dosage Quotient")
-    # plt.setp(ax1.get_xticklabels(), visible=False)
-    # plt.title(gene)
-
-    # ax2 = plt.subplot(212, sharex=ax1)
-    # plt.plot(results_clinical_df["start position"], results_clinical_df["exon"])
-    # plt.setp(ax2.get_xticklabels(), fontsize=6)
-
-    # plt.show()
-
diff --git a/dMLPA-flask/qc_data.py b/dMLPA-flask/qc_data.py
index adefb7c..cbd8cd0 100644
--- a/dMLPA-flask/qc_data.py
+++ b/dMLPA-flask/qc_data.py
@@ -15,9 +15,9 @@
 
 import re
 
-def get_pan_numer(df):
+def get_test_codes(df):
     """
-    Return virtual pan number used for analysis
+    Return virtual pan number and Rcode used for analysis
 
     Arguments:
     - df(Pandas Dataframe): assumed to be a dataframe containing the
@@ -26,6 +26,8 @@ def get_pan_numer(df):
     Return Value:
     - pan_number(Str): GSTT virtual gene pan(el) number used to filter
     raw NGS output
+    - r_code (Str): Appropriate Rcode from the national genomic test
+    directory
     """
 
     # TODO:
@@ -33,18 +35,25 @@ def get_pan_numer(df):
     # - Logging.info[panel number] and logging.error[pan number not found]
     # - Print error onto report.
 
+    # creating empty variables
+    pan_number = ""
+    r_code = ""
+
     # Get samplename from df
     samplename = df.columns[1]
 
     # Searching for Pan number by regular expression
     panel_number_match = re.search(r'Pan[0-9]+', samplename)
+    rcode_match = re.search(r'R[0-9]+', samplename)
 
     if panel_number_match:
-        # Extracting the matched PAN number
         pan_number = panel_number_match.group(0)
-        return pan_number
-    else:
-        return None
+
+    if rcode_match:
+        r_code = rcode_match.group(0)
+
+    return pan_number, r_code
+
 
 def get_qc_status(df):
     """
@@ -74,67 +83,38 @@ def summary_data(df):
     """Create summary table of key probe data"""
 
     # Merging start and end co-ordinates into one column
-    df["co-ordinates"] = df["start position"] + "-" + df["end position"]
+    # df["co-ordinates"] = df["start position"].astype(str) + "-" + df["end position"].astype(str)
 
     # defining columns desired in table from existing clinical_data.
     columns_to_keep = [
         "probe number",
+        "chr",
         "gene",
         "exon",
-        "chr",
-        "co-ordinates",
+        "start position",
+        "end position",
         "normal copy number",
-        "dosage quotient"
+        "dosage quotient",
+        "variant"
     ]
 
-    # creating copy of selected columns.
+    # creating copy of selected columns with Copy number estimate column
     summary_data_df = df[columns_to_keep].copy()
+    summary_data_df["copy number est."] = "-"
+    summary_data_df["copy number est."] = summary_data_df["copy number est."].astype(object)
 
-    # calculating copy number estimate
-    summary_data_df["copy number est."] = (
+    # defining regular expression
+    variant_type = re.compile("Ref to Scientist.*", re.IGNORECASE)
+
+    # creating condition for calculating copy number estimate
+    # TODO two methods below (both ChatGPT) decide which to keep
+    condition = ~summary_data_df['variant'].str.match(variant_type)
+    # condition = ~summary_data_df["variant"].apply(lambda x: bool(variant_type.match(x)))
+
+    # calculating copy number estimate only where condition isn't met
+    summary_data_df.loc[condition, "copy number est."] = (
         summary_data_df["normal copy number"] *
         summary_data_df["dosage quotient"]
     ).round()
 
     return summary_data_df
-
-
-"""UNUSED FUNCTIONS"""
-
-def gene_summary_data(gene, df):
-    """Create summary of probe data per gene"""
-
-    summary_data_gene =  df[df["gene"] == gene]
-
-    summary_data_gene.columns = summary_data_gene.columns.str.replace("_", " ").str.title()
-
-    gene_summary_html = df_to_html(summary_data_gene)
-
-    return gene_summary_html
-
-# # function to determine estimate but I don't think this will be needed.
-# def copy_number_estimate(row):
-#     copy_number_est = round(row["Normal Copy Number"] * row["Mean Dosage Quotient"])
-#     return copy_number_est
-
-# def parse_run_data(file, file_basename):
-    # """Old function not in use"""
-
-#     # Parse Sample Info sheet from excel into dataframe
-#     sample_info_df = pd.read_excel(file, sheet_name="Sample info")
-
-#     #Saving dataframe to html format
-#     sample_info_html = sample_info_df.to_html(classes="table", index=False)
-
-#     # Parsing test information from spreadsheet
-#     column_header = file_basename.replace("_filtered_results.xlsx", "")
-#     qc_status = sample_info_df.loc[6, column_header].upper()
-
-#     column_header = sample_info_df.columns[1]
-#     panel_number = re.search(r"Pan[0-9]+", column_header).group(0)
-#     qc_status = sample_info_df.loc[6, sample_info_df.columns[1]]
-
-#     print(panel_number)
-#     print(qc_status)
-
-#     return sample_info_html, qc_status
diff --git a/dMLPA-flask/templates/report.html b/dMLPA-flask/templates/report.html
index f019137..ee87078 100644
--- a/dMLPA-flask/templates/report.html
+++ b/dMLPA-flask/templates/report.html
@@ -77,6 +77,7 @@ <h1>dMLPA Copy Number Analysis</h1>
 <div class="container">
     <div class="element">
         <h4>Test Request</h4>
+            <p><b>R Code:</b>{{ r_code }}</p>
             <p><b>Pan Number: </b><i>{{ pan_number }}</i></p>
             <p><b>Genes: </b><i>{{ genes }}</i></p>
             <br>
@@ -96,16 +97,25 @@ <h4>Test Request</h4>
 
 <div class="body">
     <h2>Called CNV's</h2>
-        <p> {{ cnv_summary }}</p>
+        <p>{{ cnv_summary }}</p>
         {% for key, value in report_notes.items() %}
             {% if key in cnv_summary %}
-                <p> {{ value}} </p>
+                <p>{{ value}}</p>
             {% endif %}
         {% endfor %}
 </div>
 
+<br>
+
 <div class="body">
     <h3>CNV details</h3>
+    {% for gene, data in cnv_detail.items() %}
+        <p>{{ gene }}_plot</p>
+        <img src="data:image/svg+xml;base64,{{ data[0] }}" alt="{{ gene }} Plot" style="display: block; margin-left: auto; margin-right: auto">
+        <br>
+        <p>{{ data[1] }}</p>
+        <br>
+    {% endfor %}
 </div>
 
 <div class="page-break"></div>
diff --git a/requirements.txt b/requirements.txt
index f4318f5..ad0c8d4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,16 +2,17 @@ Flask==2.2.2
 flask-cors==3.0.10
 Flask-Session==0.4.0
 Flask-WTF==1.1.1
+gunicorn==20.1.0
 Jinja2==3.1.2
 kaleido==0.2.1
+matplotlib==3.8.3
+myst-parser==0.15.2
 numpy==1.22.4
 openpyxl==3.0.10
-gunicorn==20.1.0
 pandas==1.4.2
 pdfkit==1.0.0
 plotly==5.7.0
 pytest==7.2.0
 sphinx==4.5.0
 sphinx_rtd_theme==1.1.1
-myst-parser==0.15.2
 Werkzeug==2.2.2

From 8d030e0f547c35779b2f91e6beb36eedcff774ff Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Wed, 15 May 2024 19:46:58 +0100
Subject: [PATCH 09/15] Minor Refactor of script + Update to report template

1) Moving hardcoded elements to config.py
2) Removes generation of probe_df outside of the for loop
3) Updates report with version of app, genome build, warning message if not in production env
4) Adds reporting comment when no copy number variants are detected
---
 dMLPA-flask/backend.py            | 152 +++++++++--------------
 dMLPA-flask/cna.py                | 192 ++++++++++++------------------
 dMLPA-flask/config.py             | 101 ++++++++++++++++
 dMLPA-flask/notes.py              |  69 +++++------
 dMLPA-flask/qc_data.py            |  19 +--
 dMLPA-flask/templates/report.html |  44 ++++---
 6 files changed, 297 insertions(+), 280 deletions(-)

diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index e7f9573..fde6bc0 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -4,6 +4,7 @@
 from jinja2 import Environment, PackageLoader
 import pdfkit
 import pandas as pd
+import config
 from qc_data import get_test_codes, df_to_html, get_qc_status, summary_data
 from cna import split_df_column, clinical_df, copy_number_analysis, cnv_plot
 
@@ -27,120 +28,70 @@ def run_backend(single_file, array_of_files, output_directory):
     single_file = os.path.abspath(single_file)
     array_of_files = [os.path.abspath(file) for file in array_of_files]
 
+    # Reading probe data into dataframe and formatting
+    probe_df = pd.read_excel(single_file, sheet_name="probe_info")
+    probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
+    probe_df = split_df_column(probe_df, "Mapview (hg38)a", config.NEW_PROBE_DF_HEADERS)
+    probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
+
     # Iterating over files to generate reports
     for file in array_of_files:
 
-        # Getting basenames
-        file_name = os.path.basename(file)
-        probe_file = os.path.basename(single_file)
-
-        # Creating ouput file name and path
-        # TODO might need changing depending on file_naming convention.
-        report = file_name.replace("_filtered_results.xlsx", ".pdf").replace("_filtered_results.xls", ".pdf")
-        report_path = os.path.join(output_directory, report)
-
-        # Reading data in dataframes
+        # Reading data from results file in dataframes and formatting
         metadata_df = pd.read_excel(file, sheet_name="Sample info")
         raw_results_df = pd.read_excel(file, sheet_name="Gene filtered results")
-        probe_df = pd.read_excel(single_file, sheet_name="probe_info")
-
-        # Defining new column names
-        probe_column_headers = [
-            "chr",
-            "start position",
-            "end position"
-        ]
-
-        # hard manipulation/formatting of df's
-        probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
-        probe_df = split_df_column(probe_df, "Mapview (hg38)a", probe_column_headers)
-        probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
         raw_results_df.rename(columns={raw_results_df.columns[-1]:"dosage quotient"}, inplace=True)
 
-        print(raw_results_df)
-
-        # Definining unique columns needed from df's
-        columns_results_df = [
-            "dosage quotient"
-        ]
-
-        columns_probe_df = [
-            "probe number",
-            "chr",
-            "start position",
-            "end position",
-        ]
+        # controlling dataypes
+        probe_df["start position"] = probe_df["start position"].astype(int)
+        probe_df["end position"] = probe_df["end position"].astype(int)
 
         # creating results_clinical dataframe
-        results_clinical_df = clinical_df(raw_results_df, probe_df, columns_results_df, columns_probe_df)
-        # results_clinical_df.to_excel("merged_excel.xlsx")
-
-        # controlling dataypes in clinical dataframe
-        results_clinical_df["start position"] = results_clinical_df["start position"].astype(int)
-        results_clinical_df["end position"] = results_clinical_df["end position"].astype(int)
-
-        # get list of genes
+        results_clinical_df = clinical_df(
+            raw_results_df,
+            probe_df,
+            config.COLUMNS_RESULTS_DF,
+            config.COLUMNS_PROBE_DF
+        )
+
+        # get list of genes in panel
         gene_list = results_clinical_df["gene"].unique()
 
         # perform copy number analysis
         cnv_summary_df, marked_df = copy_number_analysis(results_clinical_df, gene_list)
         summary_html = df_to_html(cnv_summary_df)
 
-        # defining review messages
-        report_notes = {
-            "Ref to Scientist 1": (
-                "Ref to Scientist 1: The dosage quotient for one or "\
-                "more probes falls below the range expected for a copy "\
-                "number loss. Scientist review required."
-            ),
-            "Ref to Scientist 2": (
-                "Ref to Scientist 2: The dosage quotient for one or "\
-                "more probes falls in between the expected ranges for "\
-                "copy number loss and copy number neutral. Scientist review "\
-                "required"
-            ),
-            "Ref to Scientist 3": (
-                "Ref to Scientist 3: The dosage quotient for one or "\
-                "more probes falls in between the expected ranges for copy "\
-                "number neutral and copy number gain. Scientist review "\
-                "required."
-            ),
-            "Ref to Scientist 4": (
-                "Ref to Scientist 4: The dosage quotient for one or "\
-                "more probes falls above the range expected for a copy "\
-                "number gain. Scientist review required."
-            )
-        }
-
-        # obtaining summary data per gene for appendix
+        # generating summary dataframe with limited columns
         summary_data_df = summary_data(marked_df)
-        print(summary_data_df)
 
+        # creating empty dictionaries
         summary_tables = {}
         cnv_detail = {}
-        for gene in gene_list:
-            # summary data by gene
-            summary_data_gene =  summary_data_df[summary_data_df["gene"] == gene]
-            summary_data_gene.columns = summary_data_gene.columns.str.replace("_", " ").str.title()
-            summary_tables[gene] = df_to_html(summary_data_gene)
-
-            # cnv_detail by impacted gene
-            impacted_genes = cnv_summary_df["Gene"].unique()
-            if gene not in impacted_genes:
-                continue
-            else:
-                results_gene = marked_df[marked_df["gene"] == gene]
-                gene_plot = cnv_plot(results_gene, gene)
-                impacted_probes = summary_data_gene.dropna(subset=["Variant"])
-                impacted_probes = summary_data_gene.loc[summary_data_gene["Variant"] != ""]
-                impacted_probes = (impacted_probes.sort_values(by="Start Position"))
-                impacted_probes_html = df_to_html(impacted_probes)
 
-                # saving to cnv_detail dict.
-                cnv_detail[gene] = [gene_plot, impacted_probes_html]
+        for gene in gene_list:
+            # creating cnv summary data by gene
+            summary_data_gene_df =  summary_data_df[summary_data_df["gene"] == gene]
+            summary_data_gene_df.columns = summary_data_gene_df.columns.str.replace("_", " ").str.title()
+            summary_tables[gene] = df_to_html(summary_data_gene_df)
+
+            if not cnv_summary_df.empty:
+                # generating details of cnv by impacted gene if cnv's are detected
+                impacted_genes = cnv_summary_df["Gene"].unique()
+                if gene in impacted_genes:
+                    # plotting gene data
+                    results_gene_df = marked_df[marked_df["gene"] == gene]
+                    gene_plot = cnv_plot(results_gene_df, gene)
+
+                    # creating table of probes involved in CNV
+                    impacted_probes_df = summary_data_gene_df.dropna(subset=["Variant"])
+                    impacted_probes_df = summary_data_gene_df.loc[summary_data_gene_df["Variant"] != ""]
+                    impacted_probes_df = (impacted_probes_df.sort_values(by="Start Position"))
+                    impacted_probes_html = df_to_html(impacted_probes_df)
+
+                    # saving to cnv_detail dict.
+                    cnv_detail[gene] = [gene_plot, impacted_probes_html]
 
         # getting panel number and qc data from metadata
-        # TODO consider if i shouldn't have this futher down rather than at beginning.
         pan_number, r_code = get_test_codes(metadata_df)
         qc_status = get_qc_status(metadata_df)
         qc_data = df_to_html(metadata_df)
@@ -148,21 +99,36 @@ def run_backend(single_file, array_of_files, output_directory):
         # Converting gene_list to string
         genes = ", ".join(gene_list)
 
+        # Getting input file basenames
+        file_name = os.path.basename(file)
+        probe_file = os.path.basename(single_file)
+
+        # Creating ouput file name and path
+        # Input file name of results file is determined in GitHub repository:
+        # https://github.com/moka-guys/digital_mlpa
+        report = file_name.replace("_filtered_results.xlsx", ".pdf") \
+                        .replace("_filtered_results.xls", ".pdf")
+        report_path = os.path.join(output_directory, report)
+
         # Rendering template and saving as pdf
 
         env = Environment(loader=PackageLoader("dMLPA-flask", "templates"))
         template = env.get_template("report.html")
         data_values = {
+            "app_version": config.DMLPA_FLASK_VERSION,
+            "production": config.RELEASED_TO_PRODUCTION,
             "file_name": report,
             "r_code": r_code,
             "pan_number": pan_number,
             "genes": genes,
+            "run_id": "", # TODO to be added once file name is confirmed, 
+            "genome_build": config.GENOME_BUILD,
             "patient_file": file_name,
             "probe_file": probe_file,
             "qc_status": qc_status,
             "cnv_summary": summary_html,
             "cnv_detail": cnv_detail,
-            "report_notes": report_notes,
+            "report_notes": config.REPORTING_NOTES,
             "qc_table": qc_data,
             "summary_tables": summary_tables
         }
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
index f29c42d..fc75609 100644
--- a/dMLPA-flask/cna.py
+++ b/dMLPA-flask/cna.py
@@ -10,14 +10,14 @@
 Functions:
 - TODO UPDATE
 """
-
-import pandas as pd
-import numpy as np
 import io
 import base64
+import pandas as pd
+import numpy as np
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 import matplotlib.ticker as ticker
+import config
 
 def shared_columns(df1, df2):
     """Identify shared dataframe columns"""
@@ -71,7 +71,6 @@ def merge_df(df1, df2, df1_columns, df2_columns):
 
     # identifying shared columns
     shared_columns = list(set(df1) & set(df2))
-    # print(shared_columns)
 
     # left merge
     merged_df = df1[shared_columns + df1_columns].merge(
@@ -106,25 +105,8 @@ def clinical_df(df1, df2, df1_columns, df2_columns):
 
     merged_df =  merge_df(df1, df2, df1_columns, df2_columns)
 
-    # specifying column order
-    # TODO Change as required
-    desired_column_order = [
-        "probe order",
-        "probe number",
-        "reference probe",
-        "chr",
-        "gene",
-        "exon",
-        "start position",
-        "end position",
-        "normal copy number",
-        "dosage quotient",
-        "probe type",
-        "additional information",
-    ]
-
     # restructuring df based on column order
-    merged_df = merged_df[desired_column_order]
+    merged_df = merged_df[config.DESIRED_COLUMN_ORDER]
 
     # Filtering results by probetype
     results_clinical_df = merged_df[merged_df["probe type"] != "CTRL"]
@@ -138,25 +120,11 @@ def copy_number_analysis(df, gene_list):
     Might need to be broken down more in future
 
     """
-
-    # defining copy number categories
-    # TODO How does code handle MDQ that is 3 dp in the excel
-    cncategories = {
-        "Ref to Scientist 1*": [0, 0.39],
-        "HetDel": [0.4, 0.6],  # theoretical value 0.5, represents CN 1
-        "Ref to Scientist 2*": [0.61, 0.79],
-        # "normal": [0.8, 1.2],  # theoretical value 1, represents CN 2
-        "Ref to Scientist 3*": [1.21, 1.39],
-        "HetDup": [1.4, 1.6],  # theoretical value 1.5, represents CN 3
-        "Ref to Scientist 4*": [1.61, float("Inf")]
-    }
-
     # Empty dataframe for all identified copy number variations
     cnv_summary = pd.DataFrame()
 
-    
     # Create 'variant' column with object dtype
-    df['variant'] = ""  # Initialize with NaN values, which are compatible with object dtype
+    df['variant'] = ""  # Initialized with NaN values compatible with object dtype
     df["variant"] = df["variant"].astype(object) #controls expected type and prevents FutureError observed in manual testing
 
     # TODO This is mostly from chatGPT - Need to understand what it's actually doing
@@ -166,78 +134,73 @@ def copy_number_analysis(df, gene_list):
         results_gene =  df[df["gene"] == gene]
 
         # iterate through CN categories and identify matching probe entries
-        for direction, parameters in cncategories.items():
-            # print(direction, gene) # TODO Replace with logging
-            cnv_rows = np.where((parameters[0] <= results_gene["dosage quotient"]) &
-                    (results_gene["dosage quotient"] <= parameters[1]))[0]
-
-            # If no probe entries match CN category skipping next steps
-            if len(cnv_rows) == 0:
-                continue
-
-            # If probe entries match CN category, identify uniform CN segments
-            else:
-                # gather information for appropriate probe entries
-                cnv_data = results_gene.iloc[cnv_rows]
-                # print(cnv_data)
-                # Assess based on probe order if probes are contiguous
-                cnv_groups = np.cumsum(np.concatenate(([1], np.diff(cnv_data["probe order"]) != 1)))
-                # print(cnv_groups)
-
-                # Splitting results per contiguous CN segment if multiple
-                cnv_list = {cnv_index: group for cnv_index, group in cnv_data.groupby(cnv_groups)}
-
-                print(cnv_list)
-
-                # Processing data of individual CN segments
-                for cnv_index, cnv_lines in cnv_list.items():
-                    cnv_lines = pd.DataFrame(cnv_lines)
-                    # print(cnv_lines)
-
-                    # Update the input DataFrame by adding a new column containing the CN category
-                    probe_numbers = cnv_lines["probe number"]
-                    for probe_number in probe_numbers:
-                        df.loc[df["probe number"] == probe_number, 'variant'] = direction
-
-
-                    # Obtainins summary data of contiguous CN segments
-                    start_exon = cnv_lines["exon"].min()
-                    end_exon = cnv_lines["exon"].max()
-                    start_pos = cnv_lines["start position"].min()
-                    end_pos = cnv_lines["end position"].max()
-
-                    # Calculate average dosage quotient across CN segment and standard deviation
-                    copy_change_mean = round(cnv_lines["dosage quotient"].mean(), 3)
-                    copy_change_sd = round(cnv_lines["dosage quotient"].std(), 3)
-
-                    # Calculating Estimated Copy number for CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
-                    # TODO Does datatype need to be considered before calculation?
-                    if "Ref to Scientist" in direction:
-                        copy_number_est = "-"
-                    else:
-                        copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["dosage quotient"]))
-
-
-                    # summarize CN data into a df entry
-                    cnv_aggregate = pd.DataFrame({
-                        "Gene": [gene],
-                        "Variant": [direction],
-                        "Exon(s)": f"{start_exon} - {end_exon}" if start_exon != end_exon else str(start_exon),
-                        "Chr": cnv_lines["chr"].unique(),
-                        "Probe Co-ordinates": f"{start_pos}-{end_pos}",
-                        "Normal Copy Number": int(cnv_lines["normal copy number"].unique()), #Problem if there is more than one potential copy number?
-                        "Mean Dosage Quotient": f"{copy_change_mean}+/-{copy_change_sd}",
-                        "Est. Copy Number": [copy_number_est],
-                        "Supporting Probes": [len(cnv_lines)]
-                    })
-
-                    # Adding df entry into existing CNA_summary dataframe
-                    # TODO concat method is not ideal apparently. Consider alternative
-                    cnv_summary = pd.concat([cnv_summary,cnv_aggregate], ignore_index=True)
-
-                    # organize summary by genomic co-ordinates
-                    cnv_summary = cnv_summary.sort_values(by="Probe Co-ordinates")
-                    # print(cnv_summary)
+        for direction, parameters in config.CNC_CATEGORIES.items():
+            if direction != "Normal":
+                # print(direction, gene) # TODO Replace with logging
+                cnv_rows = np.where((parameters[0] <= results_gene["dosage quotient"]) &
+                        (results_gene["dosage quotient"] <= parameters[1]))[0]
+
+                # If no probe entries match CN category skipping next steps
+                if len(cnv_rows) == 0:
+                    continue
+
+                # If probe entries match CN category, identify uniform CN segments
+                else:
+                    # gather information for appropriate probe entries
+                    cnv_data = results_gene.iloc[cnv_rows]
+                    # Assess based on probe order if probes are contiguous
+                    cnv_groups = np.cumsum(np.concatenate(([1], np.diff(cnv_data["probe order"]) != 1)))
+
+                    # Splitting results per contiguous CN segment if multiple
+                    cnv_list = {cnv_index: group for cnv_index, group in cnv_data.groupby(cnv_groups)}
+
+                    # Processing data of individual CN segments
+                    for cnv_index, cnv_lines in cnv_list.items():
+                        cnv_lines = pd.DataFrame(cnv_lines)
+
+                        # Update the input DataFrame by adding a new column containing the CN category
+                        probe_numbers = cnv_lines["probe number"]
+                        for probe_number in probe_numbers:
+                            df.loc[df["probe number"] == probe_number, 'variant'] = direction
+
+
+                        # Obtainins summary data of contiguous CN segments
+                        start_exon = cnv_lines["exon"].min()
+                        end_exon = cnv_lines["exon"].max()
+                        start_pos = cnv_lines["start position"].min()
+                        end_pos = cnv_lines["end position"].max()
+
+                        # Calculate average dosage quotient across CN segment and standard deviation
+                        copy_change_mean = round(cnv_lines["dosage quotient"].mean(), 3)
+                        copy_change_sd = round(cnv_lines["dosage quotient"].std(), 3)
+
+                        # Calculating Estimated Copy number for CN segment. Autosomal Logic 2*1=2; 2*0.5=1; 2*1.5=3
+                        # TODO Does datatype need to be considered before calculation?
+                        if "Ref to Scientist" in direction:
+                            copy_number_est = "-"
+                        else:
+                            copy_number_est = round(np.mean(cnv_lines["normal copy number"] * cnv_lines["dosage quotient"]))
+
+
+                        # summarize CN data into a df entry
+                        cnv_aggregate = pd.DataFrame({
+                            "Gene": [gene],
+                            "Variant": [direction],
+                            "Exon(s)": f"{start_exon} - {end_exon}" if start_exon != end_exon else str(start_exon),
+                            "Chr": cnv_lines["chr"].unique(),
+                            "Probe Co-ordinates": f"{start_pos}-{end_pos}",
+                            "Normal Copy Number": int(cnv_lines["normal copy number"].unique()), #Problem if there is more than one potential copy number?
+                            "Mean Dosage Quotient": f"{copy_change_mean}+/-{copy_change_sd}",
+                            "Est. Copy Number": [copy_number_est],
+                            "Supporting Probes": [len(cnv_lines)]
+                        })
+
+                        # Adding df entry into existing CNA_summary dataframe
+                        # TODO concat method is not ideal apparently. Consider alternative
+                        cnv_summary = pd.concat([cnv_summary,cnv_aggregate], ignore_index=True)
+
+                        # organize summary by genomic co-ordinates
+                        cnv_summary = cnv_summary.sort_values(by="Probe Co-ordinates")
 
     # returning output data to backend.py
     return cnv_summary, df
@@ -271,13 +234,10 @@ def cnv_plot(df, gene, extend=1000):
     plt.gca().xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: '{:d}'.format(int(x))))
 
     # adding shading to define ranges for normal, hetdel and hetdup
-    # TODO Could be done from config.
-    normal_dq_range = (0.8, 1.2)
-    plt.axhspan(normal_dq_range[0], normal_dq_range[1], facecolor = "#00FF0044", edgecolor = None)
-    het_del_dq_range = (0.4,0.6)
-    plt.axhspan(het_del_dq_range[0], het_del_dq_range[1], facecolor = "#FF000044", edgecolor = None)
-    het_dup_dq_range = (1.4,1.6)
-    plt.axhspan(het_dup_dq_range[0], het_dup_dq_range[1], facecolor = "#318CE744", edgecolor = None)
+    cnc_categories = config.CNC_CATEGORIES
+    plt.axhspan(cnc_categories["Normal"][0], cnc_categories["Normal"][1], facecolor = "#00FF0044", edgecolor = None)
+    plt.axhspan(cnc_categories["HetDel"][0], cnc_categories["HetDel"][1], facecolor = "#FF000044", edgecolor = None)
+    plt.axhspan(cnc_categories["HetDup"][0], cnc_categories["HetDup"][1], facecolor = "#318CE744", edgecolor = None)
 
     # plot DQ value
     plt.plot(cnv_copy["start position"], cnv_copy["dosage quotient"], marker = "x", linestyle = "", color = "black")
diff --git a/dMLPA-flask/config.py b/dMLPA-flask/config.py
index e69de29..c02dad0 100644
--- a/dMLPA-flask/config.py
+++ b/dMLPA-flask/config.py
@@ -0,0 +1,101 @@
+
+# config.py
+
+# TODO Add config docstring
+
+''' ==================== Production release constants ==================== '''
+
+DMLPA_FLASK_VERSION = "v1.0.0"
+
+RELEASED_TO_PRODUCTION = False
+
+''' ==================== backend.py constants ==================== '''
+
+GENOME_BUILD = "GRCh38"
+
+NEW_PROBE_DF_HEADERS = [
+    "chr",
+    "start position",
+    "end position"
+]
+
+COLUMNS_RESULTS_DF = [
+    "dosage quotient"
+]
+
+COLUMNS_PROBE_DF = [
+    "probe number",
+    "chr",
+    "start position",
+    "end position",
+]
+REPORTING_NOTES = {
+    "Ref to Scientist 1": (
+        "Ref to Scientist 1: The dosage quotient for one or "\
+        "more probes falls below the range expected for a copy "\
+        "number loss. Scientist review required."
+    ),
+    "Ref to Scientist 2": (
+        "Ref to Scientist 2: The dosage quotient for one or "\
+        "more probes falls in between the expected ranges for "\
+        "copy number loss and copy number neutral. Scientist review "\
+        "required"
+    ),
+    "Ref to Scientist 3": (
+        "Ref to Scientist 3: The dosage quotient for one or "\
+        "more probes falls in between the expected ranges for copy "\
+        "number neutral and copy number gain. Scientist review "\
+        "required."
+    ),
+    "Ref to Scientist 4": (
+        "Ref to Scientist 4: The dosage quotient for one or "\
+        "more probes falls above the range expected for a copy "\
+        "number gain. Scientist review required."
+    )
+}
+
+''' ==================== cna.py constants ==================== '''
+
+DESIRED_COLUMN_ORDER = [
+    "probe order",
+    "probe number",
+    "reference probe",
+    "chr",
+    "gene",
+    "exon",
+    "start position",
+    "end position",
+    "normal copy number",
+    "dosage quotient",
+    "probe type",
+    "additional information",
+]
+
+GENE_UNIQUE_CUTOFF = [
+    "PMS2"
+]
+
+CNC_CATEGORIES = {
+    # TODO How does code handle MDQ that is 3 dp in the excel
+    "Ref to Scientist 1*": [0, 0.39],
+    "HetDel": [0.4, 0.6],  # theoretical value 0.5, represents CN 1
+    "Ref to Scientist 2*": [0.61, 0.79],
+    "Normal": [0.8, 1.2],  # theoretical value 1, represents CN 2
+    "Ref to Scientist 3*": [1.21, 1.39],
+    "HetDup": [1.4, 1.6],  # theoretical value 1.5, represents CN 3
+    "Ref to Scientist 4*": [1.61, float("Inf")]
+}
+
+''' ==================== qc_data.py constants ==================== '''
+
+SUMMARY_DATA_COLUMNS = [
+    "probe number",
+    "chr",
+    "gene",
+    "exon",
+    "start position",
+    "end position",
+    "normal copy number",
+    "dosage quotient",
+    "variant"
+]
\ No newline at end of file
diff --git a/dMLPA-flask/notes.py b/dMLPA-flask/notes.py
index 5167818..81573e6 100644
--- a/dMLPA-flask/notes.py
+++ b/dMLPA-flask/notes.py
@@ -2,12 +2,14 @@
 import numpy as np
 from cna import split_df_column, merge_df, clinical_df, copy_number_analysis, cnv_plot
 from qc_data import summary_data, df_to_html
+import config
 
 from jinja2 import Environment, PackageLoader
 import pdfkit
 
 # Test files to test script
-xls = "/home/epayne1/Documents/GSTT/test_input/BP01-11-dMLPA_R210v1_Pan5025_185897_SURNAME_Firstname11_filtered_results.xlsx"
+# xls = "/home/epayne1/Documents/GSTT/test_input/BP01-16-dMLPA_R210v1_Pan5025_218513_SURNAME_Unaffected_filtered_results.xlsx""
+xls = "/home/epayne1/Documents/GSTT/test_input/BP01-16-dMLPA_R210v1_Pan5025_218513_SURNAME_Firstname16_filtered_results.xlsx"
 probe = "/home/epayne1/Documents/GSTT/probe_file/PIF_D001-C1_Hereditary_Cancer_Panel_1-v02_modB1probe.xlsx"
 
 # Reading data in dataframes
@@ -28,20 +30,14 @@
 probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
 raw_results_df.rename(columns={raw_results_df.columns[-1]:"dosage quotient"}, inplace=True)
 
-# Definining unique columns needed from df's
-columns_results_df = [
-    "dosage quotient"
-]
-
-columns_probe_df = [
-    "probe number",
-    "chr",
-    "start position",
-    "end position",
-]
-
 # creating results_clinical dataframe
-results_clinical_df = clinical_df(raw_results_df, probe_df, columns_results_df, columns_probe_df)
+results_clinical_df = clinical_df(
+            raw_results_df,
+            probe_df,
+            config.COLUMNS_RESULTS_DF,
+            config.COLUMNS_PROBE_DF
+        )
+
 results_clinical_df["start position"] = results_clinical_df["start position"].astype(int)
 results_clinical_df["end position"] = results_clinical_df["end position"].astype(int)
 # results_clinical_df.to_excel("merged_excel.xlsx")
@@ -53,6 +49,7 @@
 cnv_summary_df, marked_df = copy_number_analysis(results_clinical_df, gene_list)
 # marked_df.to_excel("marked_df.xlsx")
 
+
 # obtaining summary data per gene for appendix
 summary_data_df = summary_data(marked_df)
 # print(summary_data_df)
@@ -62,34 +59,30 @@
 for gene in gene_list:
     # summary data by gene
     summary_data_gene =  summary_data_df[summary_data_df["gene"] == gene]
-    print(summary_data_gene)
+    # print(summary_data_gene)
     summary_data_gene.columns = summary_data_gene.columns.str.replace("_", " ").str.title()
     summary_tables[gene] = df_to_html(summary_data_gene)
     # summary_tables[gene] = gene_summary_data(gene, summary_data_df)
 
-    # cnv_detail by impacted gene
-    impacted_genes = cnv_summary_df["Gene"].unique()
-    if gene not in impacted_genes:
-        continue
-    else:
-        results_gene = marked_df[marked_df["gene"] == gene]
-        gene_plot = cnv_plot(results_gene, gene)
-        impacted_probes = summary_data_gene.dropna(subset=["Variant"])
-        impacted_probes = summary_data_gene.loc[summary_data_gene["Variant"] != ""]
-        impacted_probes = (impacted_probes.sort_values(by="Start Position"))
-        print(impacted_probes)
-
-        impacted_probes_html = df_to_html(impacted_probes)
-
-        # saving to cnv_detail dict.
-        cnv_detail[gene] = [gene_plot, impacted_probes_html]
-
-gene_plots = {}
-
-for gene in gene_list:
-    results_gene = results_clinical_df[results_clinical_df["gene"] == gene]
-    gene_plot = cnv_plot(results_gene, gene)
-    gene_plots[gene] = gene_plot
+    print(cnv_summary_df)
+
+    if not cnv_summary_df.empty:
+        # cnv_detail by impacted gene
+        impacted_genes = cnv_summary_df["Gene"].unique()
+        if gene not in impacted_genes:
+            continue
+        else:
+            results_gene = marked_df[marked_df["gene"] == gene]
+            gene_plot = cnv_plot(results_gene, gene)
+            impacted_probes = summary_data_gene.dropna(subset=["Variant"])
+            impacted_probes = summary_data_gene.loc[summary_data_gene["Variant"] != ""]
+            impacted_probes = (impacted_probes.sort_values(by="Start Position"))
+            print(impacted_probes)
+
+            impacted_probes_html = df_to_html(impacted_probes)
+
+            # saving to cnv_detail dict.
+            cnv_detail[gene] = [gene_plot, impacted_probes_html]
 
 # env = Environment(loader=PackageLoader("dMLPA-flask", "templates"))
 # template = env.get_template("report.html")
diff --git a/dMLPA-flask/qc_data.py b/dMLPA-flask/qc_data.py
index cbd8cd0..3507b7c 100644
--- a/dMLPA-flask/qc_data.py
+++ b/dMLPA-flask/qc_data.py
@@ -14,6 +14,7 @@
 """
 
 import re
+import config
 
 def get_test_codes(df):
     """
@@ -82,24 +83,8 @@ def df_to_html(df):
 def summary_data(df):
     """Create summary table of key probe data"""
 
-    # Merging start and end co-ordinates into one column
-    # df["co-ordinates"] = df["start position"].astype(str) + "-" + df["end position"].astype(str)
-
-    # defining columns desired in table from existing clinical_data.
-    columns_to_keep = [
-        "probe number",
-        "chr",
-        "gene",
-        "exon",
-        "start position",
-        "end position",
-        "normal copy number",
-        "dosage quotient",
-        "variant"
-    ]
-
     # creating copy of selected columns with Copy number estimate column
-    summary_data_df = df[columns_to_keep].copy()
+    summary_data_df = df[config.SUMMARY_DATA_COLUMNS].copy()
     summary_data_df["copy number est."] = "-"
     summary_data_df["copy number est."] = summary_data_df["copy number est."].astype(object)
 
diff --git a/dMLPA-flask/templates/report.html b/dMLPA-flask/templates/report.html
index ee87078..e9220d2 100644
--- a/dMLPA-flask/templates/report.html
+++ b/dMLPA-flask/templates/report.html
@@ -64,14 +64,17 @@
 <body>
 
 <header>
-    <div class="header-element1"> dMLPA report generator v.<!-- TODO Insert Jinja2 --></div>
+    <div class="header-element1"> dMLPA report generator {{ app_version }}</div>
     <div class="header-element2"> {{ file_name }}</div>
 </header>
 <hr><br>
 
+{% if prouction == FALSE %}
+            <h4 style="color: red">WARNING: This report was generated in a development environment. Do not use for clinical reporting</h4>
+{% endif %}
+
 <h1>dMLPA Copy Number Analysis</h1>
 <p><i>South East Genomics Laboratory Hub</i></p>
-
 <br>
 
 <div class="container">
@@ -81,9 +84,10 @@ <h4>Test Request</h4>
             <p><b>Pan Number: </b><i>{{ pan_number }}</i></p>
             <p><b>Genes: </b><i>{{ genes }}</i></p>
             <br>
-            <p><b>Run ID: </b><i><!-- TODO To be parsed after confirming sample naming convention --></i></p>
+            <p><b>Run ID: </b><i> {{ run_id }}</i></p>
+            <p><b>Genome Build: </b><i> {{ genome_build }}</i></p>
             <p><b>File-Name: </b><i>{{ patient_file }}</i></p>
-            <p><b>Probe-file: </b><i>{{ probe_file }}</i><!-- TODO Insert Probe file used --> </p>
+            <p><b>Probe-file: </b><i>{{ probe_file }}</i></p>
     </div>
     <div class="element">
         {% if qc_status == "PASSED" %}
@@ -97,25 +101,33 @@ <h4>Test Request</h4>
 
 <div class="body">
     <h2>Called CNV's</h2>
+    {% if '<td>' is in cnv_summary%}
         <p>{{ cnv_summary }}</p>
-        {% for key, value in report_notes.items() %}
-            {% if key in cnv_summary %}
-                <p>{{ value}}</p>
-            {% endif %}
-        {% endfor %}
+            {% for key, value in report_notes.items() %}
+                {% if key in cnv_summary %}
+                    <p>{{ value}}</p>
+                {% endif %}
+            {% endfor %}
+    {% else %}
+        <p>NO COPY NUMBER VARIANTS DETECTED</p>
+    {% endif %}
 </div>
 
 <br>
 
 <div class="body">
     <h3>CNV details</h3>
-    {% for gene, data in cnv_detail.items() %}
-        <p>{{ gene }}_plot</p>
-        <img src="data:image/svg+xml;base64,{{ data[0] }}" alt="{{ gene }} Plot" style="display: block; margin-left: auto; margin-right: auto">
-        <br>
-        <p>{{ data[1] }}</p>
-        <br>
-    {% endfor %}
+    {% if cnv_detail %}
+        {% for gene, data in cnv_detail.items() %}
+            <p>{{ gene }}_plot</p>
+            <img src="data:image/svg+xml;base64,{{ data[0] }}" alt="{{ gene }} Plot" style="display: block; margin-left: auto; margin-right: auto">
+            <br>
+            <p>{{ data[1] }}</p>
+            <br>
+        {% endfor %}
+    {% else %}
+        <p>NO COPY NUMBER VARIANTS DETECTED</p>
+    {% endif %}
 </div>
 
 <div class="page-break"></div>

From 1723a73b58f136e5dff47457ee33eb58993ea44b Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Tue, 28 May 2024 12:04:20 +0100
Subject: [PATCH 10/15] Adds initial draft for special reporting of PMS2

- Alternative DQ ranges specified for PMS2 in config.py
- Generates a different graph when reporting on PMS2
---
 dMLPA-flask/backend.py            |  8 +++-
 dMLPA-flask/cna.py                | 59 ++++++++++++++++++++++-
 dMLPA-flask/config.py             | 79 +++++++++++++++++++++++++++++--
 dMLPA-flask/templates/report.html | 12 +++--
 4 files changed, 145 insertions(+), 13 deletions(-)

diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index fde6bc0..9cb591e 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -6,7 +6,7 @@
 import pandas as pd
 import config
 from qc_data import get_test_codes, df_to_html, get_qc_status, summary_data
-from cna import split_df_column, clinical_df, copy_number_analysis, cnv_plot
+from cna import split_df_column, clinical_df, copy_number_analysis, cnv_plot, pms2_plot
 
 img_versioned = os.getenv(
     "IMG_VERSIONED"
@@ -80,7 +80,10 @@ def run_backend(single_file, array_of_files, output_directory):
                 if gene in impacted_genes:
                     # plotting gene data
                     results_gene_df = marked_df[marked_df["gene"] == gene]
-                    gene_plot = cnv_plot(results_gene_df, gene)
+                    if gene == "PMS2":
+                        gene_plot = pms2_plot(results_gene_df, gene)
+                    else:
+                        gene_plot = cnv_plot(results_gene_df, gene)
 
                     # creating table of probes involved in CNV
                     impacted_probes_df = summary_data_gene_df.dropna(subset=["Variant"])
@@ -127,6 +130,7 @@ def run_backend(single_file, array_of_files, output_directory):
             "probe_file": probe_file,
             "qc_status": qc_status,
             "cnv_summary": summary_html,
+            "cnc_comment": config.CNC_CATEGORIES_DISCLAIMER,
             "cnv_detail": cnv_detail,
             "report_notes": config.REPORTING_NOTES,
             "qc_table": qc_data,
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
index fc75609..f31bd2b 100644
--- a/dMLPA-flask/cna.py
+++ b/dMLPA-flask/cna.py
@@ -133,8 +133,12 @@ def copy_number_analysis(df, gene_list):
         # Get results per gene
         results_gene =  df[df["gene"] == gene]
 
+        if gene == "PMS2":
+            cnc_categories = config.PMS2_CNC_CATEGORIES
+        else:
+            cnc_categories = config.CNC_CATEGORIES
         # iterate through CN categories and identify matching probe entries
-        for direction, parameters in config.CNC_CATEGORIES.items():
+        for direction, parameters in cnc_categories.items():
             if direction != "Normal":
                 # print(direction, gene) # TODO Replace with logging
                 cnv_rows = np.where((parameters[0] <= results_gene["dosage quotient"]) &
@@ -254,4 +258,57 @@ def cnv_plot(df, gene, extend=1000):
 
     plt.close()
 
+    return img_base64
+
+def pms2_plot(df, gene, extend=1000):
+    # create copy of dataframe
+    cnv_copy = df.copy()
+
+    # pull specific data from columns
+    start = min(cnv_copy["start position"]) - extend
+    end = max(cnv_copy["end position"]) + extend
+
+    # determining y-axis limit for DQ plot
+    ylim = (0, max(2.0, 1.25*cnv_copy["dosage quotient"].max()))
+
+    #setting up plot layout
+    plt.figure(figsize=(10,4))
+
+    # setting axis
+    plt.xlim(start,end)
+    plt.ylim(*ylim)
+    plt.tick_params(axis='both', labelsize=8)
+
+    # setting plot labels
+    plt.ylabel("Dosage Quotient", fontdict={'family': 'Sans', 'size': 8})
+    plt.xlabel("Co-ordinates", fontdict={'family': 'Sans', 'size': 8})
+    plt.title(gene, fontdict={'family': 'Sans', 'size': 10})
+
+    # formatting x-axis ticks
+    # but this is from chatGPT and i need to question if this is the best way to do this.
+    plt.gca().xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: '{:d}'.format(int(x))))
+
+    # adding shading to define ranges for normal, hetdel and hetdup
+    cnc_categories = config.PMS2_CNC_CATEGORIES
+    plt.axhspan(cnc_categories["Normal"][0], cnc_categories["Normal"][1], facecolor = "#00FF0044", edgecolor = None)
+    plt.axhspan(cnc_categories["Homo Del/CompHet Del"][0], cnc_categories["Homo Del/CompHet Del"][1], facecolor = "#FF000044", edgecolor = None)
+    plt.axhspan(cnc_categories["PMS2/PMS2CL HetDel"][0], cnc_categories["PMS2/PMS2CL HetDel"][1], facecolor = "#FF7F7F44", edgecolor = None)
+    plt.axhspan(cnc_categories["PMS2/PMS2Cl HetDup"][0], cnc_categories["PMS2/PMS2Cl HetDup"][1], facecolor = "#98C5F144", edgecolor = None)
+    plt.axhspan(cnc_categories["Homo Dup/CompHet Dup"][0], cnc_categories["Homo Dup/CompHet Dup"][1], facecolor = "#318CE744", edgecolor = None)
+
+    # plot DQ value
+    plt.plot(cnv_copy["start position"], cnv_copy["dosage quotient"], marker = "x", linestyle = "", color = "black")
+
+    # plt.show()
+
+    # Save plot as image
+    img = io.BytesIO()
+    plt.savefig(img, format="svg")
+    img.seek(0)
+
+    # Encode SVG image as Base64
+    img_base64 = base64.b64encode(img.getvalue()).decode()
+
+    plt.close()
+
     return img_base64
\ No newline at end of file
diff --git a/dMLPA-flask/config.py b/dMLPA-flask/config.py
index c02dad0..9f1988c 100644
--- a/dMLPA-flask/config.py
+++ b/dMLPA-flask/config.py
@@ -29,31 +29,89 @@
     "start position",
     "end position",
 ]
+
 REPORTING_NOTES = {
+    # Generic message
+    "Ref to Scientist": (
+        "Ref to Scientist: The dosage quotient for one or "\
+        "more probes falls outside the typical ranges used "\
+        "to evaluate copy number. Scientist review required"
+    ),
+    # Below HetDel
     "Ref to Scientist 1": (
         "Ref to Scientist 1: The dosage quotient for one or "\
         "more probes falls below the range expected for a copy "\
         "number loss. Scientist review required."
     ),
+    # Between HetDel and Normal CN
     "Ref to Scientist 2": (
         "Ref to Scientist 2: The dosage quotient for one or "\
         "more probes falls in between the expected ranges for "\
         "copy number loss and copy number neutral. Scientist review "\
         "required"
     ),
+    # Between Normal CN and HetDup
     "Ref to Scientist 3": (
         "Ref to Scientist 3: The dosage quotient for one or "\
         "more probes falls in between the expected ranges for copy "\
         "number neutral and copy number gain. Scientist review "\
         "required."
     ),
+    # Above HetDup
     "Ref to Scientist 4": (
         "Ref to Scientist 4: The dosage quotient for one or "\
         "more probes falls above the range expected for a copy "\
         "number gain. Scientist review required."
+    ),
+    # Below typical ranges for any CN loss in PMS2/PMS2CL
+    "Ref to Scientist 5": (
+        "Ref to Scientist 5: The dosage quotient for one or "\
+        "more probes falls below the ranges typical used to " \
+        "assess PMS2 copy number. Scientist review required"
+    ),
+    # In between typical observed CN ranges for PMS2/PMS3CL
+    "Ref to Scientist 6": (
+        "Ref to Scientist 6: The dosage quotient for one or "\
+        "more probes falls in between the typical ranges used "\
+        "to determine PMS2 copy number. Scientist review "\
+        "required"
+    ),
+    "Ref to Scientist 7": (
+        "Ref to Scientist 7: The dosage quotient for one or "\
+        "more probes falls in between the typical ranges used "\
+        "to determine PMS2/PMS2CL copy number. Scientist review "\
+        "required"
+    ),
+    "Ref to Scientist 8": (
+        "Ref to Scientist 8: The dosage quotient for one or "\
+        "more probes falls in between the typical ranges used "\
+        "to determine PMS2/PMS2CL copy number. Scientist review "\
+        "required"
+    ),
+    "Ref to Scientist 9": (
+        "Ref to Scientist 9: The dosage quotient for one or "\
+        "more probes falls in between the typical ranges used "\
+        "to determine PMS2/PMS2CL copy number. Scientist review "\
+        "required"
+    ),
+    # Above typical ranges for any CN gain in PMS2/PMS2CL
+    "Ref to Scientist 10": (
+        "Ref to Scientist 7: The dosage quotient for one or "\
+        "more probes falls above the ranges typical used to " \
+        "assess PMS2 copy number. Scientist review required"
     )
 }
 
+CNC_CATEGORIES_DISCLAIMER = """The following DQ ranges are used to \
+    assess copy number in most genes: Normal Copy Number 0.8-1.2; \
+    Heterzygous Copy Number Loss 0.4-0.6; Heterzygous Copy Number \
+    Gain 1.4-16.
+    Of note, analysis of copy number variation \
+    in PMS2 is complicated by the pseudogene PMS2CL. Alternative \
+    DQ ranges have been used to asses copy number in this gene and \
+    Scientist Review is Recommended for all PMS2 Cases."""
+
+
 ''' ==================== cna.py constants ==================== '''
 
 DESIRED_COLUMN_ORDER = [
@@ -71,10 +129,6 @@
     "additional information",
 ]
 
-GENE_UNIQUE_CUTOFF = [
-    "PMS2"
-]
-
 CNC_CATEGORIES = {
     # TODO How does code handle MDQ that is 3 dp in the excel
     "Ref to Scientist 1*": [0, 0.39],
@@ -86,6 +140,21 @@
     "Ref to Scientist 4*": [1.61, float("Inf")]
 }
 
+PMS2_CNC_CATEGORIES = {
+    # TODO Stake holders to confirm thresholds for this
+    "Ref to Scientist 5*": [0, 0.39],
+    "Homo Del/CompHet Del": [0.4, 0.60], # theortical value of 0.5
+    "Ref to Scientist 6*": [0.61, 0.64],
+    "PMS2/PMS2CL HetDel": [0.65, 0.84],  # theoretical value 0.75
+    "Ref to Scientist 7*": [0.85, 0.85],
+    "Normal": [0.86, 1.14],  # theoretical value 1
+    "Ref to Scientist 8*": [1.15, 1.15],
+    "PMS2/PMS2Cl HetDup": [1.16, 1.35],  # theoretical value 1.25,
+    "Ref to Scientist 9*": [1.36, 1.39],
+    "Homo Dup/CompHet Dup": [1.4, 1.6],
+    "Ref to Scientist 10*": [1.61, float("Inf")]
+}
+
 ''' ==================== qc_data.py constants ==================== '''
 
 SUMMARY_DATA_COLUMNS = [
@@ -98,4 +167,4 @@
     "normal copy number",
     "dosage quotient",
     "variant"
-]
\ No newline at end of file
+]
diff --git a/dMLPA-flask/templates/report.html b/dMLPA-flask/templates/report.html
index e9220d2..bdf7cd9 100644
--- a/dMLPA-flask/templates/report.html
+++ b/dMLPA-flask/templates/report.html
@@ -103,13 +103,15 @@ <h4>Test Request</h4>
     <h2>Called CNV's</h2>
     {% if '<td>' is in cnv_summary%}
         <p>{{ cnv_summary }}</p>
-            {% for key, value in report_notes.items() %}
-                {% if key in cnv_summary %}
-                    <p>{{ value}}</p>
-                {% endif %}
-            {% endfor %}
+        <p><i>{{ cnc_comment }}</i></p>
+        {% for key, value in report_notes.items() %}
+            {% if key in cnv_summary %}
+                <p>{{ value}}</p>
+            {% endif %}
+        {% endfor %}
     {% else %}
         <p>NO COPY NUMBER VARIANTS DETECTED</p>
+        <p><i>{{ cnc_comment }}</i></p>
     {% endif %}
 </div>
 

From d1cfa6b046e4c956efd597ac142967e07b236d0b Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Wed, 29 May 2024 12:38:36 +0100
Subject: [PATCH 11/15] Implements minor changes to PMS2 reporting strategy

- Adds PMS2 reporting disclaimer
- Adds exon 12 boundary demarcation
- Renames categories used for PMS2 CN variants
[Issue:]
---
 dMLPA-flask/backend.py            |   6 +-
 dMLPA-flask/cna.py                |  18 +++-
 dMLPA-flask/config.py             | 150 ++++++++++++++++--------------
 dMLPA-flask/templates/report.html |  22 ++++-
 4 files changed, 111 insertions(+), 85 deletions(-)

diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index 9cb591e..708fce9 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -99,9 +99,6 @@ def run_backend(single_file, array_of_files, output_directory):
         qc_status = get_qc_status(metadata_df)
         qc_data = df_to_html(metadata_df)
 
-        # Converting gene_list to string
-        genes = ", ".join(gene_list)
-
         # Getting input file basenames
         file_name = os.path.basename(file)
         probe_file = os.path.basename(single_file)
@@ -123,7 +120,7 @@ def run_backend(single_file, array_of_files, output_directory):
             "file_name": report,
             "r_code": r_code,
             "pan_number": pan_number,
-            "genes": genes,
+            "genes": gene_list,
             "run_id": "", # TODO to be added once file name is confirmed, 
             "genome_build": config.GENOME_BUILD,
             "patient_file": file_name,
@@ -131,6 +128,7 @@ def run_backend(single_file, array_of_files, output_directory):
             "qc_status": qc_status,
             "cnv_summary": summary_html,
             "cnc_comment": config.CNC_CATEGORIES_DISCLAIMER,
+            "pms2_comment": config.PMS2_DISCLAIMER,
             "cnv_detail": cnv_detail,
             "report_notes": config.REPORTING_NOTES,
             "qc_table": qc_data,
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
index f31bd2b..3064be7 100644
--- a/dMLPA-flask/cna.py
+++ b/dMLPA-flask/cna.py
@@ -291,10 +291,20 @@ def pms2_plot(df, gene, extend=1000):
     # adding shading to define ranges for normal, hetdel and hetdup
     cnc_categories = config.PMS2_CNC_CATEGORIES
     plt.axhspan(cnc_categories["Normal"][0], cnc_categories["Normal"][1], facecolor = "#00FF0044", edgecolor = None)
-    plt.axhspan(cnc_categories["Homo Del/CompHet Del"][0], cnc_categories["Homo Del/CompHet Del"][1], facecolor = "#FF000044", edgecolor = None)
-    plt.axhspan(cnc_categories["PMS2/PMS2CL HetDel"][0], cnc_categories["PMS2/PMS2CL HetDel"][1], facecolor = "#FF7F7F44", edgecolor = None)
-    plt.axhspan(cnc_categories["PMS2/PMS2Cl HetDup"][0], cnc_categories["PMS2/PMS2Cl HetDup"][1], facecolor = "#98C5F144", edgecolor = None)
-    plt.axhspan(cnc_categories["Homo Dup/CompHet Dup"][0], cnc_categories["Homo Dup/CompHet Dup"][1], facecolor = "#318CE744", edgecolor = None)
+    plt.axhspan(cnc_categories["PMS2 CN loss"][0], cnc_categories["PMS2 CN loss"][1], facecolor = "#FF000044", edgecolor = None)
+    plt.axhspan(cnc_categories["Potential PMS2 CN loss"][0], cnc_categories["Potential PMS2 CN loss"][1], facecolor = "#FF7F7F44", edgecolor = None)
+    plt.axhspan(cnc_categories["Potential PMS2 CN gain"][0], cnc_categories["Potential PMS2 CN gain"][1], facecolor = "#98C5F144", edgecolor = None)
+    plt.axhspan(cnc_categories["PMS2 CN gain"][0], cnc_categories["PMS2 CN gain"][1], facecolor = "#318CE744", edgecolor = None)
+
+    # adding boundary marker between PMS2 exon 11 and exon 12
+    plt.axvline(
+        x=config.PMS2_PMS2CL_BOUNDARY,
+        # ymin = cnc_categories["Del2"][0],
+        # ymax = cnc_categories["Dup2"][1],
+        color='black',
+        linestyle = '--',
+        lw = 0.75
+    )
 
     # plot DQ value
     plt.plot(cnv_copy["start position"], cnv_copy["dosage quotient"], marker = "x", linestyle = "", color = "black")
diff --git a/dMLPA-flask/config.py b/dMLPA-flask/config.py
index 9f1988c..169307c 100644
--- a/dMLPA-flask/config.py
+++ b/dMLPA-flask/config.py
@@ -37,79 +37,82 @@
         "more probes falls outside the typical ranges used "\
         "to evaluate copy number. Scientist review required"
     ),
-    # Below HetDel
-    "Ref to Scientist 1": (
-        "Ref to Scientist 1: The dosage quotient for one or "\
-        "more probes falls below the range expected for a copy "\
-        "number loss. Scientist review required."
-    ),
-    # Between HetDel and Normal CN
-    "Ref to Scientist 2": (
-        "Ref to Scientist 2: The dosage quotient for one or "\
-        "more probes falls in between the expected ranges for "\
-        "copy number loss and copy number neutral. Scientist review "\
-        "required"
-    ),
-    # Between Normal CN and HetDup
-    "Ref to Scientist 3": (
-        "Ref to Scientist 3: The dosage quotient for one or "\
-        "more probes falls in between the expected ranges for copy "\
-        "number neutral and copy number gain. Scientist review "\
-        "required."
-    ),
-    # Above HetDup
-    "Ref to Scientist 4": (
-        "Ref to Scientist 4: The dosage quotient for one or "\
-        "more probes falls above the range expected for a copy "\
-        "number gain. Scientist review required."
-    ),
-    # Below typical ranges for any CN loss in PMS2/PMS2CL
-    "Ref to Scientist 5": (
-        "Ref to Scientist 5: The dosage quotient for one or "\
-        "more probes falls below the ranges typical used to " \
-        "assess PMS2 copy number. Scientist review required"
-    ),
-    # In between typical observed CN ranges for PMS2/PMS3CL
-    "Ref to Scientist 6": (
-        "Ref to Scientist 6: The dosage quotient for one or "\
-        "more probes falls in between the typical ranges used "\
-        "to determine PMS2 copy number. Scientist review "\
-        "required"
-    ),
-    "Ref to Scientist 7": (
-        "Ref to Scientist 7: The dosage quotient for one or "\
-        "more probes falls in between the typical ranges used "\
-        "to determine PMS2/PMS2CL copy number. Scientist review "\
-        "required"
-    ),
-    "Ref to Scientist 8": (
-        "Ref to Scientist 8: The dosage quotient for one or "\
-        "more probes falls in between the typical ranges used "\
-        "to determine PMS2/PMS2CL copy number. Scientist review "\
-        "required"
-    ),
-    "Ref to Scientist 9": (
-        "Ref to Scientist 9: The dosage quotient for one or "\
-        "more probes falls in between the typical ranges used "\
-        "to determine PMS2/PMS2CL copy number. Scientist review "\
-        "required"
-    ),
-    # Above typical ranges for any CN gain in PMS2/PMS2CL
-    "Ref to Scientist 10": (
-        "Ref to Scientist 7: The dosage quotient for one or "\
-        "more probes falls above the ranges typical used to " \
-        "assess PMS2 copy number. Scientist review required"
-    )
+    # # Below HetDel
+    # "Ref to Scientist 1": (
+    #     "Ref to Scientist 1: The dosage quotient for one or "\
+    #     "more probes falls below the range expected for a copy "\
+    #     "number loss. Scientist review required."
+    # ),
+    # # Between HetDel and Normal CN
+    # "Ref to Scientist 2": (
+    #     "Ref to Scientist 2: The dosage quotient for one or "\
+    #     "more probes falls in between the expected ranges for "\
+    #     "copy number loss and copy number neutral. Scientist review "\
+    #     "required"
+    # ),
+    # # Between Normal CN and HetDup
+    # "Ref to Scientist 3": (
+    #     "Ref to Scientist 3: The dosage quotient for one or "\
+    #     "more probes falls in between the expected ranges for copy "\
+    #     "number neutral and copy number gain. Scientist review "\
+    #     "required."
+    # ),
+    # # Above HetDup
+    # "Ref to Scientist 4": (
+    #     "Ref to Scientist 4: The dosage quotient for one or "\
+    #     "more probes falls above the range expected for a copy "\
+    #     "number gain. Scientist review required."
+    # ),
+    # # Below typical ranges for any CN loss in PMS2/PMS2CL
+    # "Ref to Scientist 5": (
+    #     "Ref to Scientist 5: The dosage quotient for one or "\
+    #     "more probes falls below the ranges typical used to " \
+    #     "assess PMS2 copy number. Scientist review required"
+    # ),
+    # # In between typical observed CN ranges for PMS2/PMS3CL
+    # "Ref to Scientist 6": (
+    #     "Ref to Scientist 6: The dosage quotient for one or "\
+    #     "more probes falls in between the typical ranges used "\
+    #     "to determine PMS2 copy number. Scientist review "\
+    #     "required"
+    # ),
+    # "Ref to Scientist 7": (
+    #     "Ref to Scientist 7: The dosage quotient for one or "\
+    #     "more probes falls in between the typical ranges used "\
+    #     "to determine PMS2/PMS2CL copy number. Scientist review "\
+    #     "required"
+    # ),
+    # "Ref to Scientist 8": (
+    #     "Ref to Scientist 8: The dosage quotient for one or "\
+    #     "more probes falls in between the typical ranges used "\
+    #     "to determine PMS2/PMS2CL copy number. Scientist review "\
+    #     "required"
+    # ),
+    # "Ref to Scientist 9": (
+    #     "Ref to Scientist 9: The dosage quotient for one or "\
+    #     "more probes falls in between the typical ranges used "\
+    #     "to determine PMS2/PMS2CL copy number. Scientist review "\
+    #     "required"
+    # ),
+    # # Above typical ranges for any CN gain in PMS2/PMS2CL
+    # "Ref to Scientist 10": (
+    #     "Ref to Scientist 7: The dosage quotient for one or "\
+    #     "more probes falls above the ranges typical used to " \
+    #     "assess PMS2 copy number. Scientist review required"
+    # )
 }
 
 CNC_CATEGORIES_DISCLAIMER = """The following DQ ranges are used to \
     assess copy number in most genes: Normal Copy Number 0.8-1.2; \
     Heterzygous Copy Number Loss 0.4-0.6; Heterzygous Copy Number \
-    Gain 1.4-16.
-    Of note, analysis of copy number variation \
-    in PMS2 is complicated by the pseudogene PMS2CL. Alternative \
-    DQ ranges have been used to asses copy number in this gene and \
-    Scientist Review is Recommended for all PMS2 Cases."""
+    Gain 1.4-16."""
+
+PMS2_DISCLAIMER = """OF NOTE, analysis of copy number variation in \
+    PMS2 is complicated by the pseudogene PMS2CL. The following \
+    alternative DQ ranges have been used to assess copy number as \
+    a result: Normal Copy Number 0.86-1.14; Potential Copy Number Loss \
+    0.4-0.84; Potential Copy Number Gain: 1.16-1.6. Scientist Review \
+    is Recommended for all PMS2 Cases."""
 
 
 ''' ==================== cna.py constants ==================== '''
@@ -143,18 +146,21 @@
 PMS2_CNC_CATEGORIES = {
     # TODO Stake holders to confirm thresholds for this
     "Ref to Scientist 5*": [0, 0.39],
-    "Homo Del/CompHet Del": [0.4, 0.60], # theortical value of 0.5
+    "PMS2 CN loss": [0.4, 0.60], # theortical value of 0.5
     "Ref to Scientist 6*": [0.61, 0.64],
-    "PMS2/PMS2CL HetDel": [0.65, 0.84],  # theoretical value 0.75
+    "Potential PMS2 CN loss": [0.65, 0.84],  # theoretical value 0.75
     "Ref to Scientist 7*": [0.85, 0.85],
     "Normal": [0.86, 1.14],  # theoretical value 1
     "Ref to Scientist 8*": [1.15, 1.15],
-    "PMS2/PMS2Cl HetDup": [1.16, 1.35],  # theoretical value 1.25,
+    "Potential PMS2 CN gain": [1.16, 1.35],  # theoretical value 1.25,
     "Ref to Scientist 9*": [1.36, 1.39],
-    "Homo Dup/CompHet Dup": [1.4, 1.6],
+    "PMS2 CN gain": [1.4, 1.6],
     "Ref to Scientist 10*": [1.61, float("Inf")]
 }
 
+PMS2_PMS2CL_BOUNDARY = 5982991 #End of PMS2 exon 12 in transcript NM_000535.7 build GRCH38.p14
+    # TODO Consider how this will be impacted by implementation of newer probe files
+
 ''' ==================== qc_data.py constants ==================== '''
 
 SUMMARY_DATA_COLUMNS = [
diff --git a/dMLPA-flask/templates/report.html b/dMLPA-flask/templates/report.html
index bdf7cd9..a4ddad2 100644
--- a/dMLPA-flask/templates/report.html
+++ b/dMLPA-flask/templates/report.html
@@ -82,7 +82,11 @@ <h1>dMLPA Copy Number Analysis</h1>
         <h4>Test Request</h4>
             <p><b>R Code:</b>{{ r_code }}</p>
             <p><b>Pan Number: </b><i>{{ pan_number }}</i></p>
-            <p><b>Genes: </b><i>{{ genes }}</i></p>
+            <p><b>Genes: </b>
+                {%for gene in genes %}
+                    <i>{{ gene }}{% if not loop.last %}, {% endif %}</i>
+                {% endfor %}
+            </p>
             <br>
             <p><b>Run ID: </b><i> {{ run_id }}</i></p>
             <p><b>Genome Build: </b><i> {{ genome_build }}</i></p>
@@ -103,18 +107,26 @@ <h4>Test Request</h4>
     <h2>Called CNV's</h2>
     {% if '<td>' is in cnv_summary%}
         <p>{{ cnv_summary }}</p>
-        <p><i>{{ cnc_comment }}</i></p>
         {% for key, value in report_notes.items() %}
-            {% if key in cnv_summary %}
-                <p>{{ value}}</p>
-            {% endif %}
+        {% if key is in cnv_summary %}
+            <p>{{ value}}</p>
+        {% endif %}
+        <p><i>{{ cnc_comment }}</i></p>
+        {% if 'PMS2' in genes %}
+            <p><i>{{ pms2_comment }}</i></p>
+        {% endif %}
+
         {% endfor %}
     {% else %}
         <p>NO COPY NUMBER VARIANTS DETECTED</p>
         <p><i>{{ cnc_comment }}</i></p>
+        {% if 'PMS2' is in genes %}
+            <p><i>{{ psm2_comment }}</i></p>
+        {% endif %}
     {% endif %}
 </div>
 
+
 <br>
 
 <div class="body">

From 8bc181a4a6010df7ce1d71363ca0cc5a91586d5d Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Wed, 29 May 2024 17:55:56 +0100
Subject: [PATCH 12/15] corrects PMS2 disclaimer comment

---
 dMLPA-flask/config.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dMLPA-flask/config.py b/dMLPA-flask/config.py
index 169307c..0027557 100644
--- a/dMLPA-flask/config.py
+++ b/dMLPA-flask/config.py
@@ -110,10 +110,10 @@
 PMS2_DISCLAIMER = """OF NOTE, analysis of copy number variation in \
     PMS2 is complicated by the pseudogene PMS2CL. The following \
     alternative DQ ranges have been used to assess copy number as \
-    a result: Normal Copy Number 0.86-1.14; Potential Copy Number Loss \
-    0.4-0.84; Potential Copy Number Gain: 1.16-1.6. Scientist Review \
-    is Recommended for all PMS2 Cases."""
-
+    a result: Normal Copy Number 0.86-1.14; PMS2 CN loss: 0.4-0.6; \
+    Potential PMS2 CN Loss 0.65-0.84; Potential PMS2 CN Gain: \
+    1.16-1.35; PMS2 CN gain: 1.4-1.6. Scientist Review is \
+    Recommended for all PMS2 Cases."""
 
 ''' ==================== cna.py constants ==================== '''
 

From 44fdf1a71cb022be3d9259a026001719b4290afb Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Thu, 20 Jun 2024 15:30:51 +0100
Subject: [PATCH 13/15] Resolves merge conflicts

Resolves conflicts from merging in PMS2_merged_categories
---
 dMLPA-flask/backend.py |  2 +-
 dMLPA-flask/cna.py     | 12 +++---
 dMLPA-flask/config.py  | 94 +++++++++++-------------------------------
 3 files changed, 32 insertions(+), 76 deletions(-)

diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index 708fce9..cbf7a48 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -130,7 +130,7 @@ def run_backend(single_file, array_of_files, output_directory):
             "cnc_comment": config.CNC_CATEGORIES_DISCLAIMER,
             "pms2_comment": config.PMS2_DISCLAIMER,
             "cnv_detail": cnv_detail,
-            "report_notes": config.REPORTING_NOTES,
+            "report_notes": config.REPORTING_NOTE,
             "qc_table": qc_data,
             "summary_tables": summary_tables
         }
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
index 3064be7..6b56029 100644
--- a/dMLPA-flask/cna.py
+++ b/dMLPA-flask/cna.py
@@ -134,7 +134,7 @@ def copy_number_analysis(df, gene_list):
         results_gene =  df[df["gene"] == gene]
 
         if gene == "PMS2":
-            cnc_categories = config.PMS2_CNC_CATEGORIES
+            cnc_categories = config.PMS2_CATEGORIES
         else:
             cnc_categories = config.CNC_CATEGORIES
         # iterate through CN categories and identify matching probe entries
@@ -289,12 +289,12 @@ def pms2_plot(df, gene, extend=1000):
     plt.gca().xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: '{:d}'.format(int(x))))
 
     # adding shading to define ranges for normal, hetdel and hetdup
-    cnc_categories = config.PMS2_CNC_CATEGORIES
+    cnc_categories = config.PMS2_GRAPH_CATEGORIES
     plt.axhspan(cnc_categories["Normal"][0], cnc_categories["Normal"][1], facecolor = "#00FF0044", edgecolor = None)
-    plt.axhspan(cnc_categories["PMS2 CN loss"][0], cnc_categories["PMS2 CN loss"][1], facecolor = "#FF000044", edgecolor = None)
-    plt.axhspan(cnc_categories["Potential PMS2 CN loss"][0], cnc_categories["Potential PMS2 CN loss"][1], facecolor = "#FF7F7F44", edgecolor = None)
-    plt.axhspan(cnc_categories["Potential PMS2 CN gain"][0], cnc_categories["Potential PMS2 CN gain"][1], facecolor = "#98C5F144", edgecolor = None)
-    plt.axhspan(cnc_categories["PMS2 CN gain"][0], cnc_categories["PMS2 CN gain"][1], facecolor = "#318CE744", edgecolor = None)
+    plt.axhspan(cnc_categories["Del2"][0], cnc_categories["Del2"][1], facecolor = "#FF000044", edgecolor = None)
+    plt.axhspan(cnc_categories["Del1"][0], cnc_categories["Del1"][1], facecolor = "#FF7F7F44", edgecolor = None)
+    plt.axhspan(cnc_categories["Dup1"][0], cnc_categories["Dup1"][1], facecolor = "#98C5F144", edgecolor = None)
+    plt.axhspan(cnc_categories["Dup2"][0], cnc_categories["Dup2"][1], facecolor = "#318CE744", edgecolor = None)
 
     # adding boundary marker between PMS2 exon 11 and exon 12
     plt.axvline(
diff --git a/dMLPA-flask/config.py b/dMLPA-flask/config.py
index 0027557..7fd2161 100644
--- a/dMLPA-flask/config.py
+++ b/dMLPA-flask/config.py
@@ -30,76 +30,13 @@
     "end position",
 ]
 
-REPORTING_NOTES = {
+REPORTING_NOTE = {
     # Generic message
     "Ref to Scientist": (
         "Ref to Scientist: The dosage quotient for one or "\
         "more probes falls outside the typical ranges used "\
         "to evaluate copy number. Scientist review required"
     ),
-    # # Below HetDel
-    # "Ref to Scientist 1": (
-    #     "Ref to Scientist 1: The dosage quotient for one or "\
-    #     "more probes falls below the range expected for a copy "\
-    #     "number loss. Scientist review required."
-    # ),
-    # # Between HetDel and Normal CN
-    # "Ref to Scientist 2": (
-    #     "Ref to Scientist 2: The dosage quotient for one or "\
-    #     "more probes falls in between the expected ranges for "\
-    #     "copy number loss and copy number neutral. Scientist review "\
-    #     "required"
-    # ),
-    # # Between Normal CN and HetDup
-    # "Ref to Scientist 3": (
-    #     "Ref to Scientist 3: The dosage quotient for one or "\
-    #     "more probes falls in between the expected ranges for copy "\
-    #     "number neutral and copy number gain. Scientist review "\
-    #     "required."
-    # ),
-    # # Above HetDup
-    # "Ref to Scientist 4": (
-    #     "Ref to Scientist 4: The dosage quotient for one or "\
-    #     "more probes falls above the range expected for a copy "\
-    #     "number gain. Scientist review required."
-    # ),
-    # # Below typical ranges for any CN loss in PMS2/PMS2CL
-    # "Ref to Scientist 5": (
-    #     "Ref to Scientist 5: The dosage quotient for one or "\
-    #     "more probes falls below the ranges typical used to " \
-    #     "assess PMS2 copy number. Scientist review required"
-    # ),
-    # # In between typical observed CN ranges for PMS2/PMS3CL
-    # "Ref to Scientist 6": (
-    #     "Ref to Scientist 6: The dosage quotient for one or "\
-    #     "more probes falls in between the typical ranges used "\
-    #     "to determine PMS2 copy number. Scientist review "\
-    #     "required"
-    # ),
-    # "Ref to Scientist 7": (
-    #     "Ref to Scientist 7: The dosage quotient for one or "\
-    #     "more probes falls in between the typical ranges used "\
-    #     "to determine PMS2/PMS2CL copy number. Scientist review "\
-    #     "required"
-    # ),
-    # "Ref to Scientist 8": (
-    #     "Ref to Scientist 8: The dosage quotient for one or "\
-    #     "more probes falls in between the typical ranges used "\
-    #     "to determine PMS2/PMS2CL copy number. Scientist review "\
-    #     "required"
-    # ),
-    # "Ref to Scientist 9": (
-    #     "Ref to Scientist 9: The dosage quotient for one or "\
-    #     "more probes falls in between the typical ranges used "\
-    #     "to determine PMS2/PMS2CL copy number. Scientist review "\
-    #     "required"
-    # ),
-    # # Above typical ranges for any CN gain in PMS2/PMS2CL
-    # "Ref to Scientist 10": (
-    #     "Ref to Scientist 7: The dosage quotient for one or "\
-    #     "more probes falls above the ranges typical used to " \
-    #     "assess PMS2 copy number. Scientist review required"
-    # )
 }
 
 CNC_CATEGORIES_DISCLAIMER = """The following DQ ranges are used to \
@@ -110,10 +47,9 @@
 PMS2_DISCLAIMER = """OF NOTE, analysis of copy number variation in \
     PMS2 is complicated by the pseudogene PMS2CL. The following \
     alternative DQ ranges have been used to assess copy number as \
-    a result: Normal Copy Number 0.86-1.14; PMS2 CN loss: 0.4-0.6; \
-    Potential PMS2 CN Loss 0.65-0.84; Potential PMS2 CN Gain: \
-    1.16-1.35; PMS2 CN gain: 1.4-1.6. Scientist Review is \
-    Recommended for all PMS2 Cases."""
+    a result: Normal Copy Number 0.86-1.14; Potential Copy Number Loss \
+    0.4-0.84; Potential Copy Number Gain: 1.16-1.6. Scientist Review \
+    is Recommended for all PMS2 Cases."""
 
 ''' ==================== cna.py constants ==================== '''
 
@@ -143,7 +79,27 @@
     "Ref to Scientist 4*": [1.61, float("Inf")]
 }
 
-PMS2_CNC_CATEGORIES = {
+PMS2_CATEGORIES = {
+    # TODO Stake holders to confirm thresholds for this
+    "Ref to Scientist 1*": [0, 0.39],
+    "Potential PMS2 CN loss": [0.4, 0.84],
+    "Ref to Scientist 2*": [0.85, 0.85],
+    "Normal": [0.86, 1.14],  # theoretical value 1
+    "Ref to Scientist 3*": [1.15, 1.15],
+    "Potential PMS2 CN gain": [1.16, 1.6],
+    "Ref to Scientist 4*": [1.61, float("Inf")]
+}
+
+PMS2_GRAPH_CATEGORIES = {
+    # TODO Stake holders to confirm thresholds for this
+    "Del2": [0.4, 0.60, "#FF000044"], # theortical value of 0.5
+    "Del1": [0.65, 0.84, "#FF7F7F44"],  # theoretical value 0.75
+    "Normal": [0.86, 1.14, "#00FF0044"],  # theoretical value 1
+    "Dup1": [1.16, 1.35, "#98C5F144"],  # theoretical value 1.25,
+    "Dup2": [1.4, 1.6, "#318CE744"] # theoretical value of 1.5
+}
+
+PMS2_CATEGORIES_PREVIOUS = {
     # TODO Stake holders to confirm thresholds for this
     "Ref to Scientist 5*": [0, 0.39],
     "PMS2 CN loss": [0.4, 0.60], # theortical value of 0.5

From 7a2c55d2b5e656bdcff0b173b0b834efad981b78 Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Thu, 20 Jun 2024 18:15:08 +0100
Subject: [PATCH 14/15] updates location of PMS2_PMS2CL_boundary line on graph

---
 dMLPA-flask/config.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dMLPA-flask/config.py b/dMLPA-flask/config.py
index 7fd2161..d08633d 100644
--- a/dMLPA-flask/config.py
+++ b/dMLPA-flask/config.py
@@ -114,8 +114,9 @@
     "Ref to Scientist 10*": [1.61, float("Inf")]
 }
 
-PMS2_PMS2CL_BOUNDARY = 5982991 #End of PMS2 exon 12 in transcript NM_000535.7 build GRCH38.p14
-    # TODO Consider how this will be impacted by implementation of newer probe files
+PMS2_PMS2CL_BOUNDARY = 5983991 # 1000bp into the intron proximal to the
+    # start of exon 12 (i.e. intron 11-12). Based on transcript
+    # NM_000535.7 build GRCH38.p14
 
 ''' ==================== qc_data.py constants ==================== '''
 

From 59d8ded0e5a87ba451d69c9d1543146209d6a44f Mon Sep 17 00:00:00 2001
From: EKPayne <elena.payne@gstt.nhs.uk>
Date: Wed, 24 Jul 2024 11:37:42 +0100
Subject: [PATCH 15/15] Adds Error Handling and updates CN Categories

Error handling added:
- Files submitted that don't match expected file naming for
probe file and results file
- Probe file submitted that doesn't meet expected file formatting

CN Categories updated to 3dp to account for fact that dq in results file
is rounded to 3dp by digital_mlpa script (see https://github.com/moka-guys/digital_mlpa)
---
 dMLPA-flask/app.py               |  73 ++++++---
 dMLPA-flask/backend.py           | 270 +++++++++++++++++--------------
 dMLPA-flask/cna.py               |   2 -
 dMLPA-flask/config.py            |  39 +++--
 dMLPA-flask/qc_data.py           |   1 -
 dMLPA-flask/templates/index.html | 193 +++++++++++-----------
 6 files changed, 317 insertions(+), 261 deletions(-)

diff --git a/dMLPA-flask/app.py b/dMLPA-flask/app.py
index 7aee98e..ac1ada9 100644
--- a/dMLPA-flask/app.py
+++ b/dMLPA-flask/app.py
@@ -28,7 +28,8 @@
 from werkzeug.utils import secure_filename
 import zipfile
 
-
+import re
+import config
 import logging
 
 log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
@@ -48,8 +49,9 @@
 app.config["SESSION_FILE_DIR"] = os.environ["SESSION_FILE_DIR"]
 app.config["UPLOAD_EXTENSIONS"] = [
     ".xlsx",
-    ".bam",
-]  # TODO: If multiple file types, add to list i.e. [".txt", ".csv", ".xlsm", ".xlsx"]
+    ".xlsm",
+    ".xls"
+]
 app.config["MAX_CONTENT_LENGTH"] = 2 * 1024 * 1024  # File has to be less than 2MB
 app.config["APPLICATION_ROOT"] = "/dMLPA-flask"
 app.config["WTF_CSRF_ENABLED"] = True
@@ -69,18 +71,28 @@ def validate_input_file(form, field):
         "xlsx",
         "xlsm",
         "xls",
-    ]  # TODO: If multiple file types, add to list i.e. ["xlsm", "xlsx"]
+    ]
     file = field.data
     if not file:
-        raise ValidationError("No file provided.")
+        raise ValidationError("No probe file provided.")
     if not file.filename:
-        raise ValidationError("No file selected.")
+        raise ValidationError("No probe file selected.")
     if not (
         "." in file.filename
         and file.filename.rsplit(".", 1)[1].lower() in allowed_extensions
     ):
         raise ValidationError(
-            f"Invalid file type for input file '{file.filename}'. Allowed types are: xlsx"  # TODO: Ammend file types as appropriate i.e. "xlsm, xlsx"
+            f"Invalid file type for probe file '{file.filename}'. "
+            "Allowed types are: xlsx, xlsm, xls"
+        )
+    file_check = re.search(config.PROBEFILE_NAME_CONVENTION, file.filename)
+    if not file_check:
+        raise ValidationError(
+            f"File {file.filename} is not a valid probe file. "
+            "File name does not match expected naming convention for a probe "
+            "file which may result in processing errors. Please provide an "
+            "appropriate probe file which matches the expected naming "
+            "convention."
         )
 
 
@@ -89,19 +101,20 @@ def validate_input_files_array(form, field):
         "xlsx",
         "xlsm",
         "xls",
-    ]  # TODO: Ammend file types as appropriate i.e. ["xlsm", "xlsx"]
+    ]
     files = field.data
     if not files:
-        raise ValidationError("No files provided.")
+        raise ValidationError("No results files provided.")
     for file in files:
         if not file.filename:
-            raise ValidationError("No files selected.")
+            raise ValidationError("No results files selected.")
         if not (
             "." in file.filename
             and file.filename.rsplit(".", 1)[1].lower() in allowed_extensions
         ):
             raise ValidationError(
-                f"Invalid file type for input files '{file.filename}'. Allowed types are: bam"  # TODO: Ammend file types as appropriate i.e. "xlsm, xlsx"
+                f"Invalid file type for results files '{file.filename}'."
+                "Allowed types are: xlsx, xlsm, xls"
             )
 
 
@@ -112,7 +125,7 @@ class ChangeForm(FlaskForm):
         render_kw={
             "accept": ".xlsm, .xlsx, .xls",
             "id": "single_file",
-        },  # TODO: If multiple file types, add to list i.e. ".xlsm, .xlsx"
+        },
     )
     multiple_array_of_files = MultipleFileField(
         "Single/Multiple Patient Data Files (Excel):",
@@ -121,7 +134,7 @@ class ChangeForm(FlaskForm):
             "accept": ".xlsm, .xlsx, .xls",
             "multiple": "True",
             "id": "multiple_array_of_files",
-        },  # TODO: If multiple file types, add to list i.e. ".xlsm, .xlsx"
+        },
     )
     submit = SubmitField("Run dMLPA-flask")
 
@@ -137,7 +150,7 @@ def form(backend_state="initial"):
             return render_template(
                 "index.html",
                 form=chgForm,
-                file_errors=chgForm.errors,
+                validation_errors=chgForm.errors,
             )
 
         # Use FileHandler() to log to a file
@@ -185,40 +198,50 @@ def form(backend_state="initial"):
         # Create the path to the output folder
         output_path = os.path.join(app.config["OUTPUT_FOLDER"], session["timestr"])
 
-        zipped_output, input_errors, input_ok_flag = run_backend(
+        zipped_output, results_ok_flag, probe_ok_flag, probe_errors, unprocessed_files = run_backend(
             input_single_file_tmp_path,
             input_files_tmp_paths,
             output_path,
         )
+
+        # Establishing count of files
+        no_files_submitted = len(input_files_basenames)
+        no_files_processed = no_files_submitted - len(unprocessed_files)
+
+        # TODO add logging of any errors encountered with the probe or results flags
+
         # Save the zipped output path to the session
         session["zipped_output"] = zipped_output
 
-        if input_ok_flag == False:
+        if probe_ok_flag == False:
             return render_template(
                 "index.html",
                 form=chgForm,
-                backend_state="initial",
-                errors=input_errors,  # Pass the errors to the template
-                file_errors=chgForm.errors,
+                backend_state="failed",
+                single_file_name=input_file.filename,
+                validation_errors=chgForm.errors,
+                probe_errors=probe_errors,  # Pass the errors to the template
             )
         else:
             return render_template(
                 "index.html",
                 form=chgForm,
-                backend_state=backend_state,
+                backend_state="finished",
+                files_submitted = no_files_submitted,
+                files_processed = no_files_processed,
                 single_file_name=input_file.filename,
-                multiple_array_of_files_names=", ".join(
-                    [x.filename for x in input_files_array]
-                ),
+                multiple_array_of_files_names=input_files_basenames,
                 zipped_folder_name=zipped_output,
-                file_errors=chgForm.errors,
+                results_flag=results_ok_flag,
+                errors=unprocessed_files,
+                validation_errors=chgForm.errors,
             )
 
     return render_template(
         "index.html",
         form=chgForm,
         backend_state=backend_state,
-        file_errors=chgForm.errors,
+        validation_errors=chgForm.errors,
     )
 
 
diff --git a/dMLPA-flask/backend.py b/dMLPA-flask/backend.py
index cbf7a48..b05242c 100644
--- a/dMLPA-flask/backend.py
+++ b/dMLPA-flask/backend.py
@@ -1,4 +1,5 @@
 import os
+import re
 import zipfile
 from datetime import datetime
 from jinja2 import Environment, PackageLoader
@@ -14,6 +15,17 @@
 
 
 def run_backend(single_file, array_of_files, output_directory):
+
+    # USER WARNINGS: Anything added to the string 'probe_errors' will be passed from the backend to
+    # the frontend and displayed in a warning to the user to give immediate feedback to the
+    # user about errors such as incorrect formatting of input files etc.
+    # 'unprocessed_results_files on the other hand records a list of file names that won't be processed
+    # and will be returned to the user after other files have been processed.
+    probe_file_ok_flag: bool = True
+    results_file_ok_flag: bool = True
+    probe_errors = ""
+    unprocessed_results_files: list[str] = []
+
     # Ensure the output directory exists, if not, create it
     os.makedirs(output_directory, exist_ok=True)
 
@@ -29,136 +41,148 @@ def run_backend(single_file, array_of_files, output_directory):
     array_of_files = [os.path.abspath(file) for file in array_of_files]
 
     # Reading probe data into dataframe and formatting
-    probe_df = pd.read_excel(single_file, sheet_name="probe_info")
-    probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
-    probe_df = split_df_column(probe_df, "Mapview (hg38)a", config.NEW_PROBE_DF_HEADERS)
-    probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
+    try:
+        probe_df = pd.read_excel(single_file, sheet_name="probe_info")
+        probe_df.rename(columns={"Exon number": "Exon"}, inplace=True)
+        probe_df = split_df_column(probe_df, "Mapview (hg38)a", config.NEW_PROBE_DF_HEADERS)
+        probe_df["chr"] = probe_df["chr"].str.replace("chr", "").str.lstrip("0")
+    except Exception: #TODO Add cusom exception when adding logging.
+        error_message = (
+            "Probe File Error: Spreadsheet headers don't match expected file format "\
+            "resultantly result files could not be processed. Confirm that an "\
+            "appropriate probe file was provided for analysis"
+        )
+        probe_file_ok_flag = False
+        return zip_name, results_file_ok_flag, probe_file_ok_flag, error_message, unprocessed_results_files
 
     # Iterating over files to generate reports
     for file in array_of_files:
 
-        # Reading data from results file in dataframes and formatting
-        metadata_df = pd.read_excel(file, sheet_name="Sample info")
-        raw_results_df = pd.read_excel(file, sheet_name="Gene filtered results")
-        raw_results_df.rename(columns={raw_results_df.columns[-1]:"dosage quotient"}, inplace=True)
-
-        # controlling dataypes
-        probe_df["start position"] = probe_df["start position"].astype(int)
-        probe_df["end position"] = probe_df["end position"].astype(int)
-
-        # creating results_clinical dataframe
-        results_clinical_df = clinical_df(
-            raw_results_df,
-            probe_df,
-            config.COLUMNS_RESULTS_DF,
-            config.COLUMNS_PROBE_DF
-        )
-
-        # get list of genes in panel
-        gene_list = results_clinical_df["gene"].unique()
-
-        # perform copy number analysis
-        cnv_summary_df, marked_df = copy_number_analysis(results_clinical_df, gene_list)
-        summary_html = df_to_html(cnv_summary_df)
-
-        # generating summary dataframe with limited columns
-        summary_data_df = summary_data(marked_df)
-
-        # creating empty dictionaries
-        summary_tables = {}
-        cnv_detail = {}
-
-        for gene in gene_list:
-            # creating cnv summary data by gene
-            summary_data_gene_df =  summary_data_df[summary_data_df["gene"] == gene]
-            summary_data_gene_df.columns = summary_data_gene_df.columns.str.replace("_", " ").str.title()
-            summary_tables[gene] = df_to_html(summary_data_gene_df)
-
-            if not cnv_summary_df.empty:
-                # generating details of cnv by impacted gene if cnv's are detected
-                impacted_genes = cnv_summary_df["Gene"].unique()
-                if gene in impacted_genes:
-                    # plotting gene data
-                    results_gene_df = marked_df[marked_df["gene"] == gene]
-                    if gene == "PMS2":
-                        gene_plot = pms2_plot(results_gene_df, gene)
-                    else:
-                        gene_plot = cnv_plot(results_gene_df, gene)
-
-                    # creating table of probes involved in CNV
-                    impacted_probes_df = summary_data_gene_df.dropna(subset=["Variant"])
-                    impacted_probes_df = summary_data_gene_df.loc[summary_data_gene_df["Variant"] != ""]
-                    impacted_probes_df = (impacted_probes_df.sort_values(by="Start Position"))
-                    impacted_probes_html = df_to_html(impacted_probes_df)
-
-                    # saving to cnv_detail dict.
-                    cnv_detail[gene] = [gene_plot, impacted_probes_html]
-
-        # getting panel number and qc data from metadata
-        pan_number, r_code = get_test_codes(metadata_df)
-        qc_status = get_qc_status(metadata_df)
-        qc_data = df_to_html(metadata_df)
-
         # Getting input file basenames
         file_name = os.path.basename(file)
         probe_file = os.path.basename(single_file)
 
-        # Creating ouput file name and path
-        # Input file name of results file is determined in GitHub repository:
-        # https://github.com/moka-guys/digital_mlpa
-        report = file_name.replace("_filtered_results.xlsx", ".pdf") \
-                        .replace("_filtered_results.xls", ".pdf")
-        report_path = os.path.join(output_directory, report)
-
-        # Rendering template and saving as pdf
-
-        env = Environment(loader=PackageLoader("dMLPA-flask", "templates"))
-        template = env.get_template("report.html")
-        data_values = {
-            "app_version": config.DMLPA_FLASK_VERSION,
-            "production": config.RELEASED_TO_PRODUCTION,
-            "file_name": report,
-            "r_code": r_code,
-            "pan_number": pan_number,
-            "genes": gene_list,
-            "run_id": "", # TODO to be added once file name is confirmed, 
-            "genome_build": config.GENOME_BUILD,
-            "patient_file": file_name,
-            "probe_file": probe_file,
-            "qc_status": qc_status,
-            "cnv_summary": summary_html,
-            "cnc_comment": config.CNC_CATEGORIES_DISCLAIMER,
-            "pms2_comment": config.PMS2_DISCLAIMER,
-            "cnv_detail": cnv_detail,
-            "report_notes": config.REPORTING_NOTE,
-            "qc_table": qc_data,
-            "summary_tables": summary_tables
-        }
-
-        html_report = template.render(data_values)
-
-        pdfkit.from_string(html_report, report_path)
-
-        # Add report to list of files to be zipped
-        files_to_zip.append(report_path)
-
-    # Prepare the files to be zipped
-    # files_to_zip: list[str] = [single_file] + array_of_files
-
-    # USER WARNINGS: Anything added to the list 'input_errors' will be passed from the backend to
-    # the frontend and displayed in a warning to the user. Used to give immediate feedback to the
-    # user about errors such as incorrect formatting of input files etc.
-    input_errors: list[str] = []
-    input_ok_flag: bool = True
-
-    # TODO: Add any relevant data validation here.  Currently only checks if the file has "error_test"
-    # in the name, and if so, sends warning to frontend.  This is just a placeholder for now.
-    for file_name in files_to_zip:
-        if "error_test" in file_name:
-            input_errors, input_ok_flag = [
-                "TEST ERROR: File passed to trigger this warning for testing purposes"
-            ], False
-            break
+        file_check = re.search(config.FILE_NAME_CONVENTION, file_name)
+
+        if not file_check:
+            unprocessed_results_files.append(file_name)
+            results_file_ok_flag = False
+            continue
+
+        else:
+            # Reading data from results file in dataframes and formatting
+            metadata_df = pd.read_excel(file, sheet_name="Sample info")
+            raw_results_df = pd.read_excel(file, sheet_name="Gene filtered results")
+            raw_results_df.rename(columns={raw_results_df.columns[-1]:"dosage quotient"}, inplace=True)
+
+            # controlling dataypes
+            probe_df["start position"] = probe_df["start position"].astype(int)
+            probe_df["end position"] = probe_df["end position"].astype(int)
+
+            # creating results_clinical dataframe
+            results_clinical_df = clinical_df(
+                raw_results_df,
+                probe_df,
+                config.COLUMNS_RESULTS_DF,
+                config.COLUMNS_PROBE_DF
+            )
+
+            # check certain columns for any NaN entries
+            # This would indicated that for some probes in the results file
+            # the matching probe data couldn't be found in the probe file
+            columns_to_check = config.NEW_PROBE_DF_HEADERS
+            columns_with_nan = results_clinical_df[columns_to_check].columns[results_clinical_df[columns_to_check].isna().any()]
+            if not columns_with_nan.empty:
+                probe_file_ok_flag = False
+                probe_errors = (
+                    "Probe File Error: Required probe information for some probes "
+                    "could not be found in the probe information file provided. "
+                    "Confirm that an appropriate probe file was provided for analysis"
+                )
+                return zip_name, results_file_ok_flag, probe_file_ok_flag, probe_errors, unprocessed_results_files
+
+            # get list of genes in panel
+            gene_list = results_clinical_df["gene"].unique()
+
+            # perform copy number analysis
+            cnv_summary_df, marked_df = copy_number_analysis(results_clinical_df, gene_list)
+            summary_html = df_to_html(cnv_summary_df)
+
+            # generating summary dataframe with limited columns
+            summary_data_df = summary_data(marked_df)
+
+            # creating empty dictionaries
+            summary_tables = {}
+            cnv_detail = {}
+
+            for gene in gene_list:
+                # creating cnv summary data by gene
+                summary_data_gene_df =  summary_data_df[summary_data_df["gene"] == gene]
+                summary_data_gene_df.columns = summary_data_gene_df.columns.str.replace("_", " ").str.title()
+                summary_tables[gene] = df_to_html(summary_data_gene_df)
+
+                if not cnv_summary_df.empty:
+                    # generating details of cnv by impacted gene if cnv's are detected
+                    impacted_genes = cnv_summary_df["Gene"].unique()
+                    if gene in impacted_genes:
+                        # plotting gene data
+                        results_gene_df = marked_df[marked_df["gene"] == gene]
+                        if gene == "PMS2":
+                            gene_plot = pms2_plot(results_gene_df, gene)
+                        else:
+                            gene_plot = cnv_plot(results_gene_df, gene)
+
+                        # creating table of probes involved in CNV
+                        # impacted_probes_df = summary_data_gene_df.dropna(subset=["Variant"])
+                        impacted_probes_df = summary_data_gene_df.loc[summary_data_gene_df["Variant"] != ""]
+                        impacted_probes_df = (impacted_probes_df.sort_values(by="Start Position"))
+                        impacted_probes_html = df_to_html(impacted_probes_df)
+
+                        # saving to cnv_detail dict.
+                        cnv_detail[gene] = [gene_plot, impacted_probes_html]
+
+            # getting panel number and qc data from metadata
+            pan_number, r_code = get_test_codes(metadata_df)
+            qc_status = get_qc_status(metadata_df)
+            qc_data = df_to_html(metadata_df)
+
+            # Creating ouput file name and path
+            # Input file name of results file is determined in GitHub repository:
+            # https://github.com/moka-guys/digital_mlpa
+            report = file_name.replace("_filtered_results.xlsx", ".pdf") \
+                            .replace("_filtered_results.xls", ".pdf")
+            report_path = os.path.join(output_directory, report)
+
+            # Rendering template and saving as pdf
+            env = Environment(loader=PackageLoader("dMLPA-flask", "templates"))
+            template = env.get_template("report.html")
+            data_values = {
+                "app_version": config.DMLPA_FLASK_VERSION,
+                "production": config.RELEASED_TO_PRODUCTION,
+                "file_name": report,
+                "r_code": r_code,
+                "pan_number": pan_number,
+                "genes": gene_list,
+                "run_id": "", # TODO to be added once file name is confirmed,
+                "genome_build": config.GENOME_BUILD,
+                "patient_file": file_name,
+                "probe_file": probe_file,
+                "qc_status": qc_status,
+                "cnv_summary": summary_html,
+                "cnc_comment": config.CNC_CATEGORIES_DISCLAIMER,
+                "pms2_comment": config.PMS2_DISCLAIMER,
+                "cnv_detail": cnv_detail,
+                "report_notes": config.REPORTING_NOTE,
+                "qc_table": qc_data,
+                "summary_tables": summary_tables
+            }
+
+            html_report = template.render(data_values)
+
+            pdfkit.from_string(html_report, report_path)
+
+            # Add report to list of files to be zipped
+            files_to_zip.append(report_path)
 
     # TODO: Add any relevant data processing here.  Currently just copies the input files to the
     # output directory.  This is just a placeholder for now.
@@ -175,4 +199,4 @@ def run_backend(single_file, array_of_files, output_directory):
                     f"File {file} does not exist."
                 )  # Consider replacing with proper logging
 
-    return zip_name, input_errors, input_ok_flag
+    return zip_name, results_file_ok_flag, probe_file_ok_flag, probe_errors, unprocessed_results_files
diff --git a/dMLPA-flask/cna.py b/dMLPA-flask/cna.py
index 6b56029..8bea129 100644
--- a/dMLPA-flask/cna.py
+++ b/dMLPA-flask/cna.py
@@ -7,8 +7,6 @@
 data), manipulating the relevant dataframes, creating copy number
 summary and detailed copy number reports.
 
-Functions:
-- TODO UPDATE
 """
 import io
 import base64
diff --git a/dMLPA-flask/config.py b/dMLPA-flask/config.py
index d08633d..65be8dd 100644
--- a/dMLPA-flask/config.py
+++ b/dMLPA-flask/config.py
@@ -9,8 +9,15 @@
 
 RELEASED_TO_PRODUCTION = False
 
+''' ==================== app.py constants ==================== '''
+
+PROBEFILE_NAME_CONVENTION = "PIF_D001-[a-zA-Z][0-9]" #TODO to be confirmed.
+
 ''' ==================== backend.py constants ==================== '''
 
+# TODO to be confirmed
+FILE_NAME_CONVENTION = "BP01-[0-9]+-dMLPA_R[0-9]+v[0-9]_Pan[0-9]+_[0-9]+_[a-zA-Z]+_[a-zA-Z]+[0-9]+_filtered_results.xlsx"
+
 GENOME_BUILD = "GRCh38"
 
 NEW_PROBE_DF_HEADERS = [
@@ -33,8 +40,8 @@
 REPORTING_NOTE = {
     # Generic message
     "Ref to Scientist": (
-        "Ref to Scientist: The dosage quotient for one or "\
-        "more probes falls outside the typical ranges used "\
+        "Ref to Scientist: The dosage quotient for one or "
+        "more probes falls outside the typical ranges used "
         "to evaluate copy number. Scientist review required"
     ),
 }
@@ -70,24 +77,24 @@
 
 CNC_CATEGORIES = {
     # TODO How does code handle MDQ that is 3 dp in the excel
-    "Ref to Scientist 1*": [0, 0.39],
-    "HetDel": [0.4, 0.6],  # theoretical value 0.5, represents CN 1
-    "Ref to Scientist 2*": [0.61, 0.79],
-    "Normal": [0.8, 1.2],  # theoretical value 1, represents CN 2
-    "Ref to Scientist 3*": [1.21, 1.39],
-    "HetDup": [1.4, 1.6],  # theoretical value 1.5, represents CN 3
-    "Ref to Scientist 4*": [1.61, float("Inf")]
+    "Ref to Scientist 1*": [0, 0.399],
+    "HetDel": [0.400, 0.600],  # theoretical value 0.5, represents CN 1
+    "Ref to Scientist 2*": [0.601, 0.799],
+    "Normal": [0.800, 1.200],  # theoretical value 1, represents CN 2
+    "Ref to Scientist 3*": [1.201, 1.399],
+    "HetDup": [1.400, 1.600],  # theoretical value 1.5, represents CN 3
+    "Ref to Scientist 4*": [1.601, float("Inf")]
 }
 
 PMS2_CATEGORIES = {
     # TODO Stake holders to confirm thresholds for this
-    "Ref to Scientist 1*": [0, 0.39],
-    "Potential PMS2 CN loss": [0.4, 0.84],
-    "Ref to Scientist 2*": [0.85, 0.85],
-    "Normal": [0.86, 1.14],  # theoretical value 1
-    "Ref to Scientist 3*": [1.15, 1.15],
-    "Potential PMS2 CN gain": [1.16, 1.6],
-    "Ref to Scientist 4*": [1.61, float("Inf")]
+    "Ref to Scientist 1*": [0, 0.399],
+    "Potential PMS2 CN loss": [0.40, 0.840],
+    "Ref to Scientist 2*": [0.841, 0.859],
+    "Normal": [0.860, 1.140],  # theoretical value 1
+    "Ref to Scientist 3*": [1.141, 1.159],
+    "Potential PMS2 CN gain": [1.160, 1.600],
+    "Ref to Scientist 4*": [1.601, float("Inf")]
 }
 
 PMS2_GRAPH_CATEGORIES = {
diff --git a/dMLPA-flask/qc_data.py b/dMLPA-flask/qc_data.py
index 3507b7c..201a4f1 100644
--- a/dMLPA-flask/qc_data.py
+++ b/dMLPA-flask/qc_data.py
@@ -73,7 +73,6 @@ def get_qc_status(df):
 
 def df_to_html(df):
     """Convert dataframe table to html without index"""
-    #TODO Format table style Including getting rid of row index
 
     #Saving dataframe to html format
     html_format = df.to_html(classes="table", index=False)
diff --git a/dMLPA-flask/templates/index.html b/dMLPA-flask/templates/index.html
index a6481f4..805f128 100644
--- a/dMLPA-flask/templates/index.html
+++ b/dMLPA-flask/templates/index.html
@@ -29,102 +29,109 @@ <h1>dMLPA Results Generator</h1>
 
     <div class="container">
 
-        {% if errors %}
-        <div class="alert alert-danger" role="alert">
-            <h4 class="alert-heading">Spreadsheet Errors:</h4>
-            <ul>
-                {% for error in errors %}
-                <li> {{ error }} </li>
-                {% endfor %}
-            </ul>
-        </div>
+        <!-- if some results files are not processed -->
+        {% if results_flag == False %}
+            <div class="alert alert-danger" role="alert">
+                <h4 class="alert-heading">File Errors:</h4>
+                <ul>
+                    <!-- {% if not results_flag %} -->
+                        <p>The following files do not meet the expected naming convention for patient files and could not be processed:</p>
+                    <!-- {% endif %} -->
+                    {% for error in errors %}
+                        <li> {{ error }} </li>
+                    {% endfor %} <!--TODO Confirm naming convention-->
+                    <br>
+                    <p>Expected naming convention for results files: "dMLPA_Rcode_PanNumber_Sample/TestID_Surname_Name"</p>
+                </ul>
+            </div>
         {% endif %}
 
-        {% if file_errors %}
-        <div class="alert alert-warning" role="alert">
-            <h4 class="alert-heading">File Errors:</h4>
-            <ul>
-                {% for field, errors in file_errors.items() %}
-                {% for error in errors %}
-                <li> {{ error }} </li>
-                {% endfor %}
-                {% endfor %}
-            </ul>
-        </div>
+        <!-- if FlaskForm validation raises validation errors -->
+        {% if validation_errors %}
+            <div class="alert alert-warning" role="alert">
+                <h4 class="alert-heading">File Error:</h4>
+                <ul>
+                    {% for field, errors in validation_errors.items() %}
+                        {% for error in errors %}
+                        <li> {{ error }} </li>
+                        {% endfor %}
+                    {% endfor %}
+                </ul>
+            </div>
         {% endif %}
 
         {% if backend_state == "initial" %}
-        <div class="alert alert-dark" role="alert">
-            <h4 class="alert-heading">Welcome to dMLPA Results Generator</h4>
-            <p> This web-interface can be used to generate a pdf report of the results obtained from
-                dMLPA analysis. <br>
-                To do this the following 2 inputs are required:<br>
-                <ul>
-                    <li>Probe Information File (xls/xlsx)- This is an excel document containing
-                        information on the genomic location of all probes present in the digitalMLPA
-                        probemix.</li>
-                    <li>Patient Files (xls/xlsx) - These are inidividual excel files generated
-                        by the digitalMLPA filtering script</li>
-                    </ul>
-                Please select the appropriate files below and then click the "Run dMLPA-flask" button.</p>
-        </div>
-        <div class="container">
-            <h4>File Upload</h4>
-            <form action=" {{ url_for('dMLPA-flask.form') }}  " method="post" enctype="multipart/form-data">
-                {{ form.hidden_tag() }}
-                <div class="row">
-                    <div class="col-sm-6 form-group">
-                        {{ form.single_file.label }} <br>
-                        {{ form.single_file(class="form-control-file") }}
-                    </div>
-                    <div class="col-sm-6 form-group">
-                        {{ form.multiple_array_of_files.label }} <br>
-                        {{ form.multiple_array_of_files(class="form-control-file", multiple=true) }}
+            <div class="alert alert-dark" role="alert">
+                <h4 class="alert-heading">Welcome to dMLPA Results Generator</h4>
+                <p> This web-interface can be used to generate a pdf report of the results obtained from
+                    dMLPA analysis. <br>
+                    To do this the following 2 inputs are required:<br>
+                    <ul>
+                        <li>Probe Information File (xls/xlsx)- This is an excel document containing
+                            information on the genomic location of all probes present in the digitalMLPA
+                            probemix.<br>
+                            Expected naming convention for this file: Name beginning with "PIF_D001"
+                        </li> <!--TODO Confirm naming convention-->
+                        <li>Patient Files (xls/xlsx) - These are inidividual excel files generated
+                            by the digitalMLPA filtering script</li>
+                            Expected naming convention for this file: "dMLPA_Rcode_PanNumber_Sample/TestID_Surname_Name"
+                        </ul> <!--TODO Confirm naming convention-->
+                    Please select the appropriate files below and then click the "Run dMLPA-flask" button.</p>
+            </div>
+
+            <div class="container">
+                <h4>File Upload</h4>
+                <form action=" {{ url_for('dMLPA-flask.form') }}  " method="post" enctype="multipart/form-data">
+                    {{ form.hidden_tag() }}
+                    <div class="row">
+                        <div class="col-sm-6 form-group">
+                            {{ form.single_file.label }} <br>
+                            {{ form.single_file(class="form-control-file") }}
+                        </div>
+                        <div class="col-sm-6 form-group">
+                            {{ form.multiple_array_of_files.label }} <br>
+                            {{ form.multiple_array_of_files(class="form-control-file", multiple=true) }}
+                        </div>
                     </div>
-                </div>
-                {{ form.submit(class="btn btn-primary btn-lg btn-block") }}
-            </form>
-        </div>
-
-        {%elif backend_state == "started" %}
-        <div class="alert alert-success" role="alert">
-            <h4 class="alert-heading">dMLPA Analysis Complete</h4>
-            <p>You submitted the following probe file: </p>
-            <p style="color:blue;"> {{ single_file_name }} </p>
-            <p>You submitted the following patient files:</p>
-            <p style="color:blue;"> {{ multiple_array_of_files_names }} </p>
-            <hr>
-            <h4>To download the pdf reports to your computer clink the download button below.</h4>
-            <p style="color:blue;"> {{ report_name }} </p>
-            <br>
-            <a class="btn btn-success" href=" {{url_for('dMLPA-flask.download')}}  ">Download</a>
-            <hr>
-            <p class="mb-0">If you have any issues with this tool please raise a ticket with Genome Support.</p>
-        </div>
-    </div>
-
-    {%elif backend_state == "finished" %}
-
-    <div class="alert alert-success" role="alert">
-        <h4 class="alert-heading">Analysis Completed!</h4>
-        <p>The report can be downloaded to your computer using the button below.</p>
-        <br>
-        <a class="btn btn-success" href=" {{url_for('dMLPA-flask.download')}}  ">Download
-            Report</a>
-        <hr>
-        <p class="mb-0">If you have any issues with this tool please raise a ticket with Genome Support.</p>
-    </div>
-
-    {%elif backend_state == "failed" %}
-
-    <div class="alert alert-danger" role="alert">
-        <h4 class="alert-heading">WARNING!</h4>
-        <p>Backend has failed with the following error:</p>
-        <hr>
-        <p class="mb-0"> ERROR: <br> If you require any help please raise a ticket with Genome Support.
-        </p>
-    </div>
-    {%endif %}
+                    {{ form.submit(class="btn btn-primary btn-lg btn-block") }}
+                </form>
+            </div>
+
+        {%elif backend_state == "finished" %}
+            <div class="alert alert-success" role="alert">
+                <h4 class="alert-heading">dMLPA Analysis Complete</h4>
+                <br>
+                <h5><b>{{files_processed}}/{{files_submitted}}</b> files successfully processed</h5>
+                <br>
+                <p>You submitted the following probe file: </p>
+                <p style="color:blue;"> {{ single_file_name }} </p>
+                <p>You submitted {{files_submitted}} patient file(s):</p>
+                    {% for file_name in multiple_array_of_files_names %}
+                    <li style="color:blue;"> {{file_name}} </li>
+                    {% endfor %}
+                <hr>
+                <h4>To download the pdf reports to your computer clink the download button below.</h4>
+                <p style="color:blue;"> {{ report_name }} </p>
+                <br>
+                <a class="btn btn-success" href=" {{url_for('dMLPA-flask.download')}}  ">Download</a>
+                <hr>
+                <p class="mb-0">If you have any issues with this tool please contact Synnovis bioinformatics support.</p>
+            </div>
+        <!-- </div> -->
+
+        {%elif backend_state == "failed" %}
+
+            <div class="alert alert-danger" role="alert">
+                <h4 class="alert-heading">WARNING! Analysis Failed!</h4>
+                <p>Backend has failed with the following error:</p>
+                <p><b>{{ probe_errors }}</b></p>
+                    <p>You submitted the following probe file: </p>
+                    <p style="color:blue;"> {{ single_file_name }} </p>
+                <hr>
+                <p>If you have any issues with this tool please contact Synnovis bioinformatics support.</p>
+            </div>
+
+        {%endif %}
     </div>
 
     {% block scripts %}
@@ -149,15 +156,13 @@ <h4 class="alert-heading">WARNING!</h4>
         }
 
         document.getElementById("single_file").addEventListener("change", function () {
-            validateFileType(this, ["xlsm", "xlsx", "xls"]); // TODO Add more file extensions here i.e. ["xlsm", "xlsx"]
+            validateFileType(this, ["xlsm", "xlsx", "xls"]);
         });
 
         document.getElementById("multiple_array_of_files").addEventListener("change", function () {
-            validateFileType(this, ["xlsm", "xlsx", "xls"]); // TODO Add more file extensions here i.e. ["xlsm", "xlsx"]
+            validateFileType(this, ["xlsm", "xlsx", "xls"]);
         });
     </script>
     {% endblock %}
 
-
-
 </body>
\ No newline at end of file
