Python rebuild of dMLPA-reporting

.vscode/launch.json

@@ -7,7 +7,7 @@
             "request": "attach",
             "connect": {
                 "host": "localhost",
-                "port": "5678"
+                "port": 5678
             },
             "pathMappings": [
                 {

Dockerfile.development

@@ -12,24 +12,31 @@ ENV FLASK_APP=/dMLPA-flask/app.py
 ENV PYTHONPATH=/dMLPA-flask/dMLPA-flask
 ENV SESSION_FILE_DIR /var/local/dMLPA-flask/flask_sessions
 ENV SCRIPT_NAME=dMLPA-flask
-ENV UPLOAD_FOLDER /var/local/dMLPA-flask/uploads
-ENV FLASK_DEBUG=1 
+ENV UPLOAD_FOLDER_1 /var/local/dMLPA-flask/probe_file
+ENV UPLOAD_FOLDER_2 /var/local/dMLPA-flask/inputs
+ENV OUTPUT_FOLDER /var/local/dMLPA-flask/outputs
+ENV FLASK_DEBUG=1
 # only for development
 
 # add and install requirements
 COPY ./requirements.txt .
 RUN pip3 install -r requirements.txt
-USER 0 
+USER 0
 RUN mkdir -p /var/local/dMLPA-flask/logs/
-RUN mkdir -p /var/local/dMLPA-flask/uploads/
+RUN mkdir -p /var/local/dMLPA-flask/probe_file
+RUN mkdir -p /var/local/dMLPA-flask/inputs
+RUN mkdir -p /var/local/dMLPA-flask/outputs
 RUN mkdir -p /var/local/dMLPA-flask/flask_sessions/
 RUN chmod 777 /var/local/dMLPA-flask/logs/
-RUN chmod 777 /var/local/dMLPA-flask/uploads/
+RUN chmod 777 /var/local/dMLPA-flask/probe_file
+RUN chmod 777 /var/local/dMLPA-flask/inputs
+RUN chmod 777 /var/local/dMLPA-flask/outputs
 RUN chmod 777 /var/local/dMLPA-flask/flask_sessions/
 
 USER $CONTAINER_USER_ID
 
 RUN pip install debugpy
+RUN apt-get update && apt-get install -y wkhtmltopdf
 
 # add app
 COPY . .

dMLPA-flask/app.py

@@ -28,7 +28,8 @@
 from werkzeug.utils import secure_filename
 import zipfile
 
-
+import re
+import config
 import logging
 
 log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
@@ -39,15 +40,18 @@
 
 
 app = Flask(__name__)
-app.config["UPLOAD_FOLDER"] = os.environ["UPLOAD_FOLDER"]
+app.config["UPLOAD_FOLDER_1"] = os.environ["UPLOAD_FOLDER_1"]
+app.config["UPLOAD_FOLDER_2"] = os.environ["UPLOAD_FOLDER_2"]
+app.config["OUTPUT_FOLDER"] = os.environ["OUTPUT_FOLDER"]
 app.config["SECRET_KEY"] = "catchmeifyoucan"
 app.config["SESSION_TYPE"] = "filesystem"
 app.config["SESSION_PERMANENT"] = True
 app.config["SESSION_FILE_DIR"] = os.environ["SESSION_FILE_DIR"]
 app.config["UPLOAD_EXTENSIONS"] = [
     ".xlsx",
-    ".bam",
-]  # TODO: If multiple file types, add to list i.e. [".txt", ".csv", ".xlsm", ".xlsx"]
+    ".xlsm",
+    ".xls"
+]
 app.config["MAX_CONTENT_LENGTH"] = 2 * 1024 * 1024  # File has to be less than 2MB
 app.config["APPLICATION_ROOT"] = "/dMLPA-flask"
 app.config["WTF_CSRF_ENABLED"] = True
@@ -67,18 +71,28 @@ def validate_input_file(form, field):
         "xlsx",
         "xlsm",
         "xls",
-    ]  # TODO: If multiple file types, add to list i.e. ["xlsm", "xlsx"]
+    ]
     file = field.data
     if not file:
-        raise ValidationError("No file provided.")
+        raise ValidationError("No probe file provided.")
     if not file.filename:
-        raise ValidationError("No file selected.")
+        raise ValidationError("No probe file selected.")
     if not (
         "." in file.filename
         and file.filename.rsplit(".", 1)[1].lower() in allowed_extensions
     ):
         raise ValidationError(
-            f"Invalid file type for input file '{file.filename}'. Allowed types are: xlsx"  # TODO: Ammend file types as appropriate i.e. "xlsm, xlsx"
+            f"Invalid file type for probe file '{file.filename}'. "
+            "Allowed types are: xlsx, xlsm, xls"
+        )
+    file_check = re.search(config.PROBEFILE_NAME_CONVENTION, file.filename)
+    if not file_check:
+        raise ValidationError(
+            f"File {file.filename} is not a valid probe file. "
+            "File name does not match expected naming convention for a probe "
+            "file which may result in processing errors. Please provide an "
+            "appropriate probe file which matches the expected naming "
+            "convention."
         )
 
 
@@ -87,19 +101,20 @@ def validate_input_files_array(form, field):
         "xlsx",
         "xlsm",
         "xls",
-    ]  # TODO: Ammend file types as appropriate i.e. ["xlsm", "xlsx"]
+    ]
     files = field.data
     if not files:
-        raise ValidationError("No files provided.")
+        raise ValidationError("No results files provided.")
     for file in files:
         if not file.filename:
-            raise ValidationError("No files selected.")
+            raise ValidationError("No results files selected.")
         if not (
             "." in file.filename
             and file.filename.rsplit(".", 1)[1].lower() in allowed_extensions
         ):
             raise ValidationError(
-                f"Invalid file type for input files '{file.filename}'. Allowed types are: bam"  # TODO: Ammend file types as appropriate i.e. "xlsm, xlsx"
+                f"Invalid file type for results files '{file.filename}'."
+                "Allowed types are: xlsx, xlsm, xls"
             )
 
 
@@ -110,7 +125,7 @@ class ChangeForm(FlaskForm):
         render_kw={
             "accept": ".xlsm, .xlsx, .xls",
             "id": "single_file",
-        },  # TODO: If multiple file types, add to list i.e. ".xlsm, .xlsx"
+        },
     )
     multiple_array_of_files = MultipleFileField(
         "Single/Multiple Patient Data Files (Excel):",
@@ -119,7 +134,7 @@ class ChangeForm(FlaskForm):
             "accept": ".xlsm, .xlsx, .xls",
             "multiple": "True",
             "id": "multiple_array_of_files",
-        },  # TODO: If multiple file types, add to list i.e. ".xlsm, .xlsx"
+        },
     )
     submit = SubmitField("Run dMLPA-flask")
 
@@ -135,13 +150,15 @@ def form(backend_state="initial"):
             return render_template(
                 "index.html",
                 form=chgForm,
-                file_errors=chgForm.errors,
+                validation_errors=chgForm.errors,
             )
 
         # Use FileHandler() to log to a file
 
         session["timestr"] = datetime.now().strftime("%Y%m%d-%H%M%S")
-        os.mkdir(os.path.join(app.config["UPLOAD_FOLDER"], session["timestr"]))
+        os.mkdir(os.path.join(app.config["UPLOAD_FOLDER_1"], session["timestr"]))
+        os.mkdir(os.path.join(app.config["UPLOAD_FOLDER_2"], session["timestr"]))
+        os.mkdir(os.path.join(app.config["OUTPUT_FOLDER"], session["timestr"]))
         file_handler = logging.FileHandler(
             f"/var/local/dMLPA-flask/logs/dMLPA-flask_error.log"
         )
@@ -170,48 +187,61 @@ def form(backend_state="initial"):
 
         # Create the paths to the uploaded files
         input_single_file_tmp_path = os.path.join(
-            app.config["UPLOAD_FOLDER"], session["timestr"], input_single_file_basename
+            app.config["UPLOAD_FOLDER_1"], session["timestr"], input_single_file_basename
         )
 
         input_files_tmp_paths = [
-            os.path.join(app.config["UPLOAD_FOLDER"], session["timestr"], basename)
+            os.path.join(app.config["UPLOAD_FOLDER_2"], session["timestr"], basename)
             for basename in input_files_basenames
         ]
 
-        zipped_output, input_errors, input_ok_flag = run_backend(
+        # Create the path to the output folder
+        output_path = os.path.join(app.config["OUTPUT_FOLDER"], session["timestr"])
+
+        zipped_output, results_ok_flag, probe_ok_flag, probe_errors, unprocessed_files = run_backend(
             input_single_file_tmp_path,
             input_files_tmp_paths,
-            app.config["UPLOAD_FOLDER"],
+            output_path,
         )
+
+        # Establishing count of files
+        no_files_submitted = len(input_files_basenames)
+        no_files_processed = no_files_submitted - len(unprocessed_files)
+
+        # TODO add logging of any errors encountered with the probe or results flags
+
         # Save the zipped output path to the session
         session["zipped_output"] = zipped_output
 
-        if input_ok_flag == False:
+        if probe_ok_flag == False:
             return render_template(
                 "index.html",
                 form=chgForm,
-                backend_state="initial",
-                errors=input_errors,  # Pass the errors to the template
-                file_errors=chgForm.errors,
+                backend_state="failed",
+                single_file_name=input_file.filename,
+                validation_errors=chgForm.errors,
+                probe_errors=probe_errors,  # Pass the errors to the template
             )
         else:
             return render_template(
                 "index.html",
                 form=chgForm,
-                backend_state=backend_state,
+                backend_state="finished",
+                files_submitted = no_files_submitted,
+                files_processed = no_files_processed,
                 single_file_name=input_file.filename,
-                multiple_array_of_files_names=", ".join(
-                    [x.filename for x in input_files_array]
-                ),
+                multiple_array_of_files_names=input_files_basenames,
                 zipped_folder_name=zipped_output,
-                file_errors=chgForm.errors,
+                results_flag=results_ok_flag,
+                errors=unprocessed_files,
+                validation_errors=chgForm.errors,
             )
 
     return render_template(
         "index.html",
         form=chgForm,
         backend_state=backend_state,
-        file_errors=chgForm.errors,
+        validation_errors=chgForm.errors,
     )
 
 
@@ -225,15 +255,15 @@ def upload(self, file):
             secure_file_name = secure_filename(file_name)
             file.save(
                 os.path.join(
-                    app.config["UPLOAD_FOLDER"],
+                    app.config["UPLOAD_FOLDER_1"],
                     session["timestr"],
                     secure_file_name,
                 )
             )
 
             return str(
                 os.path.join(
-                    app.config["UPLOAD_FOLDER"],
+                    app.config["UPLOAD_FOLDER_1"],
                     session["timestr"],
                     secure_file_name,
                 )
@@ -257,15 +287,15 @@ def upload(self, files):
                 secure_file_name = secure_filename(file_name)
                 file.save(
                     os.path.join(
-                        app.config["UPLOAD_FOLDER"],
+                        app.config["UPLOAD_FOLDER_2"],
                         session["timestr"],
                         secure_file_name,
                     )
                 )
                 file_names.append(
                     str(
                         os.path.join(
-                            app.config["UPLOAD_FOLDER"],
+                            app.config["UPLOAD_FOLDER_2"],
                             session["timestr"],
                             secure_file_name,
                         )