Merge pull request #23 from IBM/8-generate-web-application

8 generate web application

CONTRIBUTING.md

@@ -83,6 +83,14 @@ When the window appears, add the following commandline to **run parameters**
 Change the second parameter to the directory where you want the output report be created.
 2. Right click again on **javacore_analyzer.py** and select **Run** or **Debug**.
 
+To run web application:
+1. Right click on **javacore_analyzer_web.py** directory in **Project** view and select **Modify Run Configuration...**.
+2. Add the following **Environmental variables:**
+   * **DEBUG:TRUE**
+   * **REPORTS_DIR:/tmp/web_reports**  
+   You can change the report dir to the location when you want to store the report. 
+   The application will start on http://localhost:5000
+
 ## Testing
 As default the tests in Pycharm are ran in the current selected directory. However we want to run them in main 
 directory of the tool (**javacore-analyser** directory, not **test** directory). 

README.md

@@ -27,16 +27,29 @@ Steps:
    `pip install -r requirements.txt`
 
 ### Running the tool:
+
+#### Running cmd application: 
 1. Activate your created virtual environment according to activate Virtual Environment according to [Creating virtual environments](https://docs.python.org/3/tutorial/venv.html#creating-virtual-environments)
-2. Navigate in command line to unpacked tool and run the following command:  
+2. Install the requirements using pip: 
+ `pip install requirements.txt` 
+3. Navigate in command line to unpacked tool and run the following command:  
   `python javacore_analyzer.py <input-data> <output-dir>`  
 Where `<input-data>` is one of the following:
 * The directory containing javacores and optionally verbose gc
-* Archive (7z, zip, tar.gz, tar.bz2) containint the same
+* Archive (7z, zip, tar.gz, tar.bz2) containing the same
 * List of the javacores separated by `'` character. Optionally you can add `--separator` option to define your own separator.
 You can type the following command to obtain the help:  
 `python javacore_analyzer.py --help`
 
+#### Running web application:
+1. Repeat steps 1 and 2 from cmd application
+2. Navigate to unpacked tool and run the following commands: 
+  ```
+  export REPORTS_DIR=/tmp/reports  
+  flask --app javacore_analyser_web run
+  ``` 
+ The first command sets the location of the generated reports stored.  
+ The second command starts the server which can be accessed by navigating to http://localhost:5000
 
 <!-- The following are OPTIONAL, but strongly suggested to have in your repository. -->
 <!--

constants.py

@@ -33,3 +33,7 @@
 MIN_JAVACORE_SIZE = 5 * 1024 # Minimal Javacore size in bytes
 
 DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+# Web application constants
+DEFAULT_REPORTS_DIR = "reports"
+DEFAULT_PORT = 5000

javacore_analyser_web.py

@@ -0,0 +1,118 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: Apache-2.0
+#
+import locale
+import logging
+import os
+import re
+import shutil
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+from flask import Flask, render_template, request, send_from_directory, redirect
+
+import javacore_analyzer
+import logging_utils
+from constants import DEFAULT_REPORTS_DIR, DEFAULT_PORT
+
+"""
+To run the application from cmd type:
+export REPORTS_DIR=/tmp/reports
+flask --app javacore_analyser_web run
+"""
+app = Flask(__name__)
+with app.app_context():
+    logging_utils.create_console_logging()
+    logging.info("Javacore analyser")
+    logging.info("Python version: " + sys.version)
+    logging.info("Preferred encoding: " + locale.getpreferredencoding())
+    reports_dir = os.getenv("REPORTS_DIR", DEFAULT_REPORTS_DIR)
+    logging.info("Reports directory: " + reports_dir)
+    logging_utils.create_file_logging(reports_dir)
+
+
+@app.route('/')
+def index():
+    reports = [{"name": Path(f).name, "date": time.ctime(os.path.getctime(f)), "timestamp": os.path.getctime(f)}
+               for f in os.scandir(reports_dir) if f.is_dir()]
+    reports.sort(key=lambda item: item["timestamp"], reverse=True)
+    return render_template('index.html', reports=reports)
+
+
+@app.route('/reports/<path:path>')
+def dir_listing(path):
+    return send_from_directory(reports_dir, path)
+
+
+@app.route('/zip/<path:path>')
+def compress(path):
+    try:
+        temp_zip_dir = tempfile.TemporaryDirectory()
+        temp_zip_dir_name = temp_zip_dir.name
+        zip_filename = path + ".zip"
+        report_location = os.path.join(reports_dir, path)
+        shutil.make_archive(os.path.join(temp_zip_dir_name, path), 'zip', report_location)
+        logging.debug("Generated zip file location:" + os.path.join(temp_zip_dir_name, zip_filename))
+        logging.debug("Temp zip dir name: " + temp_zip_dir_name)
+        logging.debug("Zip filename: " + zip_filename)
+        return send_from_directory(temp_zip_dir_name, zip_filename, as_attachment=True)
+    finally:
+        temp_zip_dir.cleanup()
+
+
+@app.route('/delete/<path:path>')
+def delete(path):
+    # Checking if the report exists. This is to prevent attempt to delete any data by deleting any file outside
+    # report dir if you prepare path variable.
+    reports_list = os.listdir(reports_dir)
+    report_location = os.path.normpath(os.path.join(reports_dir, path))
+    if not report_location.startswith(reports_dir):
+        logging.error("Deleted report in report list. Not deleting")
+        return "Cannot delete the report.", 503
+    shutil.rmtree(report_location)
+
+    return redirect("/")
+
+
+# Assisted by WCA@IBM
+# Latest GenAI contribution: ibm/granite-20b-code-instruct-v2
+@app.route('/upload', methods=['POST'])
+def upload_file():
+    try:
+        # Create a temporary directory to store uploaded files
+        javacores_temp_dir = tempfile.TemporaryDirectory()
+        javacores_temp_dir_name = javacores_temp_dir.name
+
+        # Get the list of files from webpage
+        files = request.files.getlist("files")
+
+        input_files = []
+        # Iterate for each file in the files List, and Save them
+        for file in files:
+            file_name = os.path.join(javacores_temp_dir_name, file.filename)
+            file.save(file_name)
+            input_files.append(file_name)
+
+        report_name = request.values.get("report_name")
+        report_name = re.sub(r'[^a-zA-Z0-9]', '_', report_name)
+
+        # Process the uploaded file
+        report_output_dir = reports_dir + '/' + report_name
+        javacore_analyzer.process_javacores_and_generate_report_data(input_files, report_output_dir)
+
+        return redirect("/reports/" + report_name + "/index.html")
+    finally:
+        javacores_temp_dir.cleanup()
+
+
+if __name__ == '__main__':
+    """
+    The application passes the following environmental variables:
+    DEBUG (default: False) - defines if we should run an app in debug mode
+    PORT - application port
+    REPORTS_DIR - the directory when the reports are stored as default
+    """
+    app.run(debug=os.getenv("DEBUG", False), port=os.getenv("PORT", DEFAULT_PORT))

javacore_analyzer.py

@@ -14,27 +14,18 @@
 import tempfile
 import traceback
 import zipfile
-from pathlib import Path
 
 import py7zr
 
-from constants import DEFAULT_FILE_DELIMITER, DATA_OUTPUT_SUBDIR
+import logging_utils
+from constants import DEFAULT_FILE_DELIMITER
 from javacore_set import JavacoreSet
 
-LOGGING_FORMAT = '%(asctime)s [%(levelname)s][%(filename)s:%(lineno)s] %(message)s'
-
 SUPPORTED_ARCHIVES_FORMATS = {"zip", "gz", "tgz", "bz2", "lzma", "7z"}
 
 
 
 
-def create_file_logging(logging_file_dir):
-    logging_file = logging_file_dir + "/wait2-debug.log"
-    Path(logging_file_dir).mkdir(parents=True, exist_ok=True) # Sometimes the folder of logging might not exist
-    file_handler = logging.FileHandler(logging_file, mode='w')
-    file_handler.setLevel(logging.DEBUG)
-    file_handler.setFormatter(logging.Formatter(LOGGING_FORMAT))
-    logging.getLogger().addHandler(file_handler)
 
 def extract_archive(input_archive_filename, output_path):
     """
@@ -64,10 +55,9 @@ def extract_archive(input_archive_filename, output_path):
         file = py7zr.SevenZipFile(input_archive_filename)
         logging.info("Processing 7z file")
     else:
-        logging.error("The format of file is not supported. "
+        raise Exception("The format of file is not supported. "
                       "Currently we support only zip, tar.gz, tgz, tar.bz2 and 7z. "
-                      "Cannot proceed. Exiting")
-        exit(13)
+                      "Cannot proceed.")
 
     file.extractall(path=output_path)
     file.close()
@@ -76,11 +66,7 @@ def extract_archive(input_archive_filename, output_path):
 
 
 def main():
-    logging.getLogger().setLevel(logging.NOTSET)
-    console_handler = logging.StreamHandler(sys.stdout)
-    console_handler.setLevel(logging.INFO)
-    console_handler.setFormatter(logging.Formatter(LOGGING_FORMAT))
-    logging.getLogger().addHandler(console_handler)
+    logging_utils.create_console_logging()
     logging.info("Wait2 tool")
     logging.info("Python version: " + sys.version)
     logging.info("Preferred encoding: " + locale.getpreferredencoding())
@@ -101,7 +87,7 @@ def main():
     logging.info("Report directory: " + output_param)
 
     # Needs to be created once output file structure is ready.
-    create_file_logging(output_param)
+    logging_utils.create_file_logging(output_param)
 
     # Check whether as input we got list of files or single file
     # Semicolon is separation mark for list of input files
@@ -134,22 +120,23 @@ def generate_javecore_set_data(files):
     """
 
     # Location when we store extracted archive or copied javacores files
-    javacores_temp_dir = tempfile.TemporaryDirectory()
-    # It is strange but sometimes the temp directory contains the content from previous run
-    # javacores_temp_dir.cleanup()
-    javacores_temp_dir_name = javacores_temp_dir.name
-    for file in files:
-        # file = file.strip() # Remove leading or trailing space in file path
-        if os.path.isdir(file):
-            shutil.copytree(file, javacores_temp_dir_name, dirs_exist_ok=True)
-        else:
-            filename, extension = os.path.splitext(file)
-            extension = extension[1:]  # trim trailing "."
-            if extension.lower() in SUPPORTED_ARCHIVES_FORMATS:
-                extract_archive(file, javacores_temp_dir_name)  # Extract archive to temp dir
+    try:
+        javacores_temp_dir = tempfile.TemporaryDirectory()
+
+        javacores_temp_dir_name = javacores_temp_dir.name
+        for file in files:
+            if os.path.isdir(file):
+                shutil.copytree(file, javacores_temp_dir_name, dirs_exist_ok=True)
             else:
-                shutil.copy2(file, javacores_temp_dir_name)
-    return JavacoreSet.process_javacores(javacores_temp_dir_name)
+                filename, extension = os.path.splitext(file)
+                extension = extension[1:]  # trim trailing "."
+                if extension.lower() in SUPPORTED_ARCHIVES_FORMATS:
+                    extract_archive(file, javacores_temp_dir_name)  # Extract archive to temp dir
+                else:
+                    shutil.copy2(file, javacores_temp_dir_name)
+        return JavacoreSet.process_javacores(javacores_temp_dir_name)
+    finally:
+        javacores_temp_dir.cleanup()
 
 
 
@@ -170,6 +157,5 @@ def process_javacores_and_generate_report_data(input_files, output_dir):
     javacore_set.generate_report_files(output_dir)
 
 
-
 if __name__ == "__main__":
     main()

javacore_set.py

@@ -87,7 +87,6 @@ def process_javacores(input_path):
         jset.generate_tips()
         return jset
 
-
     # Assisted by WCA@IBM
     # Latest GenAI contribution: ibm/granite-8b-code-instruct
     def generate_report_files(self, output_dir):
@@ -288,7 +287,7 @@ def print_thread_states(self):
         for thread in self.threads:
             logging.debug("max running states:" + str(thread.get_continuous_running_states()))
             logging.debug(thread.name + "(id: " + str(thread.id) + "; hash: " + thread.get_hash() + ") " + \
-            "states: " + thread.get_snapshot_states())
+                          "states: " + thread.get_snapshot_states())
 
     # Assisted by WCA@IBM
     # Latest GenAI contribution: ibm/granite-8b-code-instruct

logging_utils.py

@@ -0,0 +1,27 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import logging
+import sys
+from pathlib import Path
+
+LOGGING_FORMAT = '%(asctime)s [thread: %(thread)d][%(levelname)s][%(filename)s:%(lineno)s] %(message)s'
+
+
+def create_file_logging(logging_file_dir):
+    logging_file = logging_file_dir + "/wait2-debug.log"
+    Path(logging_file_dir).mkdir(parents=True, exist_ok=True)  # Sometimes the folder of logging might not exist
+    file_handler = logging.FileHandler(logging_file, mode='w')
+    file_handler.setLevel(logging.DEBUG)
+    file_handler.setFormatter(logging.Formatter(LOGGING_FORMAT))
+    logging.getLogger().addHandler(file_handler)
+
+
+def create_console_logging():
+    logging.getLogger().setLevel(logging.NOTSET)
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(logging.INFO)
+    console_handler.setFormatter(logging.Formatter(LOGGING_FORMAT))
+    logging.getLogger().addHandler(console_handler)

requirements.txt

@@ -2,3 +2,4 @@ dicttoxml
 py7zr
 lxml
 pyana
+flask # WSGI server for development the code

templates/index.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<!--
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: Apache-2.0
+-->
+
+<!--
+// Assisted by WCA@IBM
+// Latest GenAI contribution: ibm/granite-20b-code-instruct-v2
+-->
+<html xmlns="http://www.w3.org/1999/html" xmlns="http://www.w3.org/1999/html">
+  <head>
+    <title>Javacore Analyser</title>
+  </head>
+  <body>
+    <h1>Javacore Analyser</h1>
+
+    <h2>Generate report:</h2>
+    <form action="/upload" method="post" enctype="multipart/form-data">
+      <br>Report Name: <input type="text" id="report_name" name="report_name" required></br>
+      <br>Archive file or multiple javacore files: <input type="file" name="files" multiple required> </br>
+      <strong>
+        NOTE: The report generation is expensive operation and might take even few minutes. Please be patient
+      </strong>
+      <br><input type="submit" value="Upload"></br>
+    </form>
+    <br></br>
+
+
+    <h2>List of generated reports:</h2>
+      <table>
+        <tr><th> Name </th><th> Creation Date </th><th> Download </th><th> Delete </th></tr>
+        {% for report in reports %}
+        {% set name = report['name'] %}
+        {% set date = report['date'] %}
+        <tr>
+          <td><a href="reports/{{ name }}/index.html">{{ name }}</a></td>
+          <td> {{ date }} </td>
+          <td><a href="zip/{{ name }}" > download </a></td>
+          <td>
+            <a href="delete/{{ name }}" onclick="return confirm('Do you want to delete report {{ name }}?')">
+              delete
+            </a>
+          </td>
+        </tr>
+        {% endfor %}
+      </table>
+  </body>
+</html>