main_system_gui.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebb21201",
   "metadata": {},
   "outputs": [],
   "source": [
    "#---------------------------------------------------------------------------------------------GENERAL-----------------------------------------------------------------------------------------------\n",
    "import torch\n",
    "import torchvision\n",
    "from torchvision.models.detection.faster_rcnn import FastRCNNPredictor\n",
    "from torchvision.datasets import VOCDetection\n",
    "from torchvision.transforms import ToTensor\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from torchvision.transforms import Compose, Resize, ToTensor\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "import torch.multiprocessing as mp\n",
    "from torchvision.transforms import functional as F\n",
    "from torchvision.models.detection.faster_rcnn import FasterRCNN_ResNet50_FPN_Weights\n",
    "from torch.cuda.amp import autocast, GradScaler\n",
    "from typing import Dict\n",
    "import numpy as np\n",
    "from torch.utils.data import Dataset\n",
    "import os\n",
    "from PIL import Image\n",
    "import collections\n",
    "from torchvision import transforms\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "import torch.nn as nn\n",
    "import torchviz\n",
    "from torchsummary import summary\n",
    "import re\n",
    "import cv2  \n",
    "import easyocr\n",
    "import math\n",
    "from datetime import datetime\n",
    "import _tkinter\n",
    "import tkinter as tk\n",
    "from tkinter import filedialog, messagebox\n",
    "from PIL import Image, ImageTk, ImageDraw, ImageFont, ImageSequence\n",
    "import cv2\n",
    "import numpy as np\n",
    "from mss import mss\n",
    "import threading\n",
    "import time\n",
    "import pyttsx3\n",
    "from docx import Document\n",
    "from scipy.interpolate import interp1d\n",
    "import pyautogui\n",
    "import gc\n",
    "\n",
    "#-------------------------------------------------------------------------------------JUPYTER NOTEBOOK SETTINGS-------------------------------------------------------------------------------------\n",
    "from IPython.core.display import display, HTML                                    \n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe569da2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the pretrained model\n",
    "model = torchvision.models.detection.fasterrcnn_resnet50_fpn(weights=FasterRCNN_ResNet50_FPN_Weights.DEFAULT)\n",
    "\n",
    "# Replace the classifier with a new one\n",
    "num_classes = 6\n",
    "in_features = model.roi_heads.box_predictor.cls_score.in_features\n",
    "\n",
    "model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n",
    "\n",
    "# Put model to device\n",
    "device = torch.device('cpu')\n",
    "model.to(device)\n",
    "\n",
    "# Load saved model\n",
    "model.load_state_dict(torch.load('ui/fasterrcnn/model.pt'))\n",
    "model.eval()\n",
    "\n",
    "# Data processing\n",
    "data_transforms = transforms.Compose([ \n",
    "    transforms.Resize(512),\n",
    "    transforms.ToTensor(), \n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0031a0b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# this function finds the average difference between all the elements in a list, then adds a new element at the end\n",
    "# equal to the last element minus the average\n",
    "def filter_coordinates(pixel_coordinates, threshold=100):\n",
    "    horizontal_lines = {}\n",
    "    vertical_lines = {}\n",
    "\n",
    "    # Check the shape of the input and reshape if needed\n",
    "    if pixel_coordinates.shape[1] == 1 and pixel_coordinates.shape[2] == 2:\n",
    "        pixel_coordinates = pixel_coordinates.reshape(-1, 2)\n",
    "\n",
    "    # Separate coordinates into horizontal and vertical lines\n",
    "    for coord in pixel_coordinates:\n",
    "        x, y = int(coord[0]), int(coord[1])\n",
    "        if y not in horizontal_lines:\n",
    "            horizontal_lines[y] = []\n",
    "        if x not in vertical_lines:\n",
    "            vertical_lines[x] = []\n",
    "        horizontal_lines[y].append(x)\n",
    "        vertical_lines[x].append(y)\n",
    "\n",
    "    # Filter horizontal lines longer than the threshold\n",
    "    for y, x_values in horizontal_lines.items():\n",
    "        if len(x_values) > threshold:\n",
    "            pixel_coordinates = pixel_coordinates[np.logical_not(np.isin(pixel_coordinates[:, 0], x_values)), :]\n",
    "\n",
    "    # Filter vertical lines longer than the threshold\n",
    "    for x, y_values in vertical_lines.items():\n",
    "        if len(y_values) > threshold:\n",
    "            pixel_coordinates = pixel_coordinates[np.logical_not(np.isin(pixel_coordinates[:, 1], y_values)), :]\n",
    "\n",
    "    return pixel_coordinates\n",
    "\n",
    "def remove_corner_pixels(coordinates, width, height):\n",
    "    # Define a function to count pixels near corners\n",
    "    def count_corner_pixels(coordinates, distance):\n",
    "        count = 0\n",
    "        for coord in coordinates:\n",
    "            x, y = coord[0], coord[1]\n",
    "            if ((x < distance or x >= width - distance) and (y < distance or y >= height - distance)):\n",
    "                count += 1\n",
    "        return count\n",
    "\n",
    "    # Start with an initial distance\n",
    "    distance = 10\n",
    "    corner_pixels_count = count_corner_pixels(coordinates, distance)\n",
    "\n",
    "    # Adjust the distance based on the count of pixels near corners\n",
    "    if corner_pixels_count != 0:\n",
    "        distance = 100 / corner_pixels_count\n",
    "\n",
    "    filtered_coords = []\n",
    "    for coord in coordinates:\n",
    "        x, y = coord[0], coord[1]\n",
    "        # Check if the pixel is in or within the calculated distance of the corners\n",
    "        if ((x < distance or x >= width - distance) and (y < distance or y >= height - distance)):\n",
    "            continue\n",
    "        filtered_coords.append(coord)\n",
    "        \n",
    "    return np.array(filtered_coords)\n",
    "\n",
    "def calculate_new_list(lst):\n",
    "    # Calculate the differences between successive elements\n",
    "    differences = [lst[i] - lst[i + 1] for i in range(len(lst) - 1)]\n",
    "    \n",
    "    # Calculate the average difference\n",
    "    avg_difference = sum(differences) / len(differences)\n",
    "    \n",
    "    # Add a new element at the end of the list\n",
    "    new_element = lst[-1] - avg_difference \n",
    "    lst.append(new_element)\n",
    "\n",
    "    return lst\n",
    "\n",
    "def extract_currency(string):\n",
    "    # Dictionary to map symbols to currency names\n",
    "    currency_map = {\n",
    "        'US': 'USD',\n",
    "        '$': 'USD',\n",
    "        '€': 'EUR',\n",
    "        '£': 'GBP',\n",
    "        '¥': 'JPY',\n",
    "        'A$': 'AUD',\n",
    "        'C$': 'CAD',\n",
    "        'S': 'USD',   # Misidentification for $\n",
    "        'E': 'EUR',   # Misidentification for €\n",
    "        'L': 'GBP',   # Misidentification for £\n",
    "        'Y': 'JPY',   # Potential misidentification for ¥\n",
    "    }\n",
    "    \n",
    "    # Build a pattern that checks for each symbol followed by optional whitespace and a digit or a period\n",
    "    pattern_elements = [re.escape(symbol) + r'\\s*[0-9\\.]' for symbol in currency_map.keys()]\n",
    "    pattern_string = '|'.join(pattern_elements) + r'|USD|EUR|GBP|JPY|AUD|CAD'\n",
    "    pattern = re.compile(pattern_string)\n",
    "    \n",
    "    match = pattern.search(string)\n",
    "    \n",
    "    if match:\n",
    "        symbol = match.group()[:-1].strip()  # Exclude the trailing digit or period and any whitespace from the match\n",
    "        return currency_map.get(symbol, symbol).lower()\n",
    "    else:\n",
    "        return None\n",
    "def remove_last_if_not_digit(s):\n",
    "    if s and not s[-1].isdigit():\n",
    "        return s[:-1]\n",
    "    return s\n",
    "\n",
    "def extract_percentage(string):\n",
    "    # Correct misidentifications within the string\n",
    "    corrected_string = correct_misidentifications(string)\n",
    "    # Find the percentage pattern in the corrected string\n",
    "    match = re.search(r'([-+]?\\d+(\\.\\d+)?)\\s*%', corrected_string)\n",
    "    if match:\n",
    "        raw_value = match.group(1)\n",
    "        raw_value = correct_decimal_point(raw_value)\n",
    "        raw_value = remove_last_if_not_digit(raw_value)\n",
    "        percentage = float(raw_value)\n",
    "        return percentage\n",
    "    return None\n",
    "\n",
    "def extract_bracketed_timeframe(string):\n",
    "    match = re.search(r'\\(([^)]+)\\)', string)\n",
    "    if match:\n",
    "        bracketed_string = match.group(1)\n",
    "        return bracketed_string.lower()\n",
    "    return None\n",
    "\n",
    "def correct_value(text):\n",
    "    if text and text[0].isdigit() and text[0] in ['S', '5']:\n",
    "        return '$' + text[1:]\n",
    "    return text\n",
    "\n",
    "def extract_float_value(input_string):\n",
    "    numbers = re.findall(r'\\d+\\.\\d+', input_string)                                      # Find all the floating-point numbers in the string using a regular expression\n",
    "    return float(numbers[0])                                                             # Return the first floating-point number found, as a float\n",
    "    \n",
    "def correct_misidentifications(text):\n",
    "    corrections = {\n",
    "        'O': '0',\n",
    "        'G': '6',\n",
    "        'B': '8',\n",
    "        'I': '1',\n",
    "        'S': '5',\n",
    "        'Z': '2',\n",
    "        'Q': '0',\n",
    "        'T': '7',\n",
    "        'L': '1',\n",
    "        'A': '4'\n",
    "    }\n",
    "    \n",
    "    corrected_text = text\n",
    "    for char, replacement in corrections.items():\n",
    "        corrected_text = corrected_text.replace(char, replacement)\n",
    "    \n",
    "    corrected_text = re.sub(r'[^\\d.]+', '', corrected_text)  # Keep only numeric characters and decimal points\n",
    "    return corrected_text\n",
    "\n",
    "def correct_decimal_point(value):\n",
    "    value_str = str(value)\n",
    "\n",
    "    # Check if the first digit is '0' and the second character is not '.'\n",
    "    if value_str[0] == '0' and (len(value_str) == 1 or value_str[1] != '.'):\n",
    "        value_str = '0.' + value_str[1:]\n",
    "\n",
    "    # If the original value was a number, return as a float; otherwise, return as a string\n",
    "    if isinstance(value, (int, float)):\n",
    "        return float(value_str)\n",
    "    return value_str\n",
    "\n",
    "def process_ocr_string(y_axis):\n",
    "    values = y_axis.split(',')\n",
    "    corrected_values = []\n",
    "    removed_indices = []\n",
    "    \n",
    "    for index, value in enumerate(values):\n",
    "        # If the value contains a space not at the beginning or end, replace with a decimal point\n",
    "        value = re.sub(r'(?<=\\S) (?=\\S)', '.', value)\n",
    "        \n",
    "        # If the value contains both digits and characters\n",
    "        if re.search(r'\\d', value) and re.search(r'[a-zA-Z]', value):\n",
    "            corrected_value = correct_misidentifications(value)\n",
    "        else:\n",
    "            corrected_value = value\n",
    "\n",
    "        # If the corrected value is still a valid float, add it to the results\n",
    "        try:\n",
    "            corrected_values.append(float(correct_decimal_point(corrected_value)))\n",
    "        except ValueError:\n",
    "            removed_indices.append(index)  # Save the index of the value that was removed\n",
    "\n",
    "    return corrected_values, set(removed_indices)\n",
    "\n",
    "def time_mapping(input_string):\n",
    "    mapping = {\n",
    "        '1h': '1 hour',\n",
    "        '4h': '4 hours',\n",
    "        '24h': '1 day',\n",
    "        '1d': '1 day',\n",
    "        '7d': '7 days',\n",
    "        '1w': '7 days',\n",
    "        '1m': '1 month',\n",
    "        '3m': '3 months',\n",
    "        '6m': '6 months',\n",
    "        '1y': '1 year',\n",
    "        'ytd': 'year to date',\n",
    "        'all': 'year to date'\n",
    "    }\n",
    "    return mapping.get(input_string, 'N/A')\n",
    "\n",
    "def clean_memory():\n",
    "    tensor = None\n",
    "    torch.cuda.empty_cache()\n",
    "    del tensor\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5eb7476",
   "metadata": {},
   "outputs": [],
   "source": [
    "def resize_frame(frame):\n",
    "    original_width, original_height = frame.size\n",
    "    if original_width < original_height:\n",
    "        new_width = 512\n",
    "        aspect_ratio = original_height / original_width\n",
    "        new_height = int(new_width * aspect_ratio)\n",
    "    else:\n",
    "        new_height = 512\n",
    "        aspect_ratio = original_width / original_height\n",
    "        new_width = int(new_height * aspect_ratio)\n",
    "    return frame.resize((new_width, new_height))\n",
    "\n",
    "def process_frame(frame_received, frame_type): \n",
    "    CUSTOM_CLASSES = {\"name\": 1, \"value\": 2, \"x-axis\": 3, \"y-axis\": 4, \"plot\":5}\n",
    "\n",
    "    img = frame_received.convert('RGB')                      # Ensure the image is in RGB mode\n",
    "    orig_width, orig_height = img.size                                   # Store the original image dimensions\n",
    "\n",
    "    # Process the image\n",
    "    img_processed = data_transforms(img)\n",
    "\n",
    "    # Calculate the scale ratios\n",
    "    new_width, new_height = img_processed.shape[1], img_processed.shape[2]\n",
    "\n",
    "    # Calculate the scale ratios\n",
    "    if orig_width <= orig_height:\n",
    "        new_height = 512\n",
    "        new_width = int(orig_width * (new_height / orig_height))\n",
    "    else:\n",
    "        new_width = 512\n",
    "        new_height = int(orig_height * (new_width / orig_width))\n",
    "\n",
    "    width_ratio = new_width / orig_width\n",
    "    height_ratio = new_height / orig_height\n",
    "\n",
    "    with torch.no_grad():\n",
    "        prediction = model([img_processed])\n",
    "\n",
    "    # Scale bounding boxes back to original image size\n",
    "    for box in prediction[0]['boxes']:\n",
    "        box[0] *= (1 / width_ratio)\n",
    "        box[2] *= (1 / width_ratio)\n",
    "        box[1] *= (1 / height_ratio)\n",
    "        box[3] *= (1 / height_ratio)\n",
    "\n",
    "    # Debugging\n",
    "    #print(prediction[0]['boxes'])\n",
    "    #print(prediction[0]['scores'])\n",
    "\n",
    "    results = {}                                 # Dictionary to store the results\n",
    "    yaxis_coordinates = []                       # Initialize a list to store the Y coordinates\n",
    "    label5_image = None                          # Variable to store the cropped image for label 5\n",
    "    y_axis_image = None\n",
    "\n",
    "    reader = easyocr.Reader(lang_list=['en'])       # Create a reader to recognize the English language\n",
    "\n",
    "    for box, label, score in zip(prediction[0]['boxes'], prediction[0]['labels'], prediction[0]['scores']):\n",
    "        if label >= 1 and label <= 4 and score > 0.7:                                                 # Check if the score is above 0.7 and label is between 1 and 4\n",
    "            xmin, ymin, xmax, ymax = [int(i) for i in box]                                            # Convert box coordinates to integers\n",
    "            cropped_image = img.crop((xmin, ymin, xmax, ymax))                                        # Crop the image          \n",
    "            bw_img = cropped_image.convert(\"L\")                                                       # Convert the cropped image to grayscale (black and white)\n",
    "            bw_img_array = np.array(bw_img)\n",
    "\n",
    "            # Resize the image (e.g., double the size)\n",
    "            resized_img = cv2.resize(bw_img_array, (0,0), fx=10, fy=10, interpolation = cv2.INTER_CUBIC)\n",
    "\n",
    "            # Contrast Enhancement\n",
    "            clahe = cv2.createCLAHE(clipLimit=4, tileGridSize=(16, 16))\n",
    "            enhanced_img = clahe.apply(resized_img)\n",
    "\n",
    "            # Apply Otsu's thresholding\n",
    "            _, thresh_img = cv2.threshold(enhanced_img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n",
    "\n",
    "            if label == CUSTOM_CLASSES[\"y-axis\"]:\n",
    "                y_axis_image = cropped_image\n",
    "\n",
    "                result = reader.readtext(thresh_img)\n",
    "                text_results = [item[1] for item in result]                            # Extract only the text\n",
    "                text_found = ','.join(text_results)\n",
    "\n",
    "                for (bbox, text, prob) in result:\n",
    "                    bottom_y1 = bbox[1][1]  # Get the y-coordinate of one of the bottom points\n",
    "                    bottom_y2 = bbox[2][1]  # Get the y-coordinate of the other bottom point\n",
    "                    #print(bottom_y1, bottom_y2)\n",
    "                    average_y = (bottom_y1 + bottom_y2) / 2  # Calculate the average y-coordinate\n",
    "                    quarter_y = (bottom_y2 + average_y) / 2\n",
    "                    yaxis_coordinates.append(int(quarter_y / 10))  # Add the average y-coordinate for each text box available (divide by the upscale factor)\n",
    "            else:\n",
    "                result = reader.readtext(thresh_img, detail = 0)      \n",
    "                text_found = ','.join(result)\n",
    "                text_found = re.sub('\\n+', ',', text_found)                                           # Replace one or more consecutive newline characters with a single comma\n",
    "\n",
    "\n",
    "            label_name = [name for name, id in CUSTOM_CLASSES.items() if id == label][0]              # Find the label name using the CUSTOM_CLASSES dictionary\n",
    "            results[label_name] = text_found                                                          # Store the result in the dictionary\n",
    "\n",
    "        elif label == 5 and score > 0.7:                                                              # Check if the label is 5 and the score is above 0.7\n",
    "            xmin, ymin, xmax, ymax = [int(i) for i in box]                                            # Convert box coordinates to integers\n",
    "            label5_image = img.crop((xmin, ymin, xmax, ymax))                                         # Crop the image for label 5\n",
    "    # The results dictionary now contains the extracted text for labels 1 to 4\n",
    "    #print(results)\n",
    "\n",
    "    #############################################################################################################################\n",
    "    # Plot processing\n",
    "    bw_label5_img = label5_image.convert(\"L\")                                                       # Convert the cropped image to grayscale (black and white)\n",
    "    bw_label5_img_array = np.array(bw_label5_img)\n",
    "\n",
    "    # Apply Gaussian blur to reduce noise and improve the result of Canny edge detection\n",
    "    blurred_label5_img = cv2.GaussianBlur(bw_label5_img_array, (3, 3), 0)\n",
    "\n",
    "    # First, apply a binary threshold on the image\n",
    "    thresh_label5_img = cv2.adaptiveThreshold(blurred_label5_img, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 11, 2)\n",
    "    inverted_label5_img = cv2.bitwise_not(thresh_label5_img)\n",
    "\n",
    "    # Define a kernel for the erosion (a 3x3 square is common)\n",
    "    kernel = np.ones((2,2), np.uint8)\n",
    "    closed_label5_img = cv2.morphologyEx(inverted_label5_img, cv2.MORPH_CLOSE, kernel)\n",
    "\n",
    "    ######################################################################################################################\n",
    "    # Find the contours\n",
    "    contours, hierarchy = cv2.findContours(closed_label5_img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n",
    "\n",
    "    # Calculate the perimeter for each contour and find the index of the longest one\n",
    "    max_length = 0\n",
    "    longest_contour_index = 0\n",
    "    for i, contour in enumerate(contours):\n",
    "        length = cv2.arcLength(contour, True) # The 'True' argument indicates that the contour is closed\n",
    "        if length > max_length:\n",
    "            max_length = length\n",
    "            longest_contour_index = i\n",
    "\n",
    "    # Extract the longest contour\n",
    "    longest_contour = contours[longest_contour_index]\n",
    "\n",
    "    height, width = np.zeros_like(closed_label5_img).shape\n",
    "    #print(\"Plot size is: \", height,width)\n",
    "    filtered_coordinates = filter_coordinates(longest_contour)\n",
    "    filtered_coordinates = remove_corner_pixels(filtered_coordinates, width, height)\n",
    "\n",
    "    # Finding the highest pixel (smallest y value)\n",
    "    highest_pixel = filtered_coordinates[filtered_coordinates[:, 1].argmin()]\n",
    "\n",
    "    # Finding the lowest pixel (largest y value)\n",
    "    lowest_pixel = filtered_coordinates[filtered_coordinates[:, 1].argmax()]\n",
    "\n",
    "    # Finding the rightmost pixel (largest x value), and among those, the one with the lowest y value\n",
    "    right_most_pixels = filtered_coordinates[filtered_coordinates[:, 0] == filtered_coordinates[:, 0].max()]\n",
    "    right_most_pixel = right_most_pixels[right_most_pixels[:, 1].argmin()]\n",
    "\n",
    "    ######################################################################################################################\n",
    "    # DRAWING THE Y-AXIS DATA LOCATION ON THE IMAGE\n",
    "    y_axis_str = results['y-axis']\n",
    "    y_axis_values, y_axis_val_rem_idx = process_ocr_string(y_axis_str)\n",
    "    yaxis_coordinates = [value for index, value in enumerate(yaxis_coordinates) if index not in y_axis_val_rem_idx]\n",
    "    #print(y_axis_values, len(y_axis_values))\n",
    "    #print(yaxis_coordinates, len(yaxis_coordinates))\n",
    "    #print(y_axis_val_rem_idx)\n",
    "    y_axis_values_new = calculate_new_list(y_axis_values)\n",
    "    yaxis_coordinates_new = calculate_new_list(yaxis_coordinates)\n",
    "    yaxis_coordinates_new = list(map(round, yaxis_coordinates_new))\n",
    "\n",
    "    #print(\"New Y-axis values:\", y_axis_values_new)\n",
    "    #print(\"New Y coordinates:\", yaxis_coordinates_new)\n",
    "\n",
    "    ######################################################################################################################\n",
    "    # FIRST INTERPOLATION METHOD\n",
    "    # Value interpolation from curve\n",
    "    # Create a linear interpolation function\n",
    "    f = interp1d(yaxis_coordinates_new, y_axis_values_new)\n",
    "\n",
    "    try:\n",
    "        type1_interp_current_val = round(float(f(right_most_pixel[1])), 5)\n",
    "        #print(f\"The Y-axis value for the Y coordinate {right_most_pixel[1]} is {type1_interp_current_val}\")\n",
    "    except ValueError:\n",
    "        type1_interp_current_val = 1e9\n",
    "\n",
    "    # value from plot\n",
    "    value_str = correct_value(correct_decimal_point(results['value']))\n",
    "    stock_value = extract_float_value(value_str)\n",
    "    #print(f\"The stock value from the plot is {stock_value}\")\n",
    "\n",
    "    # value difference/accuracy\n",
    "    type1_pct_diff = abs(round(((stock_value - type1_interp_current_val) / stock_value) * 100, 5))\n",
    "    #print(f\"The percentage difference is {type1_pct_diff:.2f}%\")\n",
    "\n",
    "    ######################################################################################################################\n",
    "    # SECOND INTERPOLATION METHOD\n",
    "    y_axis_val_diff = abs(y_axis_values[-1] - y_axis_values[0])\n",
    "    y_axis_coord_dif = abs(yaxis_coordinates[-1] - yaxis_coordinates[0])\n",
    "    y_axis_val_per_pixel = y_axis_val_diff/y_axis_coord_dif\n",
    "\n",
    "    #print(y_axis_val_per_pixel)\n",
    "\n",
    "    plot_height = label5_image.height\n",
    "    type2_interp_current_val = (abs(plot_height - right_most_pixel[1])) * y_axis_val_per_pixel\n",
    "\n",
    "    #print(f\"Curve value based on secondaly calculation is {type2_interp_current_val}\")\n",
    "\n",
    "    # value from plot\n",
    "    type2_pct_diff = abs(round(((stock_value - type2_interp_current_val) / stock_value) * 100, 3))\n",
    "    #print(f\"The percentage difference is {type2_pct_diff:.3f}%\")\n",
    "    ######################################################################################################################\n",
    "    # Compare the 2 methods and choose the one with the lowest percentage difference\n",
    "    if type1_pct_diff < type2_pct_diff:\n",
    "        lowest_value  = round(float(f(lowest_pixel[1])), 5)\n",
    "        highest_value = round(float(f(highest_pixel[1])), 5)\n",
    "    else:\n",
    "        lowest_value  = round((abs(plot_height - lowest_pixel[1])) * y_axis_val_per_pixel, 5)\n",
    "        highest_value = round((abs(plot_height - highest_pixel[1])) * y_axis_val_per_pixel, 5)\n",
    "\n",
    "    ######################################################################################################################\n",
    "    #print(user_choice)\n",
    "    \n",
    "    if user_choice == \"plot_to_speech\" or user_choice == \"report_creation\":\n",
    "        current_date = datetime.now().date()\n",
    "        human_readable_date = current_date.strftime('%dth of %B %Y').replace('%dth', str(current_date.day)\n",
    "                              + {1: 'st', 2: 'nd', 3: 'rd'}.get(current_date.day % 10, 'th'))\n",
    "        name = results['name'].split(\" Price\")[0]\n",
    "\n",
    "        phrase = f\"The current value for the currency {name} is {str(stock_value)}. \\\n",
    "                   This represents a drop of {str(extract_percentage(results['value']))}% \\\n",
    "                    in the timeframe of {time_mapping(extract_bracketed_timeframe(results['value']))}. \\\n",
    "                   The currency's lowest value in the same timeframe is {str(lowest_value)}, \\\n",
    "                   meanwhile the highest value is  {str(highest_value)}. \\\n",
    "                   These values are accurate as of {human_readable_date}\"\n",
    "        \n",
    "        phrase = ' '.join(phrase.split())                                                    # Removing extra whitespace and newline characters                                                          \n",
    "        #print(phrase)\n",
    "\n",
    "        # close processing dialog window\n",
    "        close_processing_dialog()\n",
    "        \n",
    "        if user_choice == \"plot_to_speech\":\n",
    "            speak_text(phrase)\n",
    "        else:\n",
    "            if frame_type == \"video\":\n",
    "                doc_path = 'system_outputs/report_video.docx'                  # Define the path to the document\n",
    "            else:\n",
    "                doc_path = 'system_outputs/report_image.docx'                  # Define the path to the document\n",
    "                \n",
    "            directory = os.path.dirname(doc_path)                    # Create the directory if it doesn't exist\n",
    "            if not os.path.exists(directory):\n",
    "                os.makedirs(directory)\n",
    "                \n",
    "            if os.path.exists(doc_path) and append_to_doc:                               # Check if the file exists and the append_to_doc variable is True       \n",
    "                doc = Document(doc_path)                                                 # Open the existing document and append the new paragraph\n",
    "                doc.add_paragraph(phrase)\n",
    "            else:\n",
    "                # Create a new document and add the paragraph\n",
    "                doc = Document()\n",
    "                doc.add_paragraph(phrase)\n",
    "                \n",
    "            # Save the document\n",
    "            doc.save(doc_path)\n",
    "        \n",
    "    elif user_choice == \"dataset_cration\":\n",
    "        # Define the observations\n",
    "        data = {\n",
    "            'name': results['name'].split(\" Price\")[0],\n",
    "            'currency': extract_currency(results['value']),\n",
    "            'value.pct.drop': extract_percentage(results['value']),\n",
    "            'current.value': stock_value,\n",
    "            'type1.interp.current.value': type1_interp_current_val,\n",
    "            'type2.interp.current.value': type2_interp_current_val,\n",
    "            'type1.curr.val.pct.diff': type1_pct_diff,\n",
    "            'type2.curr.val.pct.diff': type2_pct_diff,\n",
    "            'lowest.value': lowest_value,\n",
    "            'highest.value': highest_value,\n",
    "            'timeframe': time_mapping(extract_bracketed_timeframe(results['value'])),\n",
    "            'date': datetime.now().date(),\n",
    "            'time': datetime.now().time().strftime('%H:%M:%S')                            # Format the time properly\n",
    "        }\n",
    "        # Create a DataFrame\n",
    "        df = pd.DataFrame(data, index = [0])\n",
    "        \n",
    "        currency = extract_currency(results['value'])\n",
    "        percentage_drop = extract_percentage(results['value'])\n",
    "        print(f\"The raw OCR reading fro the currency is: {results['value']}, the corrected output is: {currency}\")\n",
    "        print(f\"The raw OCR reading for the percentage drop is: {results['value']}, the corrected output is: {percentage_drop}\")\n",
    "\n",
    "        # Create the directory if it doesn't exist\n",
    "        if frame_type == \"video\":\n",
    "            dataset_path = \"system_outputs/dataset_video.csv\"\n",
    "        else:\n",
    "            dataset_path = \"system_outputs/dataset_image.csv\"\n",
    "        directory = os.path.dirname(dataset_path)\n",
    "        if not os.path.exists(directory):\n",
    "            os.makedirs(directory)\n",
    "\n",
    "        if os.path.exists(dataset_path) and frame_type == \"video\":                        # Check if the file exists and the append_to_file variable is True \n",
    "            df.to_csv(dataset_path, mode='a', header=False, index=False)                     # Append to the existing file (without writing the header again)\n",
    "        else:\n",
    "            df.to_csv(dataset_path, index=False)                                      # Save the DataFrame to a new CSV file (or replace the existing one)\n",
    "            \n",
    "        clean_memory()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33686ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_press(event):\n",
    "    global x, y\n",
    "    x, y = event.x_root, event.y_root\n",
    "\n",
    "def on_drag(event):\n",
    "    global x, y\n",
    "    deltax = event.x_root - x\n",
    "    deltay = event.y_root - y\n",
    "    new_x = transparent_window.winfo_x() + deltax\n",
    "    new_y = transparent_window.winfo_y() + deltay\n",
    "    transparent_window.geometry(f\"{width}x{height}+{new_x}+{new_y}\")\n",
    "    x, y = event.x_root, event.y_root\n",
    "\n",
    "def on_resize(event):\n",
    "    global width, height\n",
    "    width = event.x\n",
    "    height = event.y\n",
    "    transparent_window.geometry(f\"{width}x{height}+{transparent_window.winfo_x()}+{transparent_window.winfo_y()}\")\n",
    "\n",
    "def start_capture(event):\n",
    "    global capturing, left, top, width, height\n",
    "    capturing = True\n",
    "    left = transparent_window.winfo_x()\n",
    "    top = transparent_window.winfo_y()\n",
    "    threading.Thread(target=desktop_capture_process).start()\n",
    "    \n",
    "    transparent_window.destroy()  # Close the transparent window after capturing\n",
    "    #show_processing_dialog()\n",
    "\n",
    "def desktop_capture_process():\n",
    "    #time.sleep(3)\n",
    "    \n",
    "    global left, top, width, height, capturing\n",
    "\n",
    "    with mss() as sct:\n",
    "        while capturing:  # Stop when capturing is False\n",
    "            monitor = {\"top\": top, \"left\": left, \"width\": width, \"height\": height}\n",
    "            capture = sct.grab(monitor)\n",
    "            pil_frame = Image.frombytes(\"RGB\", capture.size, capture.rgb)\n",
    "            resized_frame = resize_frame(pil_frame)\n",
    "            process_frame(resized_frame, \"video\")\n",
    "\n",
    "    #close_processing_dialog()\n",
    "    \n",
    "def create_highlighted_window():\n",
    "    global transparent_window, width, height\n",
    "    width, height = 900, 600\n",
    "    transparent_window = tk.Toplevel(root)\n",
    "    transparent_window.overrideredirect(1) # Remove window decorations\n",
    "    transparent_window.attributes('-alpha', 0.3) # Make the window transparent\n",
    "    transparent_window.geometry(f\"{width}x{height}+100+100\")\n",
    "    transparent_window.attributes('-topmost', True) # Keep the window always on top\n",
    "\n",
    "    frame = tk.Frame(transparent_window, borderwidth=10, relief=\"solid\")\n",
    "    frame.pack(fill=\"both\", expand=True)\n",
    "\n",
    "    transparent_window.bind('<ButtonPress-1>', on_press)\n",
    "    transparent_window.bind('<B1-Motion>', on_drag)\n",
    "    transparent_window.bind('<B3-Motion>', on_resize)\n",
    "    transparent_window.bind('<ButtonPress-2>', start_capture)\n",
    "\n",
    "def desktop_capture():\n",
    "    global capture_window, capture_canvas\n",
    "    # Create the transparent window for selecting the capture area\n",
    "    create_highlighted_window()\n",
    "\n",
    "    \n",
    "def open_system():\n",
    "    root.withdraw()\n",
    "    system_page.deiconify()\n",
    "    \n",
    "def open_options():\n",
    "    root.withdraw()\n",
    "    system_page.withdraw()\n",
    "    options_page.deiconify()\n",
    "\n",
    "def show_description():\n",
    "    messagebox.showinfo(\"System Description\", \"System from charts to data conversion.\")\n",
    "\n",
    "def quit_program():\n",
    "    root.destroy()\n",
    "\n",
    "def choose_image():\n",
    "    image_path = filedialog.askopenfilename(filetypes=[(\"Image files\", \"*.jpg *.png *.bmp *.jpeg *.tiff\")])\n",
    "    if image_path == \"\":\n",
    "        messagebox.showinfo(\"User information\", \"No file has been selected, the operation has been cancelled!\")\n",
    "        pass\n",
    "    else:\n",
    "        show_processing_dialog()\n",
    "        # Start the processing task in a separate thread, passing the image path as an argument\n",
    "        threading.Thread(target=process_image, args=(image_path,)).start()\n",
    "    \n",
    "def process_image(image_path):\n",
    "    image_selected = Image.open(image_path)\n",
    "    process_frame(image_selected, \"image\")\n",
    "    \n",
    "    # Once processing is complete, inform the user and close the dialog\n",
    "    close_processing_dialog()\n",
    "    messagebox.showinfo(\"Processing complete\", \"The selected action has been completed!\")\n",
    "\n",
    "def choose_video():\n",
    "    video_path = filedialog.askopenfilename(filetypes=[(\"Video files\", \"*.mkv *.mp4 *.avi *.mov *.wmv\")])\n",
    "    if video_path == \"\":\n",
    "        messagebox.showinfo(\"User information\", \"No file has been selected, the operation has been cancelled!\")\n",
    "        pass\n",
    "    else:\n",
    "        show_processing_dialog()\n",
    "    \n",
    "        # Start the processing task in a separate thread, passing the necessary information as arguments\n",
    "        threading.Thread(target=process_video, args=(video_path, )).start()\n",
    "\n",
    "\n",
    "def process_video(video_path):\n",
    "    global capturing\n",
    "    capturing = True  # Set capturing to True when starting the capture\n",
    "\n",
    "    cap = cv2.VideoCapture(video_path)\n",
    "\n",
    "    while cap.isOpened():\n",
    "        ret, frame = cap.read()\n",
    "        if not ret:\n",
    "            break\n",
    "        if capturing is False:\n",
    "            break\n",
    "\n",
    "        # Convert the BGR frame to RGB\n",
    "        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "        # Convert the OpenCV image to a PIL Image\n",
    "        pil_frame = Image.fromarray(frame_rgb)\n",
    "        resized_frame = resize_frame(pil_frame)\n",
    "        process_frame(resized_frame, \"video\")\n",
    "\n",
    "    # Once processing is complete, inform the user and close the dialog\n",
    "    close_processing_dialog()\n",
    "    messagebox.showinfo(\"Processing complete\", \"The selected action has been completed or manually stopped!\")\n",
    "\n",
    "    \n",
    "def back_to_main():\n",
    "    global capturing\n",
    "    capturing = False\n",
    "    \n",
    "    options_page.withdraw()\n",
    "    root.deiconify()\n",
    "    \n",
    "def on_sys_btn_click(choice):\n",
    "    global user_choice\n",
    "    user_choice = choice\n",
    "    open_options()\n",
    "\n",
    "def blend_image_with_color(image_path, color, size=(193, 54)):\n",
    "    image = Image.open(image_path).convert(\"RGBA\")\n",
    "    image = image.resize(size, Image.LANCZOS)  # Anti-aliasing applied here\n",
    "    blended_image = Image.new(\"RGBA\", size, color)\n",
    "    blended_image.paste(image, (0, 0), image)\n",
    "    return ImageTk.PhotoImage(blended_image)\n",
    "\n",
    "def animate():\n",
    "    global gif_index, animation_id\n",
    "    try:\n",
    "        if dialog.winfo_exists():                                         # Check if the dialog still exists\n",
    "            gif_index = (gif_index + 1) % len(frames)\n",
    "            label_image.config(image=frames[gif_index])\n",
    "            animation_id = dialog.after(15, animate)\n",
    "        else:\n",
    "            dialog.after_cancel(animation_id)\n",
    "            animation_id = None\n",
    "    except _tkinter.TclError:\n",
    "        pass\n",
    "\n",
    "def show_processing_dialog():\n",
    "    global dialog, label_image, frames, gif_index, animation_id\n",
    "    animation_id = None\n",
    "    gif_index = 0\n",
    "    dialog = tk.Toplevel(root)\n",
    "    dialog.geometry(\"500x500\")\n",
    "    dialog.title(\"Processing... Please wait!\")\n",
    "    dialog.iconbitmap(icon_path)          # Set the icon for the main page\n",
    "    \n",
    "    # Open the GIF and extract the frames\n",
    "    image = Image.open('ui/processing/original.gif') # Correct the path if necessary\n",
    "    frames = [ImageTk.PhotoImage(frame.copy().convert('RGBA')) for frame in ImageSequence.Iterator(image)]\n",
    "    image.close()\n",
    "\n",
    "    label_image = tk.Label(dialog, image=frames[0])\n",
    "    label_image.pack(expand=True)\n",
    "\n",
    "    # Start the animation\n",
    "    animate()\n",
    "\n",
    "def close_processing_dialog():\n",
    "    try: \n",
    "        dialog.destroy()\n",
    "    except _tkinter.TclError:\n",
    "        pass\n",
    "    \n",
    "    clean_memory()\n",
    "    \n",
    "def speak_text(text):\n",
    "    engine = pyttsx3.init()\n",
    "    \n",
    "    \n",
    "    rate = engine.getProperty('rate')\n",
    "    engine.setProperty('rate', rate + 10)        # increase the speech rate (words per minute)\n",
    "    \n",
    "    engine.say(text)\n",
    "    engine.runAndWait()                          # blocks while processing all the currently queued commands\n",
    "\n",
    "    #time.sleep(1)                               # ensure there's a pause between each piece of text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d919f3e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################################################################################\n",
    "root = tk.Tk()\n",
    "root.title('Plot Data Extraction via Computer Vision')\n",
    "root.geometry(\"970x450\")  # Set default size to 970x450\n",
    "root.resizable(False, False) # Make the window non-resizable\n",
    "\n",
    "original_img = Image.open(\"ui/backgrounds/page_1_background.png\")\n",
    "resized_img = original_img.resize((970, 450), Image.LANCZOS) # Resize the image\n",
    "img = ImageTk.PhotoImage(resized_img)\n",
    "\n",
    "background_label = tk.Label(root, image=img)\n",
    "background_label.place(x=0, y=0, relwidth=1, relheight=1)\n",
    "\n",
    "# Create a frame to hold the buttons\n",
    "background_color = (0,9,64,255)\n",
    "\n",
    "button_frame = tk.Frame(root, bg=\"#000940\")\n",
    "button_frame.place(relx=0.5, rely=0.555, anchor=tk.CENTER)\n",
    "\n",
    "main_btn1_img = blend_image_with_color(\"ui/buttons/system_options.png\", background_color)\n",
    "main_btn2_img = blend_image_with_color(\"ui/buttons/system_details.png\", background_color)\n",
    "main_btn3_img = blend_image_with_color(\"ui/buttons/close_system.png\", background_color)\n",
    "\n",
    "# Create buttons within the frame\n",
    "main_btn1 = tk.Button(button_frame, image=main_btn1_img, command=open_system, borderwidth=0, relief='flat', highlightthickness=0)\n",
    "main_btn1.grid(row=0, column=0, pady=10)\n",
    "\n",
    "main_btn2 = tk.Button(button_frame, image=main_btn2_img, command=show_description, borderwidth=0, relief='flat', highlightthickness=0)\n",
    "main_btn2.grid(row=1, column=0, pady=10)\n",
    "\n",
    "main_btn3 = tk.Button(button_frame, image=main_btn3_img, command=quit_program, borderwidth=0, relief='flat', highlightthickness=0)\n",
    "main_btn3.grid(row=2, column=0, pady=10)\n",
    "\n",
    "###############################################################################################################################################\n",
    "system_page = tk.Toplevel(root)\n",
    "system_page.title('System Page')\n",
    "system_page.geometry(\"970x450\")  # Set default size to 970x450\n",
    "system_page.resizable(False, False) # Make the window non-resizable\n",
    "system_page.withdraw()\n",
    "\n",
    "system_img = img\n",
    "\n",
    "system_background_label = tk.Label(system_page, image=system_img)\n",
    "system_background_label.place(x=0, y=0, relwidth=1, relheight=1)\n",
    "\n",
    "# Create a frame to hold the system buttons\n",
    "system_button_frame = tk.Frame(system_page, bg=\"#000940\")\n",
    "system_button_frame.place(relx=0.5, rely=0.555, anchor=tk.CENTER)\n",
    "\n",
    "img_sys_btn1 = blend_image_with_color(\"ui/buttons/plot_to_speech.png\", background_color)\n",
    "img_sys_btn2 = blend_image_with_color(\"ui/buttons/dataset_creation.png\", background_color)\n",
    "img_sys_btn3 = blend_image_with_color(\"ui/buttons/report_creation.png\", background_color)\n",
    "\n",
    "# Create buttons within the system frame\n",
    "sys_btn1 = tk.Button(system_button_frame, image=img_sys_btn1, command=lambda: on_sys_btn_click(\"plot_to_speech\"), borderwidth=0, relief='flat', highlightthickness=0)\n",
    "sys_btn1.grid(row=0, column=0, pady=10)\n",
    "\n",
    "sys_btn2 = tk.Button(system_button_frame, image=img_sys_btn2, command=lambda: on_sys_btn_click(\"dataset_cration\"), borderwidth=0, relief='flat', highlightthickness=0)\n",
    "sys_btn2.grid(row=1, column=0, pady=10)\n",
    "\n",
    "sys_btn3 = tk.Button(system_button_frame, image=img_sys_btn3, command=lambda: on_sys_btn_click(\"report_creation\"), borderwidth=0, relief='flat', highlightthickness=0)\n",
    "sys_btn3.grid(row=2, column=0, pady=10)\n",
    "\n",
    "###############################################################################################################################################\n",
    "options_page = tk.Toplevel(root)\n",
    "options_page.title('Options Page')\n",
    "options_page.geometry(\"970x450\")  # Set default size to 970x450\n",
    "options_page.resizable(False, False) # Make the window non-resizable\n",
    "options_page.withdraw()\n",
    "\n",
    "options_img = img\n",
    "\n",
    "background_label2 = tk.Label(options_page, image=options_img)\n",
    "background_label2.place(x=0, y=0, relwidth=1, relheight=1)\n",
    "\n",
    "# Create a frame to hold the options buttons on the options page\n",
    "options_button_frame = tk.Frame(options_page, bg=\"#000940\")\n",
    "options_button_frame.place(relx=0.5, rely=0.555, anchor=tk.CENTER)\n",
    "\n",
    "img_opt_btn1 = blend_image_with_color(\"ui/buttons/choose_picture.png\", background_color)\n",
    "img_opt_btn2 = blend_image_with_color(\"ui/buttons/choose_video.png\", background_color)\n",
    "img_opt_btn3 = blend_image_with_color(\"ui/buttons/desktop_capture.png\", background_color)\n",
    "img_opt_btn4 = blend_image_with_color(\"ui/buttons/main_window.png\", background_color)\n",
    "\n",
    "# Create buttons within the options frame\n",
    "opt_btn1 = tk.Button(options_button_frame, image=img_opt_btn1, command=choose_image, borderwidth=0, relief='flat', highlightthickness=0)\n",
    "opt_btn1.grid(row=0, column=0, pady=10)\n",
    "\n",
    "opt_btn2 = tk.Button(options_button_frame, image=img_opt_btn2, command=choose_video, borderwidth=0, relief='flat', highlightthickness=0)\n",
    "opt_btn2.grid(row=0, column=1, pady=10)\n",
    "\n",
    "opt_btn3 = tk.Button(options_button_frame, image=img_opt_btn3, command=desktop_capture, borderwidth=0, relief='flat', highlightthickness=0)\n",
    "opt_btn3.grid(row=1, column=0, pady=10)\n",
    "\n",
    "opt_btn4 = tk.Button(options_button_frame, image=img_opt_btn4, command=back_to_main, borderwidth=0, relief='flat', highlightthickness=0)\n",
    "opt_btn4.grid(row=1, column=1, pady=10)\n",
    "\n",
    "###############################################################################################################################################\n",
    "icon_path = \"ui/window_icons/bar_icon_inverted.ico\"\n",
    "\n",
    "root.iconbitmap(icon_path)          # Set the icon for the main page\n",
    "system_page.iconbitmap(icon_path)   # Set the icon for the system page\n",
    "options_page.iconbitmap(icon_path)  # Set the icon for the options page\n",
    "\n",
    "###############################################################################################################################################\n",
    "root.mainloop()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch-cude-env",
   "language": "python",
   "name": "pytorch-cude-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}