transformer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BDegenkolb/Privacy-Preserving-Stress-Transformer/blob/main/transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BYQT5T1tcb02"
      },
      "source": [
        "# Pip Installs\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jtFg8BL7ckHE"
      },
      "outputs": [],
      "source": [
        "!pip install ipython-autotime"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T5Eox1iqcmSG"
      },
      "outputs": [],
      "source": [
        "!pip install neptune"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DjuWSEXHcn3Q"
      },
      "outputs": [],
      "source": [
        "!pip install pyyaml h5py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ULiWo8TpldAb"
      },
      "outputs": [],
      "source": [
        "pip install tensorflow-privacy"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install ann_visualizer"
      ],
      "metadata": {
        "id": "GaHysHH9WHFu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pip install graphviz"
      ],
      "metadata": {
        "id": "XP2k8AymWUK9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqScMeNYLoBo"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KxnCDJgYrK9j"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import pickle\n",
        "import scipy.signal\n",
        "from scipy import fft\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "\n",
        "import tensorflow as tf\n",
        "tf.__version__\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.model_selection import LeaveOneGroupOut\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "# Neu\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow_privacy.privacy.optimizers import dp_optimizer_keras\n",
        "\n",
        "import tensorflow_privacy\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XW3qCiFK1O8n"
      },
      "outputs": [],
      "source": [
        "from tensorflow.python.client import device_lib\n",
        "\n",
        "tf.config.list_physical_devices('GPU')\n",
        "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1r0naEHz0wt6"
      },
      "outputs": [],
      "source": [
        "%load_ext autotime"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RiBUcvWLrK9k"
      },
      "source": [
        "# Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nJk7fB5yrK9m"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")\n",
        "!ls \"/content/drive/MyDrive/Masterarbeit/Transformer/data/WESAD\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fmfNdXnxrK9m"
      },
      "outputs": [],
      "source": [
        "DATA_PATH = os.path.realpath(\"/content/drive/MyDrive/Masterarbeit/Transformer/data/WESAD\")\n",
        "#DATA_PATH = os.chdir(\"/content/drive/MyDrive/Masterarbeit/Transformer/data/WESAD\")\n",
        "checkpoint_prepath = os.path.realpath(\"/content/drive/MyDrive/Masterarbeit/Transformer/code/models/\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oy-5GraPrK9n"
      },
      "outputs": [],
      "source": [
        "class Subject:\n",
        "    \"\"\"Subject of the WESAD dataset.\n",
        "    Subject Class inspired by: https://github.com/WJMatthew/WESAD\"\"\"\n",
        "\n",
        "    def __init__(self, main_path, subject_number):\n",
        "        self.name = f'S{subject_number}'\n",
        "        self.subject_keys = ['signal', 'label', 'subject']\n",
        "        self.signal_keys = ['chest', 'wrist']\n",
        "        self.chest_keys = ['ACC', 'ECG', 'EMG', 'EDA', 'Temp', 'Resp']\n",
        "        self.wrist_keys = ['ACC', 'BVP', 'EDA', 'TEMP']\n",
        "        with open(os.path.join(main_path, self.name) + '/' + self.name + '.pkl', 'rb') as file:\n",
        "            self.data = pickle.load(file, encoding='latin1')\n",
        "        self.labels = self.data['label']\n",
        "\n",
        "    def get_wrist_data(self):\n",
        "        \"\"\"Returns data measured by the E4 Empatica\"\"\"\n",
        "\n",
        "        data = self.data['signal']['wrist']\n",
        "        return data\n",
        "    \n",
        "    def get_subject_dataframe(self):\n",
        "        \"\"\"Returns a dataframe with the preprocessed data of the subject\"\"\"\n",
        "        wrist_data = self.get_wrist_data()\n",
        "        bvp_signal = wrist_data['BVP'][:,0]\n",
        "        eda_signal = wrist_data['EDA'][:,0]\n",
        "        acc_x_signal = wrist_data['ACC'][:,0]\n",
        "        acc_y_signal = wrist_data['ACC'][:,1]\n",
        "        acc_z_signal = wrist_data['ACC'][:,2]\n",
        "        temp_signal = wrist_data['TEMP'][:,0]\n",
        "        # Upsampling data to match BVP data sampling rate using fourier method as described in Paper/dataset\n",
        "        eda_upsampled = scipy.signal.resample(eda_signal, len(bvp_signal))\n",
        "        temp_upsampled = scipy.signal.resample(temp_signal, len(bvp_signal))\n",
        "        acc_x_upsampled = scipy.signal.resample(acc_x_signal, len(bvp_signal))\n",
        "        acc_y_upsampled = scipy.signal.resample(acc_y_signal, len(bvp_signal))\n",
        "        acc_z_upsampled = scipy.signal.resample(acc_z_signal, len(bvp_signal))\n",
        "        label_df = pd.DataFrame(self.labels, columns=['label'])\n",
        "        label_df.index = [(1 / 700) * i for i in range(len(label_df))] # 700 is the sampling rate of the label\n",
        "        label_df.index = pd.to_datetime(label_df.index, unit='s')\n",
        "        data_arrays = zip(bvp_signal, eda_upsampled, acc_x_upsampled, acc_y_upsampled, acc_z_upsampled, temp_upsampled)\n",
        "        df = pd.DataFrame(data=data_arrays, columns=['BVP', 'EDA', 'ACC_x', 'ACC_y', 'ACC_z', 'TEMP'])\n",
        "        df.index = [(1 / 64) * i for i in range(len(df))] # 64 = sampling rate of BVP\n",
        "        df.index = pd.to_datetime(df.index, unit='s')\n",
        "        df = df.join(label_df)\n",
        "        df['label'] = df['label'].fillna(method='ffill')\n",
        "        df.reset_index(drop=True, inplace=True)\n",
        "        df.drop(df[df['label'].isin([0.0, 4.0, 5.0, 6.0, 7.0])].index, inplace=True)\n",
        "        df['label'] = df['label'].replace([1.0, 2.0, 3.0], [0, 1, 0])\n",
        "        df.reset_index(drop=True, inplace=True)\n",
        "        df = (df-df.min())/(df.max()-df.min()) # Normalize data (no train test leakage since data frame per subject) \n",
        "        return df\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PKCCTAAurK9o"
      },
      "outputs": [],
      "source": [
        "s2 = Subject(DATA_PATH, 2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NnryI-rrrK9p"
      },
      "outputs": [],
      "source": [
        "s2.get_subject_dataframe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cwBcLAwZrK9q"
      },
      "outputs": [],
      "source": [
        "def create_subjects_data() -> dict:\n",
        "    # Create a dictionary with all the subjects and belonging dataframes\n",
        "    subjects = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17]\n",
        "    subjects_data = {}\n",
        "    for subject_num in subjects:\n",
        "        subject = Subject(DATA_PATH, subject_num)\n",
        "        subjects_data[subject.name] = subject.get_subject_dataframe()\n",
        "\n",
        "    return subjects_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V_vz0sK1rK9q"
      },
      "source": [
        "### Window"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F_8TFRBGrK9r"
      },
      "source": [
        "1. Creating the windows\n",
        "2. Create subwindows from the windows\n",
        "3. Calculate the fft of the subwindows\n",
        "4. Average the subwindows\n",
        "\n",
        "![fft](../images/fft.png)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9kdu8rgTrK9s"
      },
      "outputs": [],
      "source": [
        "# Subwindow length of the biosignals\n",
        "signal_subwindow_dict = {\n",
        "    'ACC_x': 7,\n",
        "    'ACC_y': 7,\n",
        "    'ACC_z': 7,\n",
        "    'BVP': 30,\n",
        "    'EDA': 30,\n",
        "    'TEMP': 35\n",
        "    }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4fFchRHzrK9t"
      },
      "outputs": [],
      "source": [
        "# most frequent element in list\n",
        "def most_common(lst):\n",
        "    return max(set(lst), key=lst.count)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4zWYpvW8rK9u"
      },
      "outputs": [],
      "source": [
        "def create_windows(df: pd.DataFrame) -> tuple([pd.DataFrame,list]):\n",
        "    \"\"\"Creates windows from the dataframe and returns the windows and the labels.\n",
        "    If the window is assigned to multiple labels, the most common label is chosen for that period.\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): Subject DataFrame\n",
        "\n",
        "    Returns:\n",
        "        tuple[pd.DataFrame,list]: Windows representing the activity of the subject in one minute and the corresponding labels.\n",
        "    \"\"\"\n",
        "\n",
        "    window_len = 64 * 60 # fs = 64 and window length in seconds = 60\n",
        "    windows, labels = zip(*[(df[i:i+window_len], int(most_common(df['label'][i:i+window_len].to_list()))) for i in range(0,df.shape[0],window_len)])\n",
        "    return windows, labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FQ-6UGY7rK9u"
      },
      "outputs": [],
      "source": [
        "def create_subwindows(df: pd.DataFrame, signal_subwindow_len: int, signal_name: str) -> list:\n",
        "    \"\"\"The function creates subwindows from the windows.\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): Windows representing the activity of the subject in one minute.\n",
        "        signal_subwindow_len (int): Length of the subwindows.\n",
        "        signal_name (str): Name of the signal.\n",
        "\n",
        "    Returns:\n",
        "        list: Subwindows of the signal in the window.\n",
        "    \"\"\"\n",
        "    subwindow_len = 64 * signal_subwindow_len # fs = 64 and sub-window length in seconds = 30\n",
        "    window_len = 64 * 60 # fs = 64 and window length in seconds = 60\n",
        "    window_shift = int(64 * 0.25) # fs = 64 and window shift in seconds = 0.25\n",
        "    subwindows = []\n",
        "\n",
        "    for i in range(0, window_len, window_shift):\n",
        "        if i + subwindow_len <= window_len:\n",
        "            subwindow = df[signal_name][i:i+subwindow_len]\n",
        "            subwindows.append(subwindow)\n",
        "    return subwindows\n",
        "            "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bXcKXPgQrK9v"
      },
      "outputs": [],
      "source": [
        "def fft_subwindows(subwindows: list, duration: int, f_s: int) -> list:\n",
        "    \"\"\"Calculates the fft of the subwindows.\n",
        "\n",
        "    Args:\n",
        "        subwindows (list): C\n",
        "        duration (int): _description_\n",
        "        f_s (int): _description_\n",
        "\n",
        "    Returns:\n",
        "        list: Fft coefficients of the subwindows.\n",
        "    \"\"\"\n",
        "    freqs= []\n",
        "    yfs = []\n",
        "    for subwindow in subwindows:\n",
        "        y = np.array(subwindow)\n",
        "        yf = scipy.fft.fft(y)\n",
        "        l = len(yf)\n",
        "        N = f_s * duration\n",
        "        freq = scipy.fft.fftfreq(N, 1/f_s)\n",
        "\n",
        "        l //= 2\n",
        "        amps = np.abs(yf[0:l])\n",
        "        freq = np.abs(freq[0:l])\n",
        "\n",
        "        # Sort descending amp   \n",
        "        p = amps.argsort()[::-1]\n",
        "        freq = freq[p]\n",
        "        amps = amps[p]\n",
        "\n",
        "        freqs.append(freq)\n",
        "        yfs.append(amps)\n",
        "    return np.asarray(freqs), np.asarray(yfs)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pXgM5w6GrK9v"
      },
      "outputs": [],
      "source": [
        "def average_window(subwindows_fft: list) -> list:\n",
        "    \"\"\"Calculates the average of the fft coefficients of the subwindows.\n",
        "\n",
        "    Args:\n",
        "        subwindows_fft (list): List of fft coefficients of the subwindows.\n",
        "\n",
        "    Returns:\n",
        "        list: Average of the fft coefficients of the subwindow for signals.\n",
        "    \"\"\"\n",
        "    len_yfs = len(subwindows_fft[0])\n",
        "    avg_yfs = []\n",
        "    for i in range(len_yfs):\n",
        "        i_yfs = []\n",
        "        for yf in subwindows_fft:\n",
        "            try:\n",
        "                i_yfs.append(yf[i])\n",
        "            except IndexError:\n",
        "                pass\n",
        "        avg_yfs.append(sum(i_yfs)/len(i_yfs))\n",
        "    return avg_yfs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eYdBIjAYrK9v"
      },
      "outputs": [],
      "source": [
        "def create_preprocessed_subjects_data(subjects_data: dict) -> dict:\n",
        "# Creates averaged windows for all subjects from dataframes\n",
        "\n",
        "    subjects_preprosessed_data = {}\n",
        "    for subject_name, subject_df in subjects_data.items():\n",
        "        subjects_preprosessed_data[subject_name] = {}\n",
        "        windows, labels = create_windows(subject_df)\n",
        "        yfs_per_min_for_signal = {}\n",
        "        X = []\n",
        "        for i in range(0,len(windows) - 1):\n",
        "            for signal in signal_subwindow_dict.keys():\n",
        "\n",
        "                duration_in_sec = signal_subwindow_dict[signal]\n",
        "\n",
        "                subwindows = create_subwindows(windows[i], signal_subwindow_len=duration_in_sec, signal_name=signal)\n",
        "                freqs, yfs = fft_subwindows(subwindows, duration_in_sec, 64)\n",
        "                yfs_average = average_window(yfs)[:210]\n",
        "                yfs_per_min_for_signal[signal] = yfs_average\n",
        "                \n",
        "            X.append(pd.DataFrame(yfs_per_min_for_signal).T)\n",
        "        y = list(labels[:len(windows)-1])\n",
        "        subjects_preprosessed_data[subject_name]['X'] = X\n",
        "        subjects_preprosessed_data[subject_name]['y'] = y\n",
        "    \n",
        "    return subjects_preprosessed_data\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kpyf8pRLrK9w"
      },
      "outputs": [],
      "source": [
        "subjects_data = create_subjects_data()\n",
        "subjects_preprocessed_data = create_preprocessed_subjects_data(subjects_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C20NZIK8rK9w"
      },
      "outputs": [],
      "source": [
        "subjects_preprocessed_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aYWHD0VjrK9x"
      },
      "outputs": [],
      "source": [
        "def get_subject_window_data(subjects_preprosessed_data: dict) -> tuple([list, list]):\n",
        "    # Created train and test data for leave one out cross validation\n",
        "    all_subjects_X = []\n",
        "    all_subjects_y = []\n",
        "    for subject_name, subject_data in subjects_preprosessed_data.items():\n",
        "        all_subjects_X.append(subject_data['X'])\n",
        "        all_subjects_y.append(subject_data['y'])\n",
        "    \n",
        "    return (all_subjects_X, all_subjects_y)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YpvXypU-rK9x"
      },
      "outputs": [],
      "source": [
        "all_subjects_X, all_subjects_y = get_subject_window_data(subjects_preprocessed_data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KWx9LwKIrK9x"
      },
      "source": [
        "![OS_Sensors](../images/os_sensors.png)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r3Nm9bNUrK9x"
      },
      "outputs": [],
      "source": [
        "SMARTWATCH_OS = {\n",
        "    'E4': ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP', 'EDA', 'BVP'],\n",
        "    #'Tizen': ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP', 'BVP'],\n",
        "    #'WearOS_watchOS': ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP'],\n",
        "    #'Fitbit': ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP', 'EDA'],\n",
        "    #'PiaOS': ['TEMP', 'EDA', 'BVP']\n",
        "    }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TrgOg67ErK9x"
      },
      "outputs": [],
      "source": [
        "def filter_for_smartwatch_os(smartwatch_os_name: str, all_subjects_X: list) -> list:\n",
        "    # Adjusts the data for the smartwatch os\n",
        "    all_subjects_X_adjusted_for_smartwatch_os = []\n",
        "    for subject_data in all_subjects_X:\n",
        "        subject_adjusted_for_smartwatch_os = []\n",
        "        for window in subject_data:\n",
        "            subject_adjusted_for_smartwatch_os.append(window.loc[SMARTWATCH_OS[smartwatch_os_name]])\n",
        "        all_subjects_X_adjusted_for_smartwatch_os.append(subject_adjusted_for_smartwatch_os)\n",
        "    return all_subjects_X_adjusted_for_smartwatch_os\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iVz4lLUDMSmH"
      },
      "source": [
        "# Transformer Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "77xcGqoBMcTz"
      },
      "source": [
        "## Build Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Qz6GiLRKDRo"
      },
      "source": [
        "### Transformer Model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rBzmvZAOKL9G"
      },
      "outputs": [],
      "source": [
        "def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):\n",
        "    # Normalization and Attention\n",
        "    x = layers.LayerNormalization(epsilon=1e-6)(inputs)\n",
        "    x = layers.MultiHeadAttention(\n",
        "        key_dim=head_size, num_heads=num_heads, dropout=dropout\n",
        "    )(x, x)\n",
        "    x = layers.Dropout(dropout)(x)\n",
        "    res = x + inputs\n",
        "\n",
        "    # Feed Forward Part\n",
        "    x = layers.LayerNormalization(epsilon=1e-6)(res)\n",
        "    x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"relu\")(x)\n",
        "    x = layers.Dropout(dropout)(x)\n",
        "    x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)\n",
        "    return x + res"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "STOR01UpKT0g"
      },
      "outputs": [],
      "source": [
        "def build_model(\n",
        "   input_shape,\n",
        "    head_size,\n",
        "    num_heads,\n",
        "    ff_dim,\n",
        "    num_transformer_blocks,\n",
        "    mlp_units,\n",
        "    dropout,\n",
        "    mlp_dropout\n",
        "):\n",
        "\n",
        "    inputs = keras.Input(input_shape)\n",
        "    x = inputs\n",
        "    #layers.LSTM(1)\n",
        "    for _ in range(num_transformer_blocks):\n",
        "        x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)\n",
        "\n",
        "    x = layers.GlobalAveragePooling1D(data_format=\"channels_first\")(x)\n",
        "    for dim in mlp_units:\n",
        "        x = layers.Dense(dim, activation=\"relu\")(x)\n",
        "        x = layers.Dropout(mlp_dropout)(x)\n",
        "    outputs = layers.Dense(num_output_class, activation=\"sigmoid\")(x)\n",
        "    #outputs = layers.Dense(num_output_class, activation=\"softmax\")(x)\n",
        "    \n",
        "    return keras.Model(inputs, outputs)\n",
        "    #model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OYGJ-hXWvheP"
      },
      "outputs": [],
      "source": [
        "from tensorflow_privacy.privacy.analysis.compute_noise_from_budget_lib import compute_noise as tfp_computer_noise\n",
        "\n",
        "\"\"\"\n",
        "Calculate noise for given training hyperparameters\n",
        "\"\"\"\n",
        "def compute_noise(n, batch_size, target_epsilon, epochs, delta, min_noise=1e-5):\n",
        "  return tfp_computer_noise(n, batch_size, target_epsilon, epochs, delta, min_noise)\n",
        "\n",
        "\"\"\"\n",
        "Calculate Delta for given training dataset size n\n",
        "\"\"\"\n",
        "def compute_delta(n):\n",
        "# delta should be one magnitude lower than inverse of training set size: 1/n\n",
        "# e.g. 1e-5 for n=60.000\n",
        "# take 1e-x, were x is the magnitude of training set size\n",
        "  delta = np.power(10, - float(len(str(n)))) # remove all trailing decimals\n",
        "  return delta"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t2qF6nByeiew"
      },
      "source": [
        "## Hyperparameter Tracking"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h_2Jt6fld9Yt"
      },
      "outputs": [],
      "source": [
        "import neptune\n",
        "\n",
        "run = neptune.init_run( api_token=\"eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiJmNDZjODdkNS1hMGQzLTQ1M2QtYTBhNy03Y2VlYTU2ZWIzZTkifQ==\", project=\"boris/stresstransformer\")\n",
        "from neptune.new.integrations.tensorflow_keras import NeptuneCallback\n",
        "\n",
        "\n",
        "neptune_callback = NeptuneCallback(run=run, log_model_diagram=True)\n",
        "# params = {'num_transformer_blocks':4, \n",
        "#           'min_delta_loss':0.001, \n",
        "#           'epochs':200,\n",
        "#           'batch_size':50,\n",
        "#           'lr':0.0001,\n",
        "#           'momentum':0.4,\n",
        "#           'mlp_dropout':0.4,\n",
        "#           'dropout':0.25,\n",
        "#           'num_heads':4,\n",
        "#           'ff_dim':4,\n",
        "#           'head_size':256,\n",
        "#            'dp_selected':1,\n",
        "#            'l2_norm_clip':1.5,\n",
        "#            'noise_multiplier':1.3,\n",
        "#            'num_microbatches':1\n",
        "#           }\n",
        "params = {'num_transformer_blocks':8, \n",
        "          'min_delta_loss':0.01, \n",
        "          'epochs':110,\n",
        "          'batch_size':50,\n",
        "          'lr':0.0001,\n",
        "          'mlp_dropout':0.25,\n",
        "          'dropout':0.25,\n",
        "          'num_heads':4,\n",
        "          'ff_dim':4,\n",
        "          'head_size':256,\n",
        "          'dp_selected':0,\n",
        "          'l2_norm_clip':2.0,\n",
        "          'noise_multiplier':68.61306896515201,\n",
        "          'num_microbatches':1\n",
        "          }\n",
        "\n",
        "run['hyper-parameters'] = params\n",
        "\n",
        "if params[\"batch_size\"] % params[\"num_microbatches\"] != 0:\n",
        "  raise ValueError('Batch size should be an integer multiple of the number of microbatches')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f3e8Qh1ieo7F"
      },
      "source": [
        "## Model training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-moBBEUOln2M"
      },
      "outputs": [],
      "source": [
        "# create an empty dictionary to store the number of epochs\n",
        "epochs_dict = {}\n",
        "max_epochs = 0\n",
        "tf.random.set_seed(42)\n",
        "\n",
        "for os, signals in SMARTWATCH_OS.items():\n",
        "  with tf.device('/device:GPU:0'):\n",
        "    print(f'\\n\\n\\nSmartwatchOS: {os} - Signals: {signals} - Num. Signals: {len(signals)}')\n",
        "    print(f'Number of signals: {len(signals)}')\n",
        "    all_subjects_X_os = filter_for_smartwatch_os(os, all_subjects_X)\n",
        "\n",
        "    \n",
        "    groups_set = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]\n",
        "    subject_ids = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17] # ids for subjects in WESAD dataset\n",
        "    \n",
        "    num_signals = len(signals) # Number of signals in the WESAD dataset measured by the empatica e4\n",
        "    num_output_class = 2 # Number of output classes (2 - non-stress vs stress)\n",
        "    num_epochs = params[\"epochs\"]\n",
        "\n",
        "    all_acc_histories = []\n",
        "    all_loss_histories = []\n",
        "\n",
        "    \n",
        "\n",
        "    for i in groups_set:\n",
        "        test_index = groups_set[i]\n",
        "        train_index = [x for x in groups_set if x != test_index]\n",
        "        print(train_index, test_index)\n",
        "        print(f'SmartwatchOS: {os} - Person: {test_index}')\n",
        "\n",
        "        X_train = np.concatenate(np.array([all_subjects_X_os[x] for x in train_index], dtype=object))\n",
        "        y_train = np.concatenate(np.array([all_subjects_y[y] for y in train_index], dtype=object))\n",
        "        X_test = all_subjects_X_os[test_index]\n",
        "        y_test = all_subjects_y[test_index]\n",
        "        \n",
        "        weight_balance = y_train.tolist().count(0)/y_train.tolist().count(1)\n",
        "\n",
        "        X_train = np.asarray(X_train)\n",
        "        #print(X_train)\n",
        "        X_test = np.asarray(X_test)\n",
        "        y_train = np.asarray(y_train)\n",
        "        y_test = np.asarray(y_test)\n",
        "\n",
        "        y_train = tf.keras.utils.to_categorical(y_train, num_output_class)\n",
        "        y_test = tf.keras.utils.to_categorical(y_test, num_output_class)\n",
        "\n",
        "        tf.keras.backend.clear_session()\n",
        "\n",
        "        #input_shape = (6,210,)\n",
        "        input_shape = X_train.shape[1:]\n",
        "        #input_shape = [num_signals, 210, 1]\n",
        "        print(input_shape)\n",
        "        model = build_model(\n",
        "                input_shape,\n",
        "                head_size=params[\"head_size\"],\n",
        "                num_heads=params[\"num_heads\"],\n",
        "                ff_dim=params[\"ff_dim\"],\n",
        "                num_transformer_blocks=params[\"num_transformer_blocks\"],\n",
        "                mlp_units=[128],\n",
        "                mlp_dropout=params[\"mlp_dropout\"],\n",
        "                dropout=params[\"dropout\"]\n",
        "                )\n",
        "        \n",
        "        if params[\"dp_selected\"] == 0:\n",
        "          optimizer = keras.optimizers.Adam(learning_rate=params[\"lr\"])  \n",
        "\n",
        "        elif params[\"dp_selected\"] == 1:\n",
        "          # Select your differentially private optimizer\n",
        "          optimizer = tensorflow_privacy.VectorizedDPKerasAdamOptimizer(\n",
        "                        l2_norm_clip= params[\"l2_norm_clip\"], \n",
        "                        noise_multiplier= params[\"noise_multiplier\"], \n",
        "                        num_microbatches= params[\"num_microbatches\"], \n",
        "                        learning_rate=params[\"lr\"])\n",
        "\n",
        "\n",
        "\n",
        "        model.compile(\n",
        "        #loss=\"sparse_categorical_crossentropy\",\n",
        "        #loss=\"categorical_crossentropy\",\n",
        "        loss=\"binary_crossentropy\",\n",
        "        optimizer=optimizer,\n",
        "        #metrics=[\"sparse_categorical_accuracy\"],\n",
        "        #metrics=[\"categorical_accuracy\"],\n",
        "        metrics=['accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), tf.keras.metrics.BinaryAccuracy()\n",
        "]\n",
        "        )\n",
        "        \n",
        "        \n",
        "        #build_model(num_signals, num_output_class)\n",
        "\n",
        "        checkpoint = tf.keras.callbacks.ModelCheckpoint(#filepath=checkpoint_path,  # Path to save the model file\n",
        "            checkpoint_prepath + f\"/{os}/wesad_{os}_binary_s{subject_ids[test_index]}_{num_epochs}.h5\",                                            \n",
        "            monitor=\"loss\", # The metric name to monitor\n",
        "            save_best_only=True # If True, it only saves the \"best\" model according to the quantity monitored \n",
        "        )\n",
        "\n",
        "\n",
        "        \n",
        "        early_stopping = tf.keras.callbacks.EarlyStopping(\n",
        "            monitor=\"loss\",     # Quantity to be monitored.\n",
        "            min_delta=params[\"min_delta_loss\"],     # Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.\n",
        "            patience=10,        # Number of epochs with no improvement after which training will be stopped.\n",
        "            restore_best_weights=True\n",
        "        )\n",
        "\n",
        "        callbacks = [keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)]\n",
        "        #print(X_train.shape)\n",
        "        \n",
        "        history = model.fit(\n",
        "        X_train,\n",
        "        y_train,\n",
        "        validation_split=0.2,\n",
        "        epochs=num_epochs,\n",
        "        batch_size=params[\"batch_size\"],\n",
        "        class_weight={0: 1, 1: weight_balance},\n",
        "        #callbacks=[checkpoint, callbacks, neptune_callback]\n",
        "        callbacks=[checkpoint, neptune_callback]         # No Early Stopping\n",
        "        #callbacks=[checkpoint, early_stopping]\n",
        "        #callbacks=[checkpoint, early_stopping, neptune_callback] #  Early Stopping\n",
        "        )\n",
        "\n",
        "        #Getting actual number of epochs per Subject (due to early stopping it differs)\n",
        "        number_of_epochs_it_ran = len(history.history['loss'])\n",
        "        n_epochs_best = number_of_epochs_it_ran\n",
        "        # n_epochs_best = number_of_epochs_it_ran - 10 #subtracting patience when using early stopping\n",
        "        #Rewrite max_epochs if current Epoch is higher\n",
        "        if n_epochs_best > max_epochs:\n",
        "          max_epochs = n_epochs_best\n",
        "        # Add number of epochs to dictionary for each subject != the current used subject\n",
        "        for other_subject in train_index:\n",
        "          if other_subject not in epochs_dict:\n",
        "            epochs_dict[other_subject] = n_epochs_best\n",
        "          else:\n",
        "            epochs_dict[other_subject] += n_epochs_best\n",
        "        print(epochs_dict)\n",
        "      \n",
        "      \n",
        "\n",
        "\n",
        "eval_metrics =  model.evaluate(X_test, y_test, verbose=0 )\n",
        "loss = model.evaluate(X_test, y_test, verbose=0 )[0]\n",
        "accuracy = model.evaluate(X_test, y_test, verbose=0 )[1]\n",
        "precision = model.evaluate(X_test, y_test, verbose=0 )[2]\n",
        "recall = model.evaluate(X_test, y_test, verbose=0 )[3]\n",
        "if (precision + recall) != 0:\n",
        "    f1 = 2 * precision * recall / (precision + recall)\n",
        "else:\n",
        "    f1 = 0\n",
        "\n",
        "    \n",
        "run[\"eval/loss\"] = loss\n",
        "run[\"eval/accuracy\"] = accuracy\n",
        "run[\"eval/precision\"] = precision\n",
        "run[\"eval/recall\"] = recall\n",
        "run[\"eval/f1\"] = f1\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sn8Gp_FmmgKd"
      },
      "source": [
        "# Differential Privacy\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Calculate Delta for 18900 Datapoints"
      ],
      "metadata": {
        "id": "VOuyZNsxp-to"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(compute_delta(18900))"
      ],
      "metadata": {
        "id": "tLutkqPwp2r6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Compute Noise for epsilon 1 0,1 and 10"
      ],
      "metadata": {
        "id": "_B8mWeipqG1q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(compute_noise(18900, 50, 1, max(epochs_dict.values()), 1e-5, min_noise=1e-5))"
      ],
      "metadata": {
        "id": "lqbCIuBNpj2t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(compute_noise(18900, 50, 0.1, max(epochs_dict.values()), 1e-5, min_noise=1e-5))"
      ],
      "metadata": {
        "id": "1lCqW1iEqVYT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(compute_noise(18900, 50, 10, max(epochs_dict.values()), 1e-5, min_noise=1e-5))"
      ],
      "metadata": {
        "id": "jU7B6IhbqZ6W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "18Xyh0BFt31-"
      },
      "outputs": [],
      "source": [
        "from tensorflow_privacy.privacy.analysis import compute_dp_sgd_privacy\n",
        "\n",
        "compute_dp_sgd_privacy.compute_dp_sgd_privacy(18900, #210*6*15 = 18900\n",
        "                                              batch_size=params[\"batch_size\"],\n",
        "                                              noise_multiplier=params[\"noise_multiplier\"],\n",
        "                                              epochs=max(epochs_dict.values()), # siehe unten\n",
        "                                              delta=1e-5)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#zb data set >10000 dann delta 1e-5 bei über 100000 dann 1e-6\n",
        "# Signale pro Person wird in der Signalvorverarbeitung fürs Training letztendlich auf (6,210) reduziert. Also 6 Signalarten ( ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP', 'EDA', 'BVP'] - Num. Signals: 6) mit 210 Datenpunkten\n",
        "#210*6*15 = 18900 -> 1e-5\n",
        "#funktion wie groß der noise_multiplier\n",
        "#bei epochen worst case oder genau dict für epochen jede person -> für alle personen außer aktuelle person anzahl der aktuellen epochen hinzuaddieren -> max anzahl aus dem dict als epochen nehmen\n",
        "# max_epochs = max(epochs_dict.values())\n",
        "#epsilon von 10 1 und 0,1 -> 1 weil guter Wert und 10 und 0,1 als referenzwert (zur besseren Vergleichbarkeit)\n",
        "#rdp muss nicht mit rein\n",
        "\n",
        "\n",
        "#optimizer = VectorizedDPKerasAdamOptimizer(\n",
        "#l2_norm_clip=setting.l2_clip,\n",
        "#noise_multiplier=setting.noise,\n",
        "#num_microbatches=setting.batch_size,\n",
        "#learning_rate=setting.learning_rate\n",
        "#)\n",
        "\n",
        "#l2 normclip 1; 1,5\n",
        "#num_microbatches auf die batchsize setzen und schauen was passiert (für hyperparameter auf 1 lassen und dann beim endergebnis auf batchsize setzen)"
      ],
      "metadata": {
        "id": "_C7B6R9Eyzko"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8I03wm4BrK9z"
      },
      "source": [
        "# Evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_xV_QE3HrK90"
      },
      "source": [
        "Evaluating every models on the corresponding test dataset not seen during training."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P4thgDg2rK90"
      },
      "outputs": [],
      "source": [
        "# Evaluating every models on the corresponding test dataset not seen during training.\n",
        "os_scores_acc = {}\n",
        "os_scores_f1 = {}\n",
        "\n",
        "for os, signals in SMARTWATCH_OS.items():\n",
        "\n",
        "    all_subjects_X_os = filter_for_smartwatch_os(os, all_subjects_X)\n",
        "\n",
        "    all_accuracies = []\n",
        "    all_precisions = []\n",
        "    all_recalls = []\n",
        "    all_f1s = []\n",
        "\n",
        "    for i, subject_id in enumerate(subject_ids):\n",
        "        X_test = all_subjects_X_os[i]\n",
        "        y_test = all_subjects_y[i]\n",
        "        X_test = np.asarray(X_test)\n",
        "        y_test = np.asarray(y_test)\n",
        "        \n",
        "        y_test = tf.keras.utils.to_categorical(y_test, num_output_class)\n",
        "        \n",
        "      \n",
        "        \n",
        "        model_path = checkpoint_prepath + f'/{os}/wesad_{os}_binary_s{subject_id}_{num_epochs}.h5'\n",
        "\n",
        "        #model_path = f'models/{os}/wesad_{os}_binary_s{subject_id}_{num_epochs}.h5'\n",
        "\n",
        "        if model.get_weights() == 0:\n",
        "          model = tf.keras.models.load_model(model_path)\n",
        "                                            \n",
        "          \n",
        "        accuracy = model.evaluate(X_test, y_test, verbose=0 )[1]\n",
        "        precision = model.evaluate(X_test, y_test, verbose=0 )[2]\n",
        "        recall = model.evaluate(X_test, y_test, verbose=0 )[3]\n",
        "        if (precision + recall) != 0:\n",
        "          f1 = 2 * precision * recall / (precision + recall)\n",
        "        else:\n",
        "          f1 = 0\n",
        "        all_accuracies.append(accuracy)\n",
        "        all_precisions.append(precision)\n",
        "        all_recalls.append(recall)\n",
        "        all_f1s.append(f1)\n",
        "\n",
        "    print(f'Smartwatch OS: {os}')\n",
        "    print(f'Evaluation of Transformer model trained on {num_epochs} epochs\\n')\n",
        "    print(f'Subject\\t\\t Accuracy\\tPrecision\\tRecall\\t\\tF1-Score')\n",
        "    print(\"************************************************************************\")\n",
        "    for i in range(len(all_accuracies)):\n",
        "        print(f'S{subject_ids[i]}\\t\\t {round(all_accuracies[i], 5):.5f}\\t{round(all_precisions[i], 5):.5f}\\t\\t{round(all_recalls[i], 5):.5f}\\t\\t{round(all_f1s[i], 5):.5f}')\n",
        "\n",
        "    print(\"************************************************************************\")\n",
        "    print(f'Average\\t\\t {round(np.mean(all_accuracies), 5):.5f}\\t{round(np.mean(all_precisions), 5):.5f}\\t\\t{round(np.mean(all_recalls), 5):.5f}\\t\\t{round(np.mean(all_f1s), 5):.5f}\\n\\n\\n')\n",
        "\n",
        "    os_scores_acc[os] = all_accuracies\n",
        "    os_scores_f1[os] = all_f1s\n",
        "\n",
        "    \n",
        "      "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RvNJmDlAmTcZ"
      },
      "outputs": [],
      "source": [
        "run[\"loso/accuracy\"] = round(np.mean(all_accuracies), 5)\n",
        "run[\"loso/precision\"] = round(np.mean(all_precisions), 5)\n",
        "run[\"loso/recall\"] = round(np.mean(all_recalls), 5)\n",
        "run[\"loso/f1\"] = round(np.mean(all_f1s), 5)\n",
        "run[\"eval/maxepochs\"] = max_epochs\n",
        "\n",
        "run.stop()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from ann_visualizer.visualize import ann_viz;\n",
        "#Build your model here\n",
        "ann_viz(model)"
      ],
      "metadata": {
        "id": "M4QB71dFW4aq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OFpeagR0nXqY"
      },
      "source": [
        "# Evaluation Graphics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WZfhW_wfrK90"
      },
      "outputs": [],
      "source": [
        "eval_metrics =  model.evaluate(X_test, y_test, verbose=0 )\n",
        "\n",
        "for j, metric in enumerate(eval_metrics):\n",
        "    run[\"eval/{}\".format(model.metrics_names[j])] = metric"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O6g8cMdtSxoN"
      },
      "outputs": [],
      "source": [
        "num_epochs = 200"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Dj5vf4wPrK91"
      },
      "outputs": [],
      "source": [
        "df_os_scores_acc = pd.DataFrame(os_scores_acc)\n",
        "replacements = {l1:f'S{l2}' for l1, l2 in zip(groups_set, subject_ids)}\n",
        "df_os_scores_acc = df_os_scores_acc.rename(replacements)\n",
        "df_os_scores_acc.to_csv('os_scores_acc.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oZNWLU_BrK91"
      },
      "outputs": [],
      "source": [
        "df_os_scores_f1 = pd.DataFrame(os_scores_f1)\n",
        "replacements = {l1:f'S{l2}' for l1, l2 in zip(groups_set, subject_ids)}\n",
        "df_os_scores_f1 = df_os_scores_f1.rename(replacements)\n",
        "df_os_scores_f1.to_csv('os_scores_f1.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qmh0sQTFrK91"
      },
      "outputs": [],
      "source": [
        "df_os_scores_acc.plot.bar(figsize=(20,10), title='Accuracy of different os models on each subject')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Apfl9sEarK91"
      },
      "outputs": [],
      "source": [
        "df_os_scores_f1.plot.bar(figsize=(20,10), title='F1-Score of different os models on each subject')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8cAAxlLkrK92"
      },
      "outputs": [],
      "source": [
        "# bar plot of the accuracy of the different loso models\n",
        "print(f'{df_os_scores_acc}\\n')\n",
        "print(f'{df_os_scores_acc.mean()}\\n')\n",
        "sns.barplot(x=df_os_scores_acc.keys(), y=df_os_scores_acc.mean(), palette='pastel').set_title(label='Average accuracy of different models on each OS')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R0zWyewFrK93"
      },
      "outputs": [],
      "source": [
        "# bar plot of the accuracy of the different loso models\n",
        "print(f'{df_os_scores_f1}\\n')\n",
        "print(f'{df_os_scores_f1.mean()}\\n')\n",
        "sns.barplot(x=df_os_scores_f1.keys(), y=df_os_scores_f1.mean(), palette='pastel').set_title(label='F1 score of different models on each OS')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ErFjWohMSerp"
      },
      "source": [
        "# Visualisierung"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fxE0s7h-Maio"
      },
      "outputs": [],
      "source": [
        "from skimage import data, io\n",
        "from skimage.color import rgb2gray\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "from IPython.display import display\n",
        "import cv2 as cv\n",
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "from datetime import datetime\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7qq9V3rPMnhb"
      },
      "outputs": [],
      "source": [
        "model_path = checkpoint_prepath + f'/E4/wesad_E4_binary_s2_100.h5'\n",
        "    \n",
        "model = tf.keras.models.load_model(model_path)\n",
        "\n",
        "viz_path = os.path.realpath(\"/content/drive/MyDrive/Masterarbeit/Visualizations/\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SAul1xicM57K"
      },
      "outputs": [],
      "source": [
        "now = datetime.now() # current date and time\n",
        "tf.keras.utils.plot_model(model,to_file= viz_path + \"/\" + now.strftime(\"%d.%m.%Y,%H:%M:%S\") + \"model.png\",show_shapes=True,show_dtype=False,show_layer_names=True,rankdir=\"TB\",expand_nested=True,dpi=96,layer_range=None,show_layer_activations=True,)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qGc7wflbrK93"
      },
      "source": [
        "# Prediction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yk7AIz_BrK93"
      },
      "outputs": [],
      "source": [
        "model_path = checkpoint_prepath + \"/E4/wesad_E4_binary_s3_100.h5\"\n",
        "model = tf.keras.models.load_model(model_path)\n",
        "predictions = model.predict(X_test) # make predictions on the test set using the trained model\n",
        "pred_class = np.argmax(predictions, axis=-1) # get the class with the highest probability"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S3dGPZxHrK93"
      },
      "outputs": [],
      "source": [
        "true_classified = 0\n",
        "for i in range(len(pred_class)):\n",
        "    print(\"Prediction: \", predictions[i])\n",
        "    print(\"Prediction: \", pred_class[i])\n",
        "    ground_truth = max(enumerate(y_test[i]),key=lambda x: x[1])[0]\n",
        "    print(\"Ground Truth: \", ground_truth)\n",
        "    print()\n",
        "    if ground_truth == pred_class[i]:\n",
        "        true_classified += 1\n",
        "print(\"Accuracy: \", true_classified/len(pred_class))\n",
        "    "
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "RiBUcvWLrK9k",
        "3Qz6GiLRKDRo",
        "f3e8Qh1ieo7F",
        "OFpeagR0nXqY",
        "ErFjWohMSerp"
      ],
      "machine_shape": "hm",
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3.9.13 ('pageblank')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "vscode": {
      "interpreter": {
        "hash": "05320e546f369dcc8e105d9a3b35062d70587d63bda88e4cea376da0cda0edd6"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}