yamnet_transfer_learning_P3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Libraries & Modules for YAMNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:43:32.479038Z",
     "iopub.status.busy": "2022-12-14T22:43:32.478558Z",
     "iopub.status.idle": "2022-12-14T22:43:35.532005Z",
     "shell.execute_reply": "2022-12-14T22:43:35.531017Z"
    },
    "id": "urBpRWDHTHHU"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow_io in /home/jihwan/miniconda3/lib/python3.11/site-packages (0.36.0)\r\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem==0.36.0 in /home/jihwan/miniconda3/lib/python3.11/site-packages (from tensorflow_io) (0.36.0)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow_io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:43:35.536836Z",
     "iopub.status.busy": "2022-12-14T22:43:35.536162Z",
     "iopub.status.idle": "2022-12-14T22:43:38.151313Z",
     "shell.execute_reply": "2022-12-14T22:43:38.150563Z"
    },
    "id": "7l3nqdWVF-kC"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-19 23:19:26.042593: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-02-19 23:19:26.063382: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-02-19 23:19:26.063398: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-02-19 23:19:26.064181: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-02-19 23:19:26.068021: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-02-19 23:19:26.432088: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\nimport warnings\\nwarnings.filterwarnings('ignore')\\n\""
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_io as tfio\n",
    "import soundfile as sf\n",
    "\n",
    "import glob\n",
    "import json\n",
    "import re\n",
    "\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]='-1'\n",
    "\n",
    "\"\"\"\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load pretrained models & samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:43:38.155854Z",
     "iopub.status.busy": "2022-12-14T22:43:38.155375Z",
     "iopub.status.idle": "2022-12-14T22:43:45.969263Z",
     "shell.execute_reply": "2022-12-14T22:43:45.968454Z"
    },
    "id": "06CWkBV5v3gr"
   },
   "outputs": [],
   "source": [
    "yamnet_model_handle = 'https://tfhub.dev/google/yamnet/1'\n",
    "yamnet_model = hub.load(yamnet_model_handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:43:45.991912Z",
     "iopub.status.busy": "2022-12-14T22:43:45.991280Z",
     "iopub.status.idle": "2022-12-14T22:43:45.996619Z",
     "shell.execute_reply": "2022-12-14T22:43:45.995950Z"
    },
    "id": "Xwc9Wrdg2EtY"
   },
   "outputs": [],
   "source": [
    "# Utility functions for loading audio files and making sure the sample rate is correct.\n",
    "\n",
    "@tf.function\n",
    "def load_wav_16k_mono(filename):\n",
    "    \"\"\" Load a WAV file, convert it to a float tensor, resample to 16 kHz single-channel audio. \"\"\"\n",
    "    file_contents = tf.io.read_file(filename)\n",
    "    wav, sample_rate = tf.audio.decode_wav(\n",
    "          file_contents,\n",
    "          desired_channels=1)\n",
    "    wav = tf.squeeze(wav, axis=-1)\n",
    "    sample_rate = tf.cast(sample_rate, dtype=tf.int64)\n",
    "    wav = tfio.audio.resample(wav, rate_in=sample_rate, rate_out=16000)\n",
    "    return wav"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load class mappings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiments (Don't run)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:45.417517Z",
     "iopub.status.busy": "2022-12-14T22:44:45.417235Z",
     "iopub.status.idle": "2022-12-14T22:44:45.432919Z",
     "shell.execute_reply": "2022-12-14T22:44:45.432072Z"
    },
    "id": "jwmLygPrMAbH"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>target</th>\n",
       "      <th>category</th>\n",
       "      <th>audio_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>226549</td>\n",
       "      <td>전기사고</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>220721</td>\n",
       "      <td>전기사고</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>232239</td>\n",
       "      <td>전기사고</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>234023</td>\n",
       "      <td>전기사고</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>222619</td>\n",
       "      <td>전기사고</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447060</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>594834</td>\n",
       "      <td>절도범죄</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447061</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>1238090</td>\n",
       "      <td>절도범죄</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447062</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>1238011</td>\n",
       "      <td>절도범죄</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447063</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>595821</td>\n",
       "      <td>절도범죄</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447064</th>\n",
       "      <td>/data/GDSC_AudioPoli/testset/Training/orig/[원천...</td>\n",
       "      <td>282214</td>\n",
       "      <td>절도범죄</td>\n",
       "      <td>voice</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>447065 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 filename   target category  \\\n",
       "0       /data/GDSC_AudioPoli/testset/Training/orig/[원천...   226549     전기사고   \n",
       "1       /data/GDSC_AudioPoli/testset/Training/orig/[원천...   220721     전기사고   \n",
       "2       /data/GDSC_AudioPoli/testset/Training/orig/[원천...   232239     전기사고   \n",
       "3       /data/GDSC_AudioPoli/testset/Training/orig/[원천...   234023     전기사고   \n",
       "4       /data/GDSC_AudioPoli/testset/Training/orig/[원천...   222619     전기사고   \n",
       "...                                                   ...      ...      ...   \n",
       "447060  /data/GDSC_AudioPoli/testset/Training/orig/[원천...   594834     절도범죄   \n",
       "447061  /data/GDSC_AudioPoli/testset/Training/orig/[원천...  1238090     절도범죄   \n",
       "447062  /data/GDSC_AudioPoli/testset/Training/orig/[원천...  1238011     절도범죄   \n",
       "447063  /data/GDSC_AudioPoli/testset/Training/orig/[원천...   595821     절도범죄   \n",
       "447064  /data/GDSC_AudioPoli/testset/Training/orig/[원천...   282214     절도범죄   \n",
       "\n",
       "       audio_type  \n",
       "0           voice  \n",
       "1           voice  \n",
       "2           voice  \n",
       "3           voice  \n",
       "4           voice  \n",
       "...           ...  \n",
       "447060      voice  \n",
       "447061      voice  \n",
       "447062      voice  \n",
       "447063      voice  \n",
       "447064      voice  \n",
       "\n",
       "[447065 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_train_dir = \"/data/GDSC_AudioPoli/testset/Training/label/\"\n",
    "base_valid_dir = \"/data/GDSC_AudioPoli/testset/Validation/label/\"\n",
    "base_train_audio = '/data/GDSC_AudioPoli/testset/Training/orig/'\n",
    "base_valid_audio = '/data/GDSC_AudioPoli/testset/Validation/orig/'\n",
    "\n",
    "train_json_files = glob.glob(os.path.join(base_train_dir, '*/*.json'))\n",
    "valid_json_files = glob.glob(os.path.join(base_valid_dir, '*/*.json'))\n",
    "train_orig_files = glob.glob(os.path.join(base_train_audio, '*/*.wav'), recursive=True)\n",
    "valid_orig_files = glob.glob(os.path.join(base_valid_audio, '*/*.wav'), recursive=True)\n",
    "\n",
    "output_csv = '/data/GDSC_AudioPoli/AudioPoli-AI/output.csv'\n",
    "output_valid_csv = '/data/GDSC_AudioPoli/AudioPoli-AI/output_valid.csv'\n",
    "\n",
    "\n",
    "pd_data = pd.read_csv(output_csv)\n",
    "pd_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop columns with lost filenames"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:45.436557Z",
     "iopub.status.busy": "2022-12-14T22:44:45.436283Z",
     "iopub.status.idle": "2022-12-14T22:44:45.450460Z",
     "shell.execute_reply": "2022-12-14T22:44:45.449655Z"
    },
    "id": "tFnEoQjgs14I"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "filename      441519\n",
       "target        441519\n",
       "category      441519\n",
       "audio_type    441519\n",
       "fold          441519\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection\n",
    "\n",
    "output_dropna_csv = '/data/GDSC_AudioPoli/AudioPoli-AI/output_dropna.csv'\n",
    "pd_data = pd.read_csv(output_dropna_csv)\n",
    "\n",
    "my_classes = ['강제추행(성범죄)', '강도범죄', '절도범죄', '폭력범죄',\n",
    "              '화재', '갇힘', '응급의료', '전기사고', '가스사고', '낙상', \n",
    "              '붕괴사고', '태풍-강풍', '지진', '도움요청', '실내', '실외'\n",
    "             ]\n",
    "\"\"\"\n",
    "map_class_to_id = {'강제추행(성범죄)':1, '강도범죄':2, '절도범죄':3, '폭력범죄':4,\n",
    "              '화재':5, '갇힘':6, '응급의료':7, '전기사고':8, '가스사고':9, '낙상':10, \n",
    "              '붕괴사고':11, '태풍-강풍':12, '지진':13, '도움요청':14, '실내':15, '실외':16}\n",
    "\"\"\"\n",
    "map_class_to_id = {'강제추행(성범죄)':0, '강도범죄':1, '절도범죄':2, '폭력범죄':3,\n",
    "              '화재':4, '갇힘':5, '응급의료':6, '전기사고':7, '가스사고':8, '낙상':9, \n",
    "              '붕괴사고':10, '태풍-강풍':11, '지진':12, '도움요청':13, '실내':14, '실외':15}\n",
    "\n",
    "\n",
    "\n",
    "filtered_pd = pd_data[pd_data.category.isin(my_classes)]\n",
    "\n",
    "class_id = filtered_pd['category'].apply(lambda name: map_class_to_id[name])\n",
    "filtered_pd = filtered_pd.assign(category=class_id)\n",
    "\n",
    "# KFold (n = 5)\n",
    "filtered_pd['fold'] = -1\n",
    "kf = model_selection.StratifiedKFold(n_splits = 5)\n",
    "for fold, (trn_, val_) in enumerate(kf.split(X=filtered_pd, y=filtered_pd['category'])):\n",
    "    filtered_pd.loc[val_, 'fold'] = fold\n",
    "\n",
    "# filtered_pd['Fold'].unique()\n",
    "    \n",
    "filtered_pd.head(10)\n",
    "filtered_pd.count()\n",
    "# filtered_pd['fold'].unique()\n",
    "# filtered_pd[filtered_pd.fold == 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Audio file embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:45.454727Z",
     "iopub.status.busy": "2022-12-14T22:44:45.454472Z",
     "iopub.status.idle": "2022-12-14T22:44:45.470162Z",
     "shell.execute_reply": "2022-12-14T22:44:45.469263Z"
    },
    "id": "u5Rq3_PyKLtU"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorSpec(shape=(), dtype=tf.string, name=None),\n",
       " TensorSpec(shape=(), dtype=tf.int64, name=None),\n",
       " TensorSpec(shape=(), dtype=tf.int64, name=None))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filenames = filtered_pd['filename']\n",
    "targets = filtered_pd['category']\n",
    "folds = filtered_pd['fold']\n",
    "\n",
    "main_ds = tf.data.Dataset.from_tensor_slices((filenames, targets, folds))\n",
    "main_ds.element_spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:45.473557Z",
     "iopub.status.busy": "2022-12-14T22:44:45.473298Z",
     "iopub.status.idle": "2022-12-14T22:44:45.630186Z",
     "shell.execute_reply": "2022-12-14T22:44:45.629299Z"
    },
    "id": "rsEfovDVAHGY"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-19 23:19:30.383321: I tensorflow_io/core/kernels/cpu_check.cc:128] Your CPU supports instructions that this TensorFlow IO binary was not compiled to use: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F FMA\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Using a while_loop for converting IO>AudioResample cause there is no registered converter for this op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Using a while_loop for converting IO>AudioResample cause there is no registered converter for this op.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(TensorSpec(shape=<unknown>, dtype=tf.float32, name=None),\n",
       " TensorSpec(shape=(), dtype=tf.int64, name=None),\n",
       " TensorSpec(shape=(), dtype=tf.int64, name=None))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def load_wav_for_map(filename, label, fold):\n",
    "  return load_wav_16k_mono(filename), label, fold\n",
    "\n",
    "main_ds = main_ds.map(load_wav_for_map)\n",
    "main_ds.element_spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:45.633877Z",
     "iopub.status.busy": "2022-12-14T22:44:45.633612Z",
     "iopub.status.idle": "2022-12-14T22:44:45.830830Z",
     "shell.execute_reply": "2022-12-14T22:44:45.829885Z"
    },
    "id": "k0tG8DBNAHcE"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorSpec(shape=(1024,), dtype=tf.float32, name=None),\n",
       " TensorSpec(shape=(), dtype=tf.int64, name=None),\n",
       " TensorSpec(shape=(), dtype=tf.int64, name=None))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# applies the embedding extraction model to a wav data\n",
    "def extract_embedding(wav_data, label, fold):\n",
    "  ''' run YAMNet to extract embedding from the wav data '''\n",
    "  scores, embeddings, spectrogram = yamnet_model(wav_data)\n",
    "  num_embeddings = tf.shape(embeddings)[0]\n",
    "  return (embeddings,\n",
    "            tf.repeat(label, num_embeddings),\n",
    "            tf.repeat(fold, num_embeddings))\n",
    "\n",
    "# extract embedding\n",
    "main_ds = main_ds.map(extract_embedding).unbatch()\n",
    "main_ds.element_spec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZdfPIeD0Qedk"
   },
   "source": [
    "# K-Fold\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:45.835038Z",
     "iopub.status.busy": "2022-12-14T22:44:45.834367Z",
     "iopub.status.idle": "2022-12-14T22:44:45.923476Z",
     "shell.execute_reply": "2022-12-14T22:44:45.922745Z"
    },
    "id": "1ZYvlFiVsffC"
   },
   "outputs": [],
   "source": [
    "cached_ds = main_ds.cache()\n",
    "train_ds = cached_ds.filter(lambda embedding, label, fold: fold < 3)\n",
    "val_ds = cached_ds.filter(lambda embedding, label, fold: fold == 3)\n",
    "test_ds = cached_ds.filter(lambda embedding, label, fold: fold == 4)\n",
    "\n",
    "# remove the folds column now that it's not needed anymore\n",
    "remove_fold_column = lambda embedding, label, fold: (embedding, label)\n",
    "\n",
    "train_ds = train_ds.map(remove_fold_column)\n",
    "val_ds = val_ds.map(remove_fold_column)\n",
    "test_ds = test_ds.map(remove_fold_column)\n",
    "\n",
    "train_ds = train_ds.cache().shuffle(1000).batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "val_ds = val_ds.cache().batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "test_ds = test_ds.cache().batch(32).prefetch(tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "v5PaMwvtcAIe"
   },
   "source": [
    "# Model for embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, models, regularizers\n",
    "from tensorflow.keras.callbacks import Callback\n",
    "\n",
    "class ClearMemory(Callback):\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        gc.collect()\n",
    "        print(\"Cleaning garbage\")\n",
    "        tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyModel:\n",
    "    def __init__(self, input_shape, num_classes):\n",
    "        self.input_shape = input_shape\n",
    "        self.num_classes = num_classes\n",
    "        self.model = self.build_model()\n",
    "    \n",
    "    def build_model(self):\n",
    "        input = layers.Input(shape=self.input_shape)\n",
    "        x = layers.Dense(256, activation='relu', kernel_regularizer=regularizers.l2(0.001))(input)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = layers.Dropout(0.5)(x)\n",
    "        x = layers.Dense(256, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = layers.Dropout(0.5)(x)\n",
    "        x = layers.Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = layers.Dropout(0.5)(x)\n",
    "        x = layers.Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = layers.Dropout(0.5)(x)\n",
    "        x = layers.Dense(64, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = layers.Dropout(0.5)(x)\n",
    "        x = layers.Dense(64, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = layers.Dropout(0.5)(x)\n",
    "        x = layers.Dense(self.num_classes)(x)\n",
    "        return models.Model(inputs=input, outputs=x)\n",
    "\n",
    "    def compile_model(self):\n",
    "        self.model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "                           optimizer=\"adam\",\n",
    "                           metrics=['accuracy'])\n",
    "\n",
    "    def fit_model(self, train_ds, val_ds, epochs=10, batch_size=32):\n",
    "        callback = tf.keras.callbacks.EarlyStopping(monitor='loss',\n",
    "                                                    patience=3,\n",
    "                                                    restore_best_weights=True)\n",
    "        \n",
    "        history = self.model.fit(train_ds,\n",
    "                                 epochs=epochs,\n",
    "                                 batch_size=batch_size,\n",
    "                                 validation_data=val_ds,\n",
    "                                 callbacks=[ClearMemory(), callback],\n",
    "                                 steps_per_epoch=folds.count()//batch_size\n",
    "                                )\n",
    "        \n",
    "        \"\"\"\n",
    "        history = self.model.fit(train_ds,\n",
    "                                 epochs=epochs,\n",
    "                                 validation_data=val_ds,\n",
    "                                 callbacks=[ClearMemory(), callback],\n",
    "                                 use_multiprocessing=True,\n",
    "                                 workers=20,\n",
    "                                 steps_per_epoch=folds.count()//batch_size\n",
    "                                )\n",
    "        \"\"\"\n",
    "        return history\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n",
      "Epoch 1/10\n",
      "  262/13797 [..............................] - ETA: 5:59 - loss: 3.0701 - accuracy: 0.4800"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[13], line 6\u001b[0m\n\u001b[1;32m      4\u001b[0m my_model \u001b[38;5;241m=\u001b[39m MyModel(input_shape, num_classes)\n\u001b[1;32m      5\u001b[0m my_model\u001b[38;5;241m.\u001b[39mcompile_model()\n\u001b[0;32m----> 6\u001b[0m \u001b[43mmy_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_ds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_ds\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[12], line 40\u001b[0m, in \u001b[0;36mMyModel.fit_model\u001b[0;34m(self, train_ds, val_ds, epochs, batch_size)\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit_model\u001b[39m(\u001b[38;5;28mself\u001b[39m, train_ds, val_ds, epochs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m, batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m32\u001b[39m):\n\u001b[1;32m     36\u001b[0m     callback \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mEarlyStopping(monitor\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mloss\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m     37\u001b[0m                                                 patience\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m3\u001b[39m,\n\u001b[1;32m     38\u001b[0m                                                 restore_best_weights\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m---> 40\u001b[0m     history \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_ds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     41\u001b[0m \u001b[43m                             \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     42\u001b[0m \u001b[43m                             \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     43\u001b[0m \u001b[43m                             \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mval_ds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     44\u001b[0m \u001b[43m                             \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mClearMemory\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     45\u001b[0m \u001b[43m                             \u001b[49m\u001b[43msteps_per_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfolds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcount\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[43mbatch_size\u001b[49m\n\u001b[1;32m     46\u001b[0m \u001b[43m                            \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     48\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m     49\u001b[0m \u001b[38;5;124;03m    history = self.model.fit(train_ds,\u001b[39;00m\n\u001b[1;32m     50\u001b[0m \u001b[38;5;124;03m                             epochs=epochs,\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;124;03m                            )\u001b[39;00m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m     58\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m history\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/keras/src/utils/traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/keras/src/engine/training.py:1807\u001b[0m, in \u001b[0;36mModel.fit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1799\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tf\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mexperimental\u001b[38;5;241m.\u001b[39mTrace(\n\u001b[1;32m   1800\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   1801\u001b[0m     epoch_num\u001b[38;5;241m=\u001b[39mepoch,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1804\u001b[0m     _r\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[1;32m   1805\u001b[0m ):\n\u001b[1;32m   1806\u001b[0m     callbacks\u001b[38;5;241m.\u001b[39mon_train_batch_begin(step)\n\u001b[0;32m-> 1807\u001b[0m     tmp_logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43miterator\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1808\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mshould_sync:\n\u001b[1;32m   1809\u001b[0m         context\u001b[38;5;241m.\u001b[39masync_wait()\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/util/traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:832\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    829\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    831\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[0;32m--> 832\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    834\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[1;32m    835\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:868\u001b[0m, in \u001b[0;36mFunction._call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    865\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[1;32m    866\u001b[0m   \u001b[38;5;66;03m# In this case we have created variables on the first call, so we run the\u001b[39;00m\n\u001b[1;32m    867\u001b[0m   \u001b[38;5;66;03m# defunned version which is guaranteed to never create variables.\u001b[39;00m\n\u001b[0;32m--> 868\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtracing_compilation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_function\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    869\u001b[0m \u001b[43m      \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_no_variable_creation_config\u001b[49m\n\u001b[1;32m    870\u001b[0m \u001b[43m  \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    871\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_variable_creation_config \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    872\u001b[0m   \u001b[38;5;66;03m# Release the lock early so that multiple threads can perform the call\u001b[39;00m\n\u001b[1;32m    873\u001b[0m   \u001b[38;5;66;03m# in parallel.\u001b[39;00m\n\u001b[1;32m    874\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compilation.py:139\u001b[0m, in \u001b[0;36mcall_function\u001b[0;34m(args, kwargs, tracing_options)\u001b[0m\n\u001b[1;32m    137\u001b[0m bound_args \u001b[38;5;241m=\u001b[39m function\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39mbind(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    138\u001b[0m flat_inputs \u001b[38;5;241m=\u001b[39m function\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39munpack_inputs(bound_args)\n\u001b[0;32m--> 139\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_flat\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# pylint: disable=protected-access\u001b[39;49;00m\n\u001b[1;32m    140\u001b[0m \u001b[43m    \u001b[49m\u001b[43mflat_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptured_inputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfunction\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcaptured_inputs\u001b[49m\n\u001b[1;32m    141\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/eager/polymorphic_function/concrete_function.py:1323\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[0;34m(self, tensor_inputs, captured_inputs)\u001b[0m\n\u001b[1;32m   1319\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[1;32m   1320\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[1;32m   1321\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[1;32m   1322\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[0;32m-> 1323\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inference_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_preflattened\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1324\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[1;32m   1325\u001b[0m     args,\n\u001b[1;32m   1326\u001b[0m     possible_gradient_type,\n\u001b[1;32m   1327\u001b[0m     executing_eagerly)\n\u001b[1;32m   1328\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py:216\u001b[0m, in \u001b[0;36mAtomicFunction.call_preflattened\u001b[0;34m(self, args)\u001b[0m\n\u001b[1;32m    214\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcall_preflattened\u001b[39m(\u001b[38;5;28mself\u001b[39m, args: Sequence[core\u001b[38;5;241m.\u001b[39mTensor]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[1;32m    215\u001b[0m \u001b[38;5;250m  \u001b[39m\u001b[38;5;124;03m\"\"\"Calls with flattened tensor inputs and returns the structured output.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 216\u001b[0m   flat_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_flat\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    217\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39mpack_output(flat_outputs)\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py:251\u001b[0m, in \u001b[0;36mAtomicFunction.call_flat\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m    249\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m record\u001b[38;5;241m.\u001b[39mstop_recording():\n\u001b[1;32m    250\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mexecuting_eagerly():\n\u001b[0;32m--> 251\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_bound_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_function\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    252\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    253\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    254\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunction_type\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mflat_outputs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    255\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    256\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    257\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m make_call_op_in_graph(\n\u001b[1;32m    258\u001b[0m         \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    259\u001b[0m         \u001b[38;5;28mlist\u001b[39m(args),\n\u001b[1;32m    260\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mfunction_call_options\u001b[38;5;241m.\u001b[39mas_attrs(),\n\u001b[1;32m    261\u001b[0m     )\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/eager/context.py:1486\u001b[0m, in \u001b[0;36mContext.call_function\u001b[0;34m(self, name, tensor_inputs, num_outputs)\u001b[0m\n\u001b[1;32m   1484\u001b[0m cancellation_context \u001b[38;5;241m=\u001b[39m cancellation\u001b[38;5;241m.\u001b[39mcontext()\n\u001b[1;32m   1485\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m cancellation_context \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1486\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m \u001b[43mexecute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1487\u001b[0m \u001b[43m      \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mutf-8\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1488\u001b[0m \u001b[43m      \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1489\u001b[0m \u001b[43m      \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtensor_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1490\u001b[0m \u001b[43m      \u001b[49m\u001b[43mattrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1491\u001b[0m \u001b[43m      \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1492\u001b[0m \u001b[43m  \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1493\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1494\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[1;32m   1495\u001b[0m       name\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m   1496\u001b[0m       num_outputs\u001b[38;5;241m=\u001b[39mnum_outputs,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1500\u001b[0m       cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_context,\n\u001b[1;32m   1501\u001b[0m   )\n",
      "File \u001b[0;32m~/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/tensorflow/python/eager/execute.py:53\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     51\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     52\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 53\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_Execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     54\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     55\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     56\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "input_shape = (1024,)\n",
    "num_classes = len(my_classes)\n",
    "print(num_classes)\n",
    "my_model = MyModel(input_shape, num_classes)\n",
    "my_model.compile_model()\n",
    "my_model.fit_model(train_ds, val_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:52.703812Z",
     "iopub.status.busy": "2022-12-14T22:44:52.703135Z",
     "iopub.status.idle": "2022-12-14T22:44:52.884718Z",
     "shell.execute_reply": "2022-12-14T22:44:52.883800Z"
    },
    "id": "H4Nh5nec3Sky"
   },
   "outputs": [],
   "source": [
    "# Check for overfittings\n",
    "\"\"\"\n",
    "loss, accuracy = my_model.evaluate(test_ds)\n",
    "\n",
    "print(\"Loss: \", loss)\n",
    "print(\"Accuracy: \", accuracy)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check with wav metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check whether problem due to float32 encoding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert float32 to int16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### UnicodeDecodeError issue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nCKZonrJcXab"
   },
   "source": [
    "# Testing model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:52.888572Z",
     "iopub.status.busy": "2022-12-14T22:44:52.888033Z",
     "iopub.status.idle": "2022-12-14T22:44:52.902066Z",
     "shell.execute_reply": "2022-12-14T22:44:52.901366Z"
    },
    "id": "79AFpA3_ctCF"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/GDSC_AudioPoli/testset/Training/orig/[원천]2.강도범죄_1/2.강도범죄_422517_label.wav\n",
      "WARNING:tensorflow:Using a while_loop for converting IO>AudioResample cause there is no registered converter for this op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Using a while_loop for converting IO>AudioResample cause there is no registered converter for this op.\n"
     ]
    }
   ],
   "source": [
    "test_pd = filtered_pd.loc[filtered_pd['fold'] == 4]\n",
    "# print(test_pd)\n",
    "row = test_pd.sample(1)\n",
    "filename = row['filename'].item()\n",
    "print(filename)\n",
    "tmp = load_wav_16k_mono(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scores, embeddings, spectrogram = yamnet_model(testing_wav_data)\n",
    "scores, embeddings, spectrogram = yamnet_model(tmp)\n",
    "\"\"\"\n",
    "result = my_model(embeddings).numpy()\n",
    "\n",
    "inferred_class = my_classes[result.mean(axis=0).argmax()]\n",
    "print(f'The main sound is: {inferred_class}')\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "k2yleeev645r"
   },
   "source": [
    "### Pipelines for wave-input model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:52.905643Z",
     "iopub.status.busy": "2022-12-14T22:44:52.905070Z",
     "iopub.status.idle": "2022-12-14T22:44:52.909296Z",
     "shell.execute_reply": "2022-12-14T22:44:52.908664Z"
    },
    "id": "QUVCI2Suunpw"
   },
   "outputs": [],
   "source": [
    "class ReduceMeanLayer(tf.keras.layers.Layer):\n",
    "  def __init__(self, axis=0, **kwargs):\n",
    "    super(ReduceMeanLayer, self).__init__(**kwargs)\n",
    "    self.axis = axis\n",
    "\n",
    "  def call(self, input):\n",
    "    return tf.math.reduce_mean(input, axis=self.axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:44:52.912518Z",
     "iopub.status.busy": "2022-12-14T22:44:52.911964Z",
     "iopub.status.idle": "2022-12-14T22:45:03.303892Z",
     "shell.execute_reply": "2022-12-14T22:45:03.303059Z"
    },
    "id": "zE_Npm0nzlwc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jihwan/miniconda3/envs/AudioPoli/lib/python3.9/site-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
      "  saving_api.save_model(\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    }
   ],
   "source": [
    "input_segment = tf.keras.layers.Input(shape=(), dtype=tf.float32, name='audio')\n",
    "embedding_extraction_layer = hub.KerasLayer(yamnet_model_handle,\n",
    "                                            trainable=False, name='yamnet')\n",
    "_, embeddings_output, _ = embedding_extraction_layer(input_segment)\n",
    "serving_outputs = my_model(embeddings_output)\n",
    "serving_outputs = ReduceMeanLayer(axis=0, name='classifier')(serving_outputs)\n",
    "serving_model = tf.keras.Model(input_segment, serving_outputs)\n",
    "serving_model.save(saved_model_path, include_optimizer=False)\n",
    "\n",
    "\n",
    "# saved_model_path = './seq_model'\n",
    "saved_model_path = 'seq_model.h5'\n",
    "\n",
    "input_segment = tf.keras.layers.Input(shape=(), dtype=tf.float32, name='audio')\n",
    "embedding_extraction_layer = hub.KerasLayer(yamnet_model_handle,\n",
    "                                            trainable=False, name='yamnet')\n",
    "seq_model = load_model(saved_model_path)\n",
    "_, embeddings_output, _ = embedding_extraction_layer(input_segment)\n",
    "serving_outputs = seq_model(embeddings_output)\n",
    "serving_outputs = ReduceMeanLayer(axis=0, name='classifier')(serving_outputs)\n",
    "serving_model = tf.keras.Model(input_segment, serving_outputs)\n",
    "serving_model.save(saved_model_path, include_optimizer=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:45:03.308519Z",
     "iopub.status.busy": "2022-12-14T22:45:03.307864Z",
     "iopub.status.idle": "2022-12-14T22:45:03.555639Z",
     "shell.execute_reply": "2022-12-14T22:45:03.554686Z"
    },
    "id": "y-0bY5FMme1C"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You must install pydot (`pip install pydot`) and install graphviz (see instructions at https://graphviz.gitlab.io/download/) for plot_model to work.\n"
     ]
    }
   ],
   "source": [
    "tf.keras.utils.plot_model(serving_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:45:03.559869Z",
     "iopub.status.busy": "2022-12-14T22:45:03.559155Z",
     "iopub.status.idle": "2022-12-14T22:45:08.592414Z",
     "shell.execute_reply": "2022-12-14T22:45:08.591570Z"
    },
    "id": "KkYVpJS72WWB"
   },
   "outputs": [],
   "source": [
    "reloaded_model = tf.saved_model.load(saved_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:45:08.597263Z",
     "iopub.status.busy": "2022-12-14T22:45:08.596539Z",
     "iopub.status.idle": "2022-12-14T22:45:08.903117Z",
     "shell.execute_reply": "2022-12-14T22:45:08.902404Z"
    },
    "id": "xeXtD5HO28y-"
   },
   "outputs": [],
   "source": [
    "reloaded_results = reloaded_model(testing_wav_data)\n",
    "cat_or_dog = my_classes[tf.math.argmax(reloaded_results)]\n",
    "print(f'The main sound is: {cat_or_dog}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:45:08.906899Z",
     "iopub.status.busy": "2022-12-14T22:45:08.906335Z",
     "iopub.status.idle": "2022-12-14T22:45:09.137467Z",
     "shell.execute_reply": "2022-12-14T22:45:09.136665Z"
    },
    "id": "ycC8zzDSUG2s"
   },
   "outputs": [],
   "source": [
    "serving_results = reloaded_model.signatures['serving_default'](testing_wav_data)\n",
    "cat_or_dog = my_classes[tf.math.argmax(serving_results['classifier'])]\n",
    "print(f'The main sound is: {cat_or_dog}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "da7blblCHs8c"
   },
   "source": [
    "# Comparison with YAMNet models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:45:09.141514Z",
     "iopub.status.busy": "2022-12-14T22:45:09.140850Z",
     "iopub.status.idle": "2022-12-14T22:45:09.603844Z",
     "shell.execute_reply": "2022-12-14T22:45:09.603100Z"
    },
    "id": "vDf5MASIIN1z"
   },
   "outputs": [],
   "source": [
    "test_pd = filtered_pd.loc[filtered_pd['fold'] == 4]\n",
    "# print(test_pd)\n",
    "row = test_pd.sample(1)\n",
    "filename = row['filename'].item()\n",
    "print(filename)\n",
    "waveform = load_wav_16k_mono(filename)\n",
    "print(f'Waveform values: {waveform}')\n",
    "_ = plt.plot(waveform)\n",
    "\n",
    "display.Audio(waveform, rate=16000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-14T22:45:09.607964Z",
     "iopub.status.busy": "2022-12-14T22:45:09.607393Z",
     "iopub.status.idle": "2022-12-14T22:45:09.988677Z",
     "shell.execute_reply": "2022-12-14T22:45:09.987925Z"
    },
    "id": "eYUzFxYJIcE1"
   },
   "outputs": [],
   "source": [
    "# Run the model, check the output.\n",
    "scores, embeddings, spectrogram = yamnet_model(waveform)\n",
    "class_scores = tf.reduce_mean(scores, axis=0)\n",
    "top_class = tf.math.argmax(class_scores)\n",
    "inferred_class = class_names[top_class]\n",
    "top_score = class_scores[top_class]\n",
    "print(f'[YAMNet] The main sound is: {inferred_class} ({top_score})')\n",
    "\n",
    "\"\"\"\n",
    "reloaded_results = reloaded_model(waveform)\n",
    "your_top_class = tf.math.argmax(reloaded_results)\n",
    "your_inferred_class = my_classes[your_top_class]\n",
    "class_probabilities = tf.nn.softmax(reloaded_results, axis=-1)\n",
    "your_top_score = class_probabilities[your_top_class]\n",
    "print(f'[Your model] The main sound is: {your_inferred_class} ({your_top_score})')\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "transfer_learning_audio.ipynb",
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "AudioPoli",
   "language": "python",
   "name": "audiopoli"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}