2.sequential-compression/execution-time-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the Execution Time of Compression Algorithms\n",
    "\n",
    "**Gregory Way, 2019**\n",
    "\n",
    "Observing the execution time of algorithms and latent dimensionalities across datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import time\n",
    "\n",
    "from tybalt.data_models import DataModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.logging.set_verbosity(tf.logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup constants\n",
    "datasets = [\"TARGET\", \"TCGA\", \"GTEX\"]\n",
    "ks = [2, 4, 10, 16, 25, 50, 80, 200]\n",
    "subset_mad_genes = 8000\n",
    "data_dir = os.path.join('..', '0.expression-download', 'data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing TARGET: k = 2\n",
      "Timing TARGET: k = 4\n",
      "Timing TARGET: k = 10\n",
      "Timing TARGET: k = 16\n",
      "Timing TARGET: k = 25\n",
      "Timing TARGET: k = 50\n",
      "Timing TARGET: k = 80\n",
      "Timing TARGET: k = 200\n",
      "Timing TCGA: k = 2\n",
      "Timing TCGA: k = 4\n",
      "Timing TCGA: k = 10\n",
      "Timing TCGA: k = 16\n",
      "Timing TCGA: k = 25\n",
      "Timing TCGA: k = 50\n",
      "Timing TCGA: k = 80\n",
      "Timing TCGA: k = 200\n",
      "Timing GTEX: k = 2\n",
      "Timing GTEX: k = 4\n",
      "Timing GTEX: k = 10\n",
      "Timing GTEX: k = 16\n",
      "Timing GTEX: k = 25\n",
      "Timing GTEX: k = 50\n",
      "Timing GTEX: k = 80\n",
      "Timing GTEX: k = 200\n"
     ]
    }
   ],
   "source": [
    "time_results = []\n",
    "for dataset in datasets:\n",
    "    \n",
    "    # Setup constants per dataset\n",
    "    dataset_id = dataset.lower()\n",
    "    param_config = os.path.join(\"config\", \"z_parameter_sweep_{}.tsv\".format(dataset))\n",
    "\n",
    "    # For extracting parameters from optimized parameter configuation file\n",
    "    param_df = pd.read_table(param_config, index_col=0)\n",
    "\n",
    "    # Load MAD genes per dataset\n",
    "    mad_file = os.path.join(data_dir, '{}_mad_genes.tsv'.format(dataset_id))\n",
    "    mad_genes_df = pd.read_table(mad_file)\n",
    "    mad_genes = mad_genes_df.iloc[0:subset_mad_genes, ].gene_id.astype(str)\n",
    "    \n",
    "    # Load input data per dataset and reindex to mad genes\n",
    "    train_file = os.path.join(data_dir,\n",
    "                              'train_{}_expression_matrix_processed.tsv.gz'.format(dataset_id))\n",
    "    test_file = os.path.join(data_dir,\n",
    "                             'test_{}_expression_matrix_processed.tsv.gz'.format(dataset_id))\n",
    "    rnaseq_train_df = pd.read_table(train_file, index_col=0).reindex(mad_genes, axis='columns')\n",
    "    rnaseq_test_df = pd.read_table(test_file, index_col=0).reindex(mad_genes, axis='columns')\n",
    "    \n",
    "    # Initialize DataModel class with the input data\n",
    "    dm = DataModel(df=rnaseq_train_df, test_df=rnaseq_test_df)\n",
    "    dm.transform(how='zeroone')\n",
    "    \n",
    "    # Loop over the latent dimensionalities\n",
    "    for k in ks:\n",
    "        print(\"Timing {}: k = {}\".format(dataset, k))\n",
    "\n",
    "        # Retrieve optimized parameters for neural network models\n",
    "        vae_epochs = param_df.loc['vae_epochs', str(k)]\n",
    "        dae_epochs = param_df.loc['dae_epochs', str(k)]\n",
    "        vae_lr = param_df.loc['vae_lr', str(k)]\n",
    "        dae_lr = param_df.loc['dae_lr', str(k)]\n",
    "        vae_batch_size = param_df.loc['vae_batch_size', str(k)]\n",
    "        dae_batch_size = param_df.loc['dae_batch_size', str(k)]\n",
    "        dae_noise = param_df.loc['dae_noise', str(k)]\n",
    "        dae_sparsity = param_df.loc['dae_sparsity', str(k)]\n",
    "        vae_kappa = param_df.loc['vae_kappa', str(k)]\n",
    "        \n",
    "        # Fit models\n",
    "        # 1) PCA\n",
    "        start = time.time()\n",
    "        dm.pca(n_components=k, transform_test_df=False)\n",
    "        end = time.time()\n",
    "        total_time = end - start\n",
    "        \n",
    "        result = [dataset, k, \"PCA\", total_time]\n",
    "        time_results.append(result)\n",
    "        \n",
    "        # 2) ICA\n",
    "        start = time.time()\n",
    "        dm.ica(n_components=k, transform_test_df=False)\n",
    "        end = time.time()\n",
    "        total_time = end - start\n",
    "        \n",
    "        result = [dataset, k, \"ICA\", total_time]\n",
    "        time_results.append(result)\n",
    "        \n",
    "        # 3) NMF\n",
    "        start = time.time()\n",
    "        dm.nmf(n_components=k, transform_test_df=False)\n",
    "        end = time.time()\n",
    "        total_time = end - start\n",
    "        \n",
    "        result = [dataset, k, \"NMF\", total_time]\n",
    "        time_results.append(result)\n",
    "        \n",
    "        # 4) DAE\n",
    "        start = time.time()\n",
    "        dm.nn(n_components=k,\n",
    "              model='adage',\n",
    "              loss='binary_crossentropy',\n",
    "              epochs=int(dae_epochs),\n",
    "              batch_size=int(dae_batch_size),\n",
    "              learning_rate=float(dae_lr),\n",
    "              noise=float(dae_noise),\n",
    "              sparsity=float(dae_sparsity),\n",
    "              verbose=False,\n",
    "              transform_test_df=False)\n",
    "        end = time.time()\n",
    "        total_time = end - start\n",
    "        \n",
    "        result = [dataset, k, \"DAE\", total_time]\n",
    "        time_results.append(result)\n",
    "\n",
    "        # 4) VAE\n",
    "        start = time.time()\n",
    "        dm.nn(n_components=k,\n",
    "              model='tybalt',\n",
    "              loss='binary_crossentropy',\n",
    "              epochs=int(vae_epochs),\n",
    "              batch_size=int(vae_batch_size),\n",
    "              learning_rate=float(vae_lr),\n",
    "              separate_loss=True,\n",
    "              verbose=False,\n",
    "              transform_test_df=False)\n",
    "        end = time.time()\n",
    "        total_time = end - start\n",
    "        \n",
    "        result = [dataset, k, \"VAE\", total_time]\n",
    "        time_results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(120, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>k</th>\n",
       "      <th>algorithm</th>\n",
       "      <th>seconds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>GTEX</td>\n",
       "      <td>200</td>\n",
       "      <td>VAE</td>\n",
       "      <td>1787.446866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>GTEX</td>\n",
       "      <td>80</td>\n",
       "      <td>VAE</td>\n",
       "      <td>1212.493266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>TCGA</td>\n",
       "      <td>200</td>\n",
       "      <td>VAE</td>\n",
       "      <td>1069.016813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>GTEX</td>\n",
       "      <td>50</td>\n",
       "      <td>VAE</td>\n",
       "      <td>940.811872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>GTEX</td>\n",
       "      <td>25</td>\n",
       "      <td>VAE</td>\n",
       "      <td>862.166329</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    dataset    k algorithm      seconds\n",
       "119    GTEX  200       VAE  1787.446866\n",
       "114    GTEX   80       VAE  1212.493266\n",
       "79     TCGA  200       VAE  1069.016813\n",
       "109    GTEX   50       VAE   940.811872\n",
       "104    GTEX   25       VAE   862.166329"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Combine and output the time analysis results\n",
    "time_results_df = (\n",
    "    pd.DataFrame(time_results,\n",
    "                 columns=[\"dataset\", \"k\", \"algorithm\", \"seconds\"])\n",
    "    .sort_values(\"seconds\", ascending=False)\n",
    ")\n",
    "\n",
    "time_file = os.path.join(\"results\", \"time_analysis_results.tsv\")\n",
    "time_results_df.to_csv(time_file, sep='\\t', index=False)\n",
    "\n",
    "print(time_results_df.shape)\n",
    "time_results_df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:biobombe]",
   "language": "python",
   "name": "conda-env-biobombe-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}