clean_notebooks/ExampleDataLoading/loadingRNAseqByGene.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### download data and packages "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "pip install synapseclient pandas\n",
    "####only download one file for now\n",
    "#\n",
    "mkdir tmp_data/\n",
    "cd ./tmp_data/\n",
    "#created a dummy accounts so that any one can download without registering, please don't do anything crazy with the account\n",
    "synapse -u synapse.skymap.download -p QtL-E2g-hzz-N4k get  syn11421664\n",
    "synapse -u synapse.skymap.download -p QtL-E2g-hzz-N4k get  syn11421653\n",
    "synapse -u synapse.skymap.download -p QtL-E2g-hzz-N4k get  syn11421663\n",
    "\n",
    "#syn11421653 syn11421663\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### slice "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseDir='./tmp_data/'\n",
    "\n",
    "def loadDf(fname,mmap_mode='r'):\n",
    "    with open(fname+'.index.txt') as f:\n",
    "        myIndex=map(lambda s:s.replace(\"\\n\",\"\"), f.readlines())\n",
    "    with open(fname+'.columns.txt') as f:\n",
    "        myColumns=map(lambda s:s.replace(\"\\n\",\"\"), f.readlines())\n",
    "    tmpMatrix=np.load(fname+\".npy\",mmap_mode=mmap_mode)\n",
    "    tmpDf=pd.DataFrame(tmpMatrix,index=myIndex,columns=myColumns)\n",
    "    tmpDf.columns.name='Run'\n",
    "    return tmpDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "expression_metric='TPM'\n",
    "data_matrix_dir_fmt=baseDir+'/hgGRC38__allSRAmatrix.realign.v9.base.{feature}.gene.symbol'.format(feature=expression_metric)\n",
    "#a memory mapped dataframe\n",
    "rnaseqDf=loadDf(data_matrix_dir_fmt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "layout of expression matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Run</th>\n",
       "      <th>SRR4456480</th>\n",
       "      <th>SRR4456481</th>\n",
       "      <th>SRR4456482</th>\n",
       "      <th>SRR4456483</th>\n",
       "      <th>SRR4456484</th>\n",
       "      <th>SRR4456485</th>\n",
       "      <th>SRR4456486</th>\n",
       "      <th>SRR4456487</th>\n",
       "      <th>SRR4456488</th>\n",
       "      <th>SRR4456489</th>\n",
       "      <th>...</th>\n",
       "      <th>SRR3126580</th>\n",
       "      <th>SRR3126581</th>\n",
       "      <th>SRR5076964</th>\n",
       "      <th>SRR5076965</th>\n",
       "      <th>SRR5076966</th>\n",
       "      <th>SRR5076967</th>\n",
       "      <th>SRR5076960</th>\n",
       "      <th>SRR5076961</th>\n",
       "      <th>SRR5076962</th>\n",
       "      <th>SRR3126589</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A1BG</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43.4725</td>\n",
       "      <td>42.39500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>13.633400</td>\n",
       "      <td>16.613701</td>\n",
       "      <td>30.397053</td>\n",
       "      <td>38.792152</td>\n",
       "      <td>28.311970</td>\n",
       "      <td>41.916702</td>\n",
       "      <td>21.123585</td>\n",
       "      <td>43.892700</td>\n",
       "      <td>27.982466</td>\n",
       "      <td>10.015800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A1CF</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.008490</td>\n",
       "      <td>0.731485</td>\n",
       "      <td>0.522371</td>\n",
       "      <td>0.619676</td>\n",
       "      <td>1.073180</td>\n",
       "      <td>1.200200</td>\n",
       "      <td>1.149905</td>\n",
       "      <td>0.359585</td>\n",
       "      <td>2.151589</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A2M</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.68203</td>\n",
       "      <td>9.63777</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.2231</td>\n",
       "      <td>15.0466</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.022370</td>\n",
       "      <td>2.237999</td>\n",
       "      <td>1.686480</td>\n",
       "      <td>1.736842</td>\n",
       "      <td>2.459877</td>\n",
       "      <td>2.030084</td>\n",
       "      <td>2.870860</td>\n",
       "      <td>1.394481</td>\n",
       "      <td>0.147562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A2ML1</th>\n",
       "      <td>90.239403</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>4.50392</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.383746</td>\n",
       "      <td>0.222499</td>\n",
       "      <td>0.738123</td>\n",
       "      <td>1.604510</td>\n",
       "      <td>1.749147</td>\n",
       "      <td>1.575887</td>\n",
       "      <td>0.630090</td>\n",
       "      <td>0.466597</td>\n",
       "      <td>0.836070</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A2MP1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.862630</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 126064 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Run    SRR4456480  SRR4456481  SRR4456482  SRR4456483  SRR4456484  SRR4456485  \\\n",
       "A1BG     0.000000     0.00000     0.00000         0.0         0.0     43.4725   \n",
       "A1CF     0.000000     0.00000     0.00000         0.0         0.0      0.0000   \n",
       "A2M      0.000000     9.68203     9.63777         0.0         0.0      0.0000   \n",
       "A2ML1   90.239403     0.00000     0.00000         0.0         0.0      0.0000   \n",
       "A2MP1    0.000000     0.00000     0.00000         0.0         0.0      0.0000   \n",
       "\n",
       "Run    SRR4456486  SRR4456487  SRR4456488  SRR4456489     ...      SRR3126580  \\\n",
       "A1BG     42.39500         0.0      0.0000      0.0000     ...       13.633400   \n",
       "A1CF      0.00000         0.0      0.0000      0.0000     ...        1.008490   \n",
       "A2M       0.00000         0.0     10.2231     15.0466     ...        0.000000   \n",
       "A2ML1     4.50392         0.0      0.0000      0.0000     ...        0.383746   \n",
       "A2MP1     0.00000         0.0      0.0000      0.0000     ...        0.000000   \n",
       "\n",
       "Run    SRR3126581  SRR5076964  SRR5076965  SRR5076966  SRR5076967  SRR5076960  \\\n",
       "A1BG    16.613701   30.397053   38.792152   28.311970   41.916702   21.123585   \n",
       "A1CF     0.731485    0.522371    0.619676    1.073180    1.200200    1.149905   \n",
       "A2M      1.022370    2.237999    1.686480    1.736842    2.459877    2.030084   \n",
       "A2ML1    0.222499    0.738123    1.604510    1.749147    1.575887    0.630090   \n",
       "A2MP1    0.000000    0.000000    1.862630    0.000000    0.000000    0.000000   \n",
       "\n",
       "Run    SRR5076961  SRR5076962  SRR3126589  \n",
       "A1BG    43.892700   27.982466   10.015800  \n",
       "A1CF     0.359585    2.151589    0.000000  \n",
       "A2M      2.870860    1.394481    0.147562  \n",
       "A2ML1    0.466597    0.836070    0.000000  \n",
       "A2MP1    0.000000    0.000000    0.000000  \n",
       "\n",
       "[5 rows x 126064 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnaseqDf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8 ms, sys: 4 ms, total: 12 ms\n",
      "Wall time: 12.7 ms\n"
     ]
    }
   ],
   "source": [
    "%time hitGeneS=rnaseqDf.loc['TP53']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TP53 TPM level across 126064 RNAseq human samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "126064"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hitGeneS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Run\n",
       "SRR4456480      0.000000\n",
       "SRR4456481     26.543200\n",
       "SRR4456482     25.031300\n",
       "SRR4456483      0.000000\n",
       "SRR4456484      0.000000\n",
       "SRR4456485      0.000000\n",
       "SRR4456486     14.487200\n",
       "SRR4456487      0.000000\n",
       "SRR4456488      0.000000\n",
       "SRR4456489      0.000000\n",
       "SRR934728      17.815222\n",
       "SRR2105337      0.000000\n",
       "SRR2105336      0.000000\n",
       "SRR934725      25.889841\n",
       "SRR2964299      0.000000\n",
       "SRR2964298      0.000000\n",
       "SRR2105334      0.000000\n",
       "SRR4456622     23.447300\n",
       "SRR2964291    452.062012\n",
       "SRR2964290    493.393005\n",
       "SRR2964293      0.000000\n",
       "SRR2964292      0.000000\n",
       "SRR2964295     14.198820\n",
       "SRR2964294      0.000000\n",
       "SRR2964297      7.126470\n",
       "SRR934722      29.623505\n",
       "SRR2735800      0.000000\n",
       "SRR2735801     33.014301\n",
       "SRR2735802     49.995998\n",
       "SRR1782990     35.842403\n",
       "Name: TP53, dtype: float32"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hitGeneS.head(n=30)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}