BioModelIPython.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 运行函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import gc\n",
    "import os\n",
    "import re\n",
    "import math\n",
    "import sys\n",
    "import random\n",
    "from itertools import islice\n",
    "import time\n",
    "from scipy.sparse import csr_matrix\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import bokeh\n",
    "import scipy as sp\n",
    "from scipy import stats\n",
    "from sklearn.feature_selection import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from module.module import *\n",
    "from module.process import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "T=trainer(5,-1)\n",
    "data = T.MAIN_SINGLE_TEST(0,4,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = []\n",
    "res.extend(data[0][0])\n",
    "\n",
    "arr = [0 for i in range(data[1])]\n",
    "brr = [0 for i in range(data[1])]\n",
    "for r in res:\n",
    "    for k,v in r.items():\n",
    "        arr[k]+=v\n",
    "        brr[k]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "OutputPairToFile(T.pairs,'./pairs0.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(x=[i for i in range(data[1])],y=sorted(arr),s=3,c=arr/np.max(arr),cmap='summer')\n",
    "plt.ylim([np.min(arr),np.max(arr)])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.distplot(brr,rug=True,color='g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kernel = stats.gaussian_kde(brr)\n",
    "density = kernel([i for i in range(100)])\n",
    "sum_density = np.cumsum(density)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar([i for i in range(len(sum_density))],sum_density)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mi = mutual_info_classif(arr,data[0][1],copy=False,n_neighbors=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter([i for i in range(len(mi))],mi,s=3,c=mi,cmap='summer')\n",
    "plt.show()\n",
    "plt.scatter([i for i in range(len(brr))],brr,s=3,c=brr,cmap='summer')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "select = mi>1e-3\n",
    "select = np.hstack([select,[False]])\n",
    "X = DictionaryToMatrix(data[0][0],data[0][1],feature_num=data[1]+1,select=select)\n",
    "Y = np.array(data[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.distplot(mi,rug=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "import numpy as np\n",
    "import time\n",
    "from sklearn.decomposition import TruncatedSVD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 下面开始数据转换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy as sp\n",
    "from joblib import dump, load\n",
    "from sklearn.decomposition import *\n",
    "from sklearn.ensemble import *\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RF降维"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfclf = RandomForestClassifier(n_estimators=200,n_jobs=2)\n",
    "rfclf.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "rfclf.score(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_select = list(zip([i for i in range(len(rfclf.feature_importances_))],rfclf.feature_importances_))\n",
    "rf_select = sorted(rf_select,key=lambda x:x[1],reverse=True)\n",
    "rf_select = list(map(lambda x:[x[0],x[1]],rf_select))\n",
    "rf_select = np.array(rf_select)\n",
    "topK = 4000\n",
    "print('select top K features',sum(rf_select[:topK,1]))\n",
    "rf_select = rf_select[:topK,0]\n",
    "rf_select = rf_select.astype('int32')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train[:,rf_select]\n",
    "X_test = X_test[:,rf_select]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=500,random_state=seed)\n",
    "pca.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('explained ratio sum',sum(pca.explained_variance_ratio_[:500]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pca.transform(X_train)\n",
    "X_test = pca.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### IPCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def partialMatrix(dct,start,num):\n",
    "    Len = len(dct[0][0])\n",
    "    xdice = dct[0][0][start:min(Len,start+num)]\n",
    "    Dim = dct[1]\n",
    "    X = []\n",
    "    for row in xdice:\n",
    "        x=[0 for i in range(Dim)]\n",
    "        for (k,v) in row.items():\n",
    "            x[k] = v\n",
    "        X.append(x)\n",
    "    X = np.array(X)\n",
    "    return X\n",
    "\n",
    "### fitting ###\n",
    "num = 1000\n",
    "start = 0\n",
    "pcaDim = 1000\n",
    "ipca = IncrementalPCA(n_components=pcaDim,batch_size=200)\n",
    "while start <= len(data[0][0]):\n",
    "    X_partial = partialMatrix(data,start,num)\n",
    "    ipca.partial_fit(X_partial)\n",
    "    start += num\n",
    "\n",
    "### transformation ###\n",
    "start = 0\n",
    "X = []\n",
    "while start <= len(data[0][0]):\n",
    "    X_partial = partialMatrix(data,start,num)\n",
    "    X_partial = ipca.transform(X_partial)\n",
    "    X.append(X_partial)\n",
    "    start += num\n",
    "X = np.array(X)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Into Sparse Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = Dict2Sparse(data)\n",
    "\n",
    "arr = csr_matrix( ( np.array(res[2]),\n",
    "             (np.array(res[0]), np.array(res[1])) \n",
    "            ), shape = (res[3],res[4]) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 从这里开始可以再次运行，不干扰数据，但是要考虑内存"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SVD 降维\n",
    "目前效率比较低"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "firstReducedDim = 1500\n",
    "svd = TruncatedSVD(n_components=1500, n_iter=50, random_state=42)\n",
    "svd.fit(arr)\n",
    "\n",
    "sum(svd.explained_variance_ratio_)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(svd.explained_variance_ratio_)\n",
    "\n",
    "tarr = svd.transform(arr)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "tarr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": false
   },
   "source": [
    "#### RF 特征提取\n",
    "目前不用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier as RFC\n",
    "rfc = RFC(class_weight={0:sum(np.array(res[-1])==0),1:sum(np.array(res[-1])==1)},\n",
    "         random_state=42,\n",
    "         n_jobs=3,\n",
    "         max_depth=5,\n",
    "         min_samples_leaf=5,\n",
    "         n_estimators=1000)\n",
    "\n",
    "rfc.fit( np.array(tarr) ,np.array(res[-1]))\n",
    "print(rfc.score(np.array(tarr) ,np.array(res[-1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rftarr.shape\n",
    "print(rfc.score(np.array(tarr) ,np.array(res[-1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imp = list( zip(rfc.feature_importances_,[i for i in range(1500)]) )\n",
    "imp = sorted(imp,key = lambda x:x[0])\n",
    "cum = np.cumsum(np.array(imp)[:,0])\n",
    "th = 0.9\n",
    "idx = np.where(cum>th)[0][0]\n",
    "selected_features = np.array(imp)[0:idx][:,1]\n",
    "rftarr = tarr[:,selected_features.astype(int)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 训练准备\n",
    "DTC 预训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "# Y = np.array(res[-1])\n",
    "# X = tarr\n",
    "# help(train_test_split)\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X,Y,test_size=0.3,random_state = 42)\n",
    "del X,Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.tree as Tr \n",
    "dtc = Tr.DecisionTreeClassifier()\n",
    "dtc.fit(X_train,Y_train)\n",
    "\n",
    "print(dtc.score(X_test,Y_test))\n",
    "\n",
    "print(dtc.score(X_train,Y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 使用 XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost\n",
    "xgbclf = xgboost.XGBClassifier(n_estimators=1000,\n",
    "                               nthread=5,\n",
    "                               scale_pos_weight=sum(Y_train==0)/sum(Y_train==1),\n",
    "                               min_child_weight=3,\n",
    "                               subsample=0.95,\n",
    "                               max_depth=3,\n",
    "                               booster='gbtree',\n",
    "                               learning_rate=0.15,\n",
    "                               colsample_bytree=0.8,\n",
    "#                                subsample=0.9,\n",
    "                               objective='binary:logistic',\n",
    "                               eval_metric='auc',\n",
    "                               seed=42\n",
    "                              )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbclf.fit(X_train,Y_train)\n",
    "xgbclf.score(X_test,Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "parameters = {'booster':['gbtree'\n",
    "#                          ,'dart'\n",
    "                        ],\n",
    "              'max_depth':[5,7,9,11],\n",
    "              'objective':['binary:logistic'],\n",
    "              'eval_metric':['auc'],\n",
    "#               'min_child_weight':[0.7,1,2],\n",
    "              'subsample':[1],\n",
    "              'learning_rate':[5e-2,0.1,0.3],\n",
    "#               'colsample_bytree':[0.8,1]\n",
    "             }\n",
    "gs = GridSearchCV(xgbclf,parameters,cv=3,verbose=4,n_jobs=1)\n",
    "gs.fit(X_train,Y_train)\n",
    "\n",
    "print(gs.best_params_)\n",
    "print(gs.best_score_)\n",
    "print(gs.score(X_test,Y_test))\n",
    "\n",
    "#### save model\n",
    "dump(gs, './xgb'+'066'+'.joblib') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write(content='',path='D:\\\\StatisticsData\\\\BIODATA/bio.txt'):\n",
    "    with open(path,'a+') as f:\n",
    "        f.write(content)\n",
    "    return\n",
    "\n",
    "\n",
    "write(content='xgboost\\n'+str(gs.best_params_)+'\\n'+str(gs.best_score_)+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import lightgbm as lgb\n",
    "lgb_train = lgb.Dataset(X_train, Y_train)\n",
    "lgb_test = lgb.Dataset(X_test, Y_test, reference=lgb_train)\n",
    "\n",
    "params = {    \n",
    "          'boosting_type': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'metric': 'acc',\n",
    "          'nthread':6,\n",
    "          'learning_rate':0.08,\n",
    "          'num_leaves':300, \n",
    "          'max_depth': -1,   \n",
    "          'subsample': 0.9, \n",
    "          'colsample_bytree': 0.9, \n",
    "          'feature_fraction': 1,\n",
    "          'bagging_freq': 8,\n",
    "    'num_iterations':300,\n",
    "#     'max_bin':256,\n",
    "    'min_data_in_leaf':2,\n",
    "    \n",
    "    'bagging_fraction': 0.8\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cv_results = lgb.cv(params, lgb_train, num_boost_round=2000, nfold=5, \n",
    "                    stratified=False, shuffle=True, metrics='auc',verbose_eval=1,\n",
    "#                     early_stopping_rounds=50,\n",
    "                    seed=42)\n",
    "print('best n_estimators:', len(cv_results['auc-mean']))\n",
    "print('best cv score:', pd.Series(cv_results['auc-mean']).max())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbclf = lgb.LGBMClassifier(learning_rate=0.045,\n",
    "                           max_depth=-1,\n",
    "                           objective='binary',\n",
    "                            num_leaves=1000,\n",
    "                            min_child_samples=10,\n",
    "                            n_estimators=1000,\n",
    "                            subsample=0.9,\n",
    "                            random_state=42\n",
    "                           )\n",
    "lgbclf.fit(X_train,Y_train)\n",
    "print('train',lgbclf.score(X_train,Y_train))\n",
    "print('test',lgbclf.score(X_test,Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbclf.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "Y_pred = lgb.predict(X_test, num_iteration=gbm.best_iteration)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(Y_pred)):\n",
    "    if Y_pred[i]>0.5:\n",
    "        Y_pred[i]=1\n",
    "    else:\n",
    "        Y_pred[i]=0\n",
    "sum(Y_pred==Y_test)/len(Y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}