05_02_classify_ML_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- encoding: utf-8 -*-\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 适用多项式朴素贝叶斯分类器\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import metrics\n",
    "# KNN分类器\n",
    "from sklearn.neighbors import KNeighborsClassifier  \n",
    "# SVM支持向量机\n",
    "from sklearn.svm import LinearSVC\n",
    "# kmeans 聚类\n",
    "from sklearn.cluster import KMeans \n",
    "# 随机森林\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "# 随机梯度下降（逻辑斯蒂回归）\n",
    "from sklearn import linear_model\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 可视化\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#模型保存与载入\n",
    "from sklearn.externals import joblib\n",
    "# # 注释语句用来存储你的模型\n",
    "# joblib.dump(km,  'doc_cluster.pkl')\n",
    "# km = joblib.load('doc_cluster.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 划分数据集\n",
    "def splitData(label,text):\n",
    "\n",
    "    line_count = len(label)\n",
    "    rarray=np.random.random(size=line_count)\n",
    "    train_set = []\n",
    "    train_tag = []\n",
    "    test_set = []\n",
    "    test_tag = []\n",
    "    for i in range(line_count):\n",
    "        if rarray[i]<0.8:\n",
    "            train_set.append(text[i,:])\n",
    "            train_tag.append(label[i])\n",
    "        else:\n",
    "            test_set.append(text[i,:])\n",
    "            test_tag.append(label[i])\n",
    "    return (train_set,train_tag,test_set,test_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 打印分类结果信息\n",
    "def calculate_result(actual,pred):  \n",
    "    print '***predict info***'  \n",
    "    print 'precision:{0:.3f}'.format(metrics.precision_score(actual,pred,average='macro'))  \n",
    "    print 'recall:{0:0.3f}'.format(metrics.recall_score(actual,pred,average='macro'))\n",
    "    print 'f1-score:{0:.3f}'.format(metrics.f1_score(actual,pred,average='macro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 分类器\n",
    "def train_clf(clf,clf_name,train_set,train_target,test_set,test_target):\n",
    "    print u\"*** %s 分类器 ***\" % (clf_name)\n",
    "    # 训练模型\n",
    "    clf.fit(train_set,train_target)\n",
    "    # 预测\n",
    "    pred = clf.predict(test_set)\n",
    "    # 评估模型\n",
    "    metrics.f1_score(test_target, pred, average='macro')\n",
    "    calculate_result(test_target,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 监督学习分类\n",
    "def supervisedModel(train_set,train_tag,test_set,test_tag):\n",
    "\n",
    "#     clf_nb = MultinomialNB(alpha=.01)\n",
    "    clf_svm = LinearSVC(random_state=0)\n",
    "    clf_knn = KNeighborsClassifier()\n",
    "#     clf_rf = RandomForestClassifier(n_estimators=100,max_depth=6, random_state=0)\n",
    "    clf_sgd = linear_model.SGDClassifier(loss=\"log\", penalty=\"l2\")\n",
    "    \n",
    "#     train_clf(clf_nb,\"MultinomialNB\",train_set,train_tag,test_set,test_tag)\n",
    "    train_clf(clf_svm,\"LinearSVC\",train_set,train_tag,test_set,test_tag)\n",
    "    train_clf(clf_knn,\"KNeighbors\",train_set,train_tag,test_set,test_tag)\n",
    "    train_clf(clf_sgd,\"SGDClassifier\",train_set,train_tag,test_set,test_tag)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 无监督学习模型分类\n",
    "def unSupervisedModel(train_set,train_tag,test_set,test_tag):\n",
    "    clf = KMeans(n_clusters=10) \n",
    "    clf.fit(train_set)\n",
    "    \n",
    "    pred = clf.predict(test_set)\n",
    "    # 评估模型\n",
    "    metrics.f1_score(test_tag, pred, average='macro')\n",
    "    calculate_result(test_tag,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 使用无监督学习对原始数据重新计算类别\n",
    "def trainSupervisedModel(dataSet,cluster_num):\n",
    "    clf = KMeans(n_clusters=cluster_num) \n",
    "    clf.fit(dataSet)\n",
    "    clusterPlot(dataSet,clf.labels_)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 可视化\n",
    "def clusterPlot(data,tag):\n",
    "    # PCA降维\n",
    "    reduced = TruncatedSVD(n_components=64).fit_transform(data)\n",
    "    # TSNE降维\n",
    "    embedded = TSNE(n_components=2, perplexity=30).fit_transform(reduced)\n",
    "    #TSNE可视化\n",
    "    fig = plt.figure(figsize=(15, 5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('Yahoo news dataset  with TSNE')\n",
    "    plt.scatter(embedded[:, 0], embedded[:, 1],c=tag, marker=\"x\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trainModel(dataSet,target):\n",
    "    # 划分数据\n",
    "    train_set,train_tag,test_set,test_tag = splitData(target,dataSet)\n",
    "    \n",
    "#     print \"train_set lenth = \",len(train_set)\n",
    "#     print \"test_set  lenth = \",len(test_set)\n",
    "    # 监督学习\n",
    "    supervisedModel(train_set,train_tag,test_set,test_tag)\n",
    "    \n",
    "    # 无监督学习预测\n",
    "#     unSupervisedModel(train_set,train_tag,test_set,test_tag)\n",
    "    \n",
    "    # 使用k-means对原数据重新分类\n",
    "#     trainSupervisedModel(test_set,20)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main():\n",
    "    \n",
    "    lsiData = \"./data/textData_LSI.npy\"\n",
    "    ldaData = \"./data/textData_LDA.npy\"\n",
    "    targetData = \"./data/target.npy\"\n",
    "    \n",
    "    # 读入数据 原数据矩阵需要做转置\n",
    " \n",
    "    lsiDataSet = np.load(lsiData).T\n",
    "    \n",
    "    ldaDataSet = np.load(ldaData).T\n",
    "    \n",
    "    # 读入标签\n",
    "    target = np.load(targetData)\n",
    "    \n",
    "    # 训练数据\n",
    "    print \"LSI data training\"\n",
    "    trainModel(lsiDataSet,target)\n",
    "    \n",
    "#     print \"LDA data training\"\n",
    "#     trainModel(ldaDataSet,target)   \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}