Leaf Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Leaf Classification\n",
    "______\n",
    "\n",
    "The dataset we would be considering in this example is the Leaf dataset which consists in a collection of shape and texture features extracted from digital images of leaf specimens originating from a total of 40 different plant species.\n",
    "\n",
    "It can be downloaded from : https://archive.ics.uci.edu/ml/datasets/Leaf\n",
    "\n",
    "Attribute Information:\n",
    "\n",
    "1. Class (Species) \n",
    "2. Specimen Number \n",
    "3. Eccentricity \n",
    "4. Aspect Ratio \n",
    "5. Elongation \n",
    "6. Solidity \n",
    "7. Stochastic Convexity \n",
    "8. Isoperimetric Factor \n",
    "9. Maximal Indentation Depth \n",
    "10. Lobedness \n",
    "11. Average Intensity \n",
    "12. Average Contrast \n",
    "13. Smoothness \n",
    "14. Third moment \n",
    "15. Uniformity \n",
    "16. Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from sklearn.tree import DecisionTreeClassifier, export_graphviz\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn import cross_validation, metrics\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from time import time\n",
    "from sklearn import preprocessing\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import roc_auc_score , classification_report\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import precision_score, recall_score, accuracy_score, classification_report\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# read .csv from provided dataset\n",
    "csv_filename=\"leaf/leaf.csv\"\n",
    "\n",
    "# df=pd.read_csv(csv_filename,index_col=0)\n",
    "df=pd.read_csv(csv_filename,\n",
    "              names=[\"Class\", \"No\" , \"Eccentricity\" , \"Aspect-Ratio\" , \"Elongation\" , \"Solidity\",\n",
    "                     \"Stochastic-Convexity\", \"Isoperimetric-Factor\" , \"Max-Indentation-Depth\" ,\n",
    "                     \"Lobedness\" , \"Avg-Intensity\" , \"Avg-Contrast\" , \"Smoothness\" ,\n",
    "                     \"Third-Moment\" , \"Uniformity\" , \"Entropy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>No</th>\n",
       "      <th>Eccentricity</th>\n",
       "      <th>Aspect-Ratio</th>\n",
       "      <th>Elongation</th>\n",
       "      <th>Solidity</th>\n",
       "      <th>Stochastic-Convexity</th>\n",
       "      <th>Isoperimetric-Factor</th>\n",
       "      <th>Max-Indentation-Depth</th>\n",
       "      <th>Lobedness</th>\n",
       "      <th>Avg-Intensity</th>\n",
       "      <th>Avg-Contrast</th>\n",
       "      <th>Smoothness</th>\n",
       "      <th>Third-Moment</th>\n",
       "      <th>Uniformity</th>\n",
       "      <th>Entropy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.72694</td>\n",
       "      <td>1.4742</td>\n",
       "      <td>0.32396</td>\n",
       "      <td>0.98535</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.83592</td>\n",
       "      <td>0.004657</td>\n",
       "      <td>0.003947</td>\n",
       "      <td>0.047790</td>\n",
       "      <td>0.127950</td>\n",
       "      <td>0.016108</td>\n",
       "      <td>0.005232</td>\n",
       "      <td>0.000275</td>\n",
       "      <td>1.17560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.74173</td>\n",
       "      <td>1.5257</td>\n",
       "      <td>0.36116</td>\n",
       "      <td>0.98152</td>\n",
       "      <td>0.99825</td>\n",
       "      <td>0.79867</td>\n",
       "      <td>0.005242</td>\n",
       "      <td>0.005002</td>\n",
       "      <td>0.024160</td>\n",
       "      <td>0.090476</td>\n",
       "      <td>0.008119</td>\n",
       "      <td>0.002708</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.69659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.76722</td>\n",
       "      <td>1.5725</td>\n",
       "      <td>0.38998</td>\n",
       "      <td>0.97755</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.80812</td>\n",
       "      <td>0.007457</td>\n",
       "      <td>0.010121</td>\n",
       "      <td>0.011897</td>\n",
       "      <td>0.057445</td>\n",
       "      <td>0.003289</td>\n",
       "      <td>0.000921</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.44348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.73797</td>\n",
       "      <td>1.4597</td>\n",
       "      <td>0.35376</td>\n",
       "      <td>0.97566</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.81697</td>\n",
       "      <td>0.006877</td>\n",
       "      <td>0.008607</td>\n",
       "      <td>0.015950</td>\n",
       "      <td>0.065491</td>\n",
       "      <td>0.004271</td>\n",
       "      <td>0.001154</td>\n",
       "      <td>0.000066</td>\n",
       "      <td>0.58785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.82301</td>\n",
       "      <td>1.7707</td>\n",
       "      <td>0.44462</td>\n",
       "      <td>0.97698</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.75493</td>\n",
       "      <td>0.007428</td>\n",
       "      <td>0.010042</td>\n",
       "      <td>0.007938</td>\n",
       "      <td>0.045339</td>\n",
       "      <td>0.002051</td>\n",
       "      <td>0.000560</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.34214</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class  No  Eccentricity  Aspect-Ratio  Elongation  Solidity  \\\n",
       "0      1   1       0.72694        1.4742     0.32396   0.98535   \n",
       "1      1   2       0.74173        1.5257     0.36116   0.98152   \n",
       "2      1   3       0.76722        1.5725     0.38998   0.97755   \n",
       "3      1   4       0.73797        1.4597     0.35376   0.97566   \n",
       "4      1   5       0.82301        1.7707     0.44462   0.97698   \n",
       "\n",
       "   Stochastic-Convexity  Isoperimetric-Factor  Max-Indentation-Depth  \\\n",
       "0               1.00000               0.83592               0.004657   \n",
       "1               0.99825               0.79867               0.005242   \n",
       "2               1.00000               0.80812               0.007457   \n",
       "3               1.00000               0.81697               0.006877   \n",
       "4               1.00000               0.75493               0.007428   \n",
       "\n",
       "   Lobedness  Avg-Intensity  Avg-Contrast  Smoothness  Third-Moment  \\\n",
       "0   0.003947       0.047790      0.127950    0.016108      0.005232   \n",
       "1   0.005002       0.024160      0.090476    0.008119      0.002708   \n",
       "2   0.010121       0.011897      0.057445    0.003289      0.000921   \n",
       "3   0.008607       0.015950      0.065491    0.004271      0.001154   \n",
       "4   0.010042       0.007938      0.045339    0.002051      0.000560   \n",
       "\n",
       "   Uniformity  Entropy  \n",
       "0    0.000275  1.17560  \n",
       "1    0.000075  0.69659  \n",
       "2    0.000038  0.44348  \n",
       "3    0.000066  0.58785  \n",
       "4    0.000024  0.34214  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>No</th>\n",
       "      <th>Eccentricity</th>\n",
       "      <th>Aspect-Ratio</th>\n",
       "      <th>Elongation</th>\n",
       "      <th>Solidity</th>\n",
       "      <th>Stochastic-Convexity</th>\n",
       "      <th>Isoperimetric-Factor</th>\n",
       "      <th>Max-Indentation-Depth</th>\n",
       "      <th>Lobedness</th>\n",
       "      <th>Avg-Intensity</th>\n",
       "      <th>Avg-Contrast</th>\n",
       "      <th>Smoothness</th>\n",
       "      <th>Third-Moment</th>\n",
       "      <th>Uniformity</th>\n",
       "      <th>Entropy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>18.544118</td>\n",
       "      <td>6.282353</td>\n",
       "      <td>0.719854</td>\n",
       "      <td>2.440210</td>\n",
       "      <td>0.513760</td>\n",
       "      <td>0.904158</td>\n",
       "      <td>0.943793</td>\n",
       "      <td>0.531234</td>\n",
       "      <td>0.037345</td>\n",
       "      <td>0.523845</td>\n",
       "      <td>0.051346</td>\n",
       "      <td>0.124535</td>\n",
       "      <td>0.017670</td>\n",
       "      <td>0.005928</td>\n",
       "      <td>0.000387</td>\n",
       "      <td>1.162630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>11.152514</td>\n",
       "      <td>3.462779</td>\n",
       "      <td>0.208311</td>\n",
       "      <td>2.599043</td>\n",
       "      <td>0.195583</td>\n",
       "      <td>0.114639</td>\n",
       "      <td>0.115047</td>\n",
       "      <td>0.217532</td>\n",
       "      <td>0.038575</td>\n",
       "      <td>1.039639</td>\n",
       "      <td>0.035965</td>\n",
       "      <td>0.051860</td>\n",
       "      <td>0.013755</td>\n",
       "      <td>0.005294</td>\n",
       "      <td>0.000431</td>\n",
       "      <td>0.584854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.117080</td>\n",
       "      <td>1.006600</td>\n",
       "      <td>0.107610</td>\n",
       "      <td>0.485490</td>\n",
       "      <td>0.396490</td>\n",
       "      <td>0.078376</td>\n",
       "      <td>0.002837</td>\n",
       "      <td>0.001464</td>\n",
       "      <td>0.005022</td>\n",
       "      <td>0.033415</td>\n",
       "      <td>0.001115</td>\n",
       "      <td>0.000229</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.169400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.550622</td>\n",
       "      <td>1.211300</td>\n",
       "      <td>0.349623</td>\n",
       "      <td>0.890667</td>\n",
       "      <td>0.966230</td>\n",
       "      <td>0.346818</td>\n",
       "      <td>0.009521</td>\n",
       "      <td>0.016500</td>\n",
       "      <td>0.022843</td>\n",
       "      <td>0.083362</td>\n",
       "      <td>0.006901</td>\n",
       "      <td>0.002080</td>\n",
       "      <td>0.000102</td>\n",
       "      <td>0.718900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>15.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.763450</td>\n",
       "      <td>1.570750</td>\n",
       "      <td>0.501855</td>\n",
       "      <td>0.948130</td>\n",
       "      <td>0.992980</td>\n",
       "      <td>0.579160</td>\n",
       "      <td>0.023860</td>\n",
       "      <td>0.103615</td>\n",
       "      <td>0.042087</td>\n",
       "      <td>0.119375</td>\n",
       "      <td>0.014050</td>\n",
       "      <td>0.004447</td>\n",
       "      <td>0.000239</td>\n",
       "      <td>1.077450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>29.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.895097</td>\n",
       "      <td>2.343100</td>\n",
       "      <td>0.633373</td>\n",
       "      <td>0.976897</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700712</td>\n",
       "      <td>0.047834</td>\n",
       "      <td>0.416433</td>\n",
       "      <td>0.073046</td>\n",
       "      <td>0.163795</td>\n",
       "      <td>0.026127</td>\n",
       "      <td>0.008307</td>\n",
       "      <td>0.000516</td>\n",
       "      <td>1.554575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>36.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>0.998710</td>\n",
       "      <td>19.038000</td>\n",
       "      <td>0.948340</td>\n",
       "      <td>0.993880</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.858160</td>\n",
       "      <td>0.198980</td>\n",
       "      <td>7.206200</td>\n",
       "      <td>0.190670</td>\n",
       "      <td>0.280810</td>\n",
       "      <td>0.073089</td>\n",
       "      <td>0.029786</td>\n",
       "      <td>0.002936</td>\n",
       "      <td>2.708500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Class          No  Eccentricity  Aspect-Ratio  Elongation  \\\n",
       "count  340.000000  340.000000    340.000000    340.000000  340.000000   \n",
       "mean    18.544118    6.282353      0.719854      2.440210    0.513760   \n",
       "std     11.152514    3.462779      0.208311      2.599043    0.195583   \n",
       "min      1.000000    1.000000      0.117080      1.006600    0.107610   \n",
       "25%      9.000000    3.000000      0.550622      1.211300    0.349623   \n",
       "50%     15.000000    6.000000      0.763450      1.570750    0.501855   \n",
       "75%     29.000000    9.000000      0.895097      2.343100    0.633373   \n",
       "max     36.000000   16.000000      0.998710     19.038000    0.948340   \n",
       "\n",
       "         Solidity  Stochastic-Convexity  Isoperimetric-Factor  \\\n",
       "count  340.000000            340.000000            340.000000   \n",
       "mean     0.904158              0.943793              0.531234   \n",
       "std      0.114639              0.115047              0.217532   \n",
       "min      0.485490              0.396490              0.078376   \n",
       "25%      0.890667              0.966230              0.346818   \n",
       "50%      0.948130              0.992980              0.579160   \n",
       "75%      0.976897              1.000000              0.700712   \n",
       "max      0.993880              1.000000              0.858160   \n",
       "\n",
       "       Max-Indentation-Depth   Lobedness  Avg-Intensity  Avg-Contrast  \\\n",
       "count             340.000000  340.000000     340.000000    340.000000   \n",
       "mean                0.037345    0.523845       0.051346      0.124535   \n",
       "std                 0.038575    1.039639       0.035965      0.051860   \n",
       "min                 0.002837    0.001464       0.005022      0.033415   \n",
       "25%                 0.009521    0.016500       0.022843      0.083362   \n",
       "50%                 0.023860    0.103615       0.042087      0.119375   \n",
       "75%                 0.047834    0.416433       0.073046      0.163795   \n",
       "max                 0.198980    7.206200       0.190670      0.280810   \n",
       "\n",
       "       Smoothness  Third-Moment  Uniformity     Entropy  \n",
       "count  340.000000    340.000000  340.000000  340.000000  \n",
       "mean     0.017670      0.005928    0.000387    1.162630  \n",
       "std      0.013755      0.005294    0.000431    0.584854  \n",
       "min      0.001115      0.000229    0.000007    0.169400  \n",
       "25%      0.006901      0.002080    0.000102    0.718900  \n",
       "50%      0.014050      0.004447    0.000239    1.077450  \n",
       "75%      0.026127      0.008307    0.000516    1.554575  \n",
       "max      0.073089      0.029786    0.002936    2.708500  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Convert animal labels to numbers\n",
    "le = preprocessing.LabelEncoder()\n",
    "df['Class'] = le.fit_transform(df.Class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 22, 23,\n",
       "       24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36], dtype=int64)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Class'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = list(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "features.remove('Class')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(340, 16)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for f in features:\n",
    "\n",
    "    #Get binarized columns\n",
    "    df[f] = pd.get_dummies(df[f])\n",
    " \n",
    "    # Build new array\n",
    "# train_data = pd.concat([hour, days, district], axis=1)\n",
    "# train_data['crime']=crime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Buying</th>\n",
       "      <th>Maintenance</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Persons</th>\n",
       "      <th>Lug-Boot</th>\n",
       "      <th>Safety</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Buying  Maintenance  Doors  Persons  Lug-Boot  Safety  Class\n",
       "0     0.0          0.0    1.0      1.0       0.0     0.0      2\n",
       "1     0.0          0.0    1.0      1.0       0.0     0.0      2\n",
       "2     0.0          0.0    1.0      1.0       0.0     1.0      2\n",
       "3     0.0          0.0    1.0      1.0       0.0     0.0      2\n",
       "4     0.0          0.0    1.0      1.0       0.0     0.0      2"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = df[features]\n",
    "y = df['Class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No</th>\n",
       "      <th>Eccentricity</th>\n",
       "      <th>Aspect-Ratio</th>\n",
       "      <th>Elongation</th>\n",
       "      <th>Solidity</th>\n",
       "      <th>Stochastic-Convexity</th>\n",
       "      <th>Isoperimetric-Factor</th>\n",
       "      <th>Max-Indentation-Depth</th>\n",
       "      <th>Lobedness</th>\n",
       "      <th>Avg-Intensity</th>\n",
       "      <th>Avg-Contrast</th>\n",
       "      <th>Smoothness</th>\n",
       "      <th>Third-Moment</th>\n",
       "      <th>Uniformity</th>\n",
       "      <th>Entropy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "      <td>340.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.282353</td>\n",
       "      <td>0.719854</td>\n",
       "      <td>2.440210</td>\n",
       "      <td>0.513760</td>\n",
       "      <td>0.904158</td>\n",
       "      <td>0.943793</td>\n",
       "      <td>0.531234</td>\n",
       "      <td>0.037345</td>\n",
       "      <td>0.523845</td>\n",
       "      <td>0.051346</td>\n",
       "      <td>0.124535</td>\n",
       "      <td>0.017670</td>\n",
       "      <td>0.005928</td>\n",
       "      <td>0.000387</td>\n",
       "      <td>1.162630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.462779</td>\n",
       "      <td>0.208311</td>\n",
       "      <td>2.599043</td>\n",
       "      <td>0.195583</td>\n",
       "      <td>0.114639</td>\n",
       "      <td>0.115047</td>\n",
       "      <td>0.217532</td>\n",
       "      <td>0.038575</td>\n",
       "      <td>1.039639</td>\n",
       "      <td>0.035965</td>\n",
       "      <td>0.051860</td>\n",
       "      <td>0.013755</td>\n",
       "      <td>0.005294</td>\n",
       "      <td>0.000431</td>\n",
       "      <td>0.584854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.117080</td>\n",
       "      <td>1.006600</td>\n",
       "      <td>0.107610</td>\n",
       "      <td>0.485490</td>\n",
       "      <td>0.396490</td>\n",
       "      <td>0.078376</td>\n",
       "      <td>0.002837</td>\n",
       "      <td>0.001464</td>\n",
       "      <td>0.005022</td>\n",
       "      <td>0.033415</td>\n",
       "      <td>0.001115</td>\n",
       "      <td>0.000229</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.169400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.550622</td>\n",
       "      <td>1.211300</td>\n",
       "      <td>0.349623</td>\n",
       "      <td>0.890667</td>\n",
       "      <td>0.966230</td>\n",
       "      <td>0.346818</td>\n",
       "      <td>0.009521</td>\n",
       "      <td>0.016500</td>\n",
       "      <td>0.022843</td>\n",
       "      <td>0.083362</td>\n",
       "      <td>0.006901</td>\n",
       "      <td>0.002080</td>\n",
       "      <td>0.000102</td>\n",
       "      <td>0.718900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.763450</td>\n",
       "      <td>1.570750</td>\n",
       "      <td>0.501855</td>\n",
       "      <td>0.948130</td>\n",
       "      <td>0.992980</td>\n",
       "      <td>0.579160</td>\n",
       "      <td>0.023860</td>\n",
       "      <td>0.103615</td>\n",
       "      <td>0.042087</td>\n",
       "      <td>0.119375</td>\n",
       "      <td>0.014050</td>\n",
       "      <td>0.004447</td>\n",
       "      <td>0.000239</td>\n",
       "      <td>1.077450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.895097</td>\n",
       "      <td>2.343100</td>\n",
       "      <td>0.633373</td>\n",
       "      <td>0.976897</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700712</td>\n",
       "      <td>0.047834</td>\n",
       "      <td>0.416433</td>\n",
       "      <td>0.073046</td>\n",
       "      <td>0.163795</td>\n",
       "      <td>0.026127</td>\n",
       "      <td>0.008307</td>\n",
       "      <td>0.000516</td>\n",
       "      <td>1.554575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>16.000000</td>\n",
       "      <td>0.998710</td>\n",
       "      <td>19.038000</td>\n",
       "      <td>0.948340</td>\n",
       "      <td>0.993880</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.858160</td>\n",
       "      <td>0.198980</td>\n",
       "      <td>7.206200</td>\n",
       "      <td>0.190670</td>\n",
       "      <td>0.280810</td>\n",
       "      <td>0.073089</td>\n",
       "      <td>0.029786</td>\n",
       "      <td>0.002936</td>\n",
       "      <td>2.708500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               No  Eccentricity  Aspect-Ratio  Elongation    Solidity  \\\n",
       "count  340.000000    340.000000    340.000000  340.000000  340.000000   \n",
       "mean     6.282353      0.719854      2.440210    0.513760    0.904158   \n",
       "std      3.462779      0.208311      2.599043    0.195583    0.114639   \n",
       "min      1.000000      0.117080      1.006600    0.107610    0.485490   \n",
       "25%      3.000000      0.550622      1.211300    0.349623    0.890667   \n",
       "50%      6.000000      0.763450      1.570750    0.501855    0.948130   \n",
       "75%      9.000000      0.895097      2.343100    0.633373    0.976897   \n",
       "max     16.000000      0.998710     19.038000    0.948340    0.993880   \n",
       "\n",
       "       Stochastic-Convexity  Isoperimetric-Factor  Max-Indentation-Depth  \\\n",
       "count            340.000000            340.000000             340.000000   \n",
       "mean               0.943793              0.531234               0.037345   \n",
       "std                0.115047              0.217532               0.038575   \n",
       "min                0.396490              0.078376               0.002837   \n",
       "25%                0.966230              0.346818               0.009521   \n",
       "50%                0.992980              0.579160               0.023860   \n",
       "75%                1.000000              0.700712               0.047834   \n",
       "max                1.000000              0.858160               0.198980   \n",
       "\n",
       "        Lobedness  Avg-Intensity  Avg-Contrast  Smoothness  Third-Moment  \\\n",
       "count  340.000000     340.000000    340.000000  340.000000    340.000000   \n",
       "mean     0.523845       0.051346      0.124535    0.017670      0.005928   \n",
       "std      1.039639       0.035965      0.051860    0.013755      0.005294   \n",
       "min      0.001464       0.005022      0.033415    0.001115      0.000229   \n",
       "25%      0.016500       0.022843      0.083362    0.006901      0.002080   \n",
       "50%      0.103615       0.042087      0.119375    0.014050      0.004447   \n",
       "75%      0.416433       0.073046      0.163795    0.026127      0.008307   \n",
       "max      7.206200       0.190670      0.280810    0.073089      0.029786   \n",
       "\n",
       "       Uniformity     Entropy  \n",
       "count  340.000000  340.000000  \n",
       "mean     0.000387    1.162630  \n",
       "std      0.000431    0.584854  \n",
       "min      0.000007    0.169400  \n",
       "25%      0.000102    0.718900  \n",
       "50%      0.000239    1.077450  \n",
       "75%      0.000516    1.554575  \n",
       "max      0.002936    2.708500  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split dataset to 60% training and 40% testing\n",
    "X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.4, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(204, 15) (204L,)\n"
     ]
    }
   ],
   "source": [
    "print X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature importances with forests of trees\n",
    "\n",
    "This examples shows the use of forests of trees to evaluate the importance of features on an artificial classification task. The red bars are the feature importances of the forest, along with their inter-trees variability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature ranking:\n",
      "1. feature 4 - Solidity (0.101565) \n",
      "2. feature 3 - Elongation (0.091025) \n",
      "3. feature 2 - Aspect-Ratio (0.088494) \n",
      "4. feature 6 - Isoperimetric-Factor (0.088481) \n",
      "5. feature 1 - Eccentricity (0.079418) \n",
      "6. feature 7 - Max-Indentation-Depth (0.069209) \n",
      "7. feature 8 - Lobedness (0.064789) \n",
      "8. feature 14 - Entropy (0.064162) \n",
      "9. feature 5 - Stochastic-Convexity (0.057478) \n",
      "10. feature 10 - Avg-Contrast (0.056595) \n",
      "11. feature 12 - Third-Moment (0.055509) \n",
      "12. feature 13 - Uniformity (0.053986) \n",
      "13. feature 11 - Smoothness (0.053312) \n",
      "14. feature 9 - Avg-Intensity (0.052884) \n",
      "15. feature 0 - No (0.023093) \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzUAAAJZCAYAAACQrRt5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+UlfV9J/D35UeMSlgxJIMMCCogqESIINluUic2McY2\n2CS4YozkGAy1rcdEc6o5292I/WFCbM1idM/SnEaTbgLd020cEwdMwsl1rV3CNkqyITH+quAMgjEi\nP/wRAnP3D+pEZHAGuMOd78zrdc4957nMc5/v53Pn3mHe833u96nUarVaAAAACjWk0QUAAAAcDqEG\nAAAomlADAAAUTagBAACKJtQAAABFE2oAAICiCTUAHBF/+Id/mL/8y79sdBkADEAV16kB6N8mTpyY\nZ555JsOGDUutVkulUskjjzySMWPGHPIx77vvvnz0ox/NU089VcdKy3H55Zdn/Pjx+bM/+7NGlwJA\nHZipAejnKpVK7rnnnmzfvj07duzI9u3bDyvQJOkKR4dqz549hzV+I3V2dja6BADqTKgBKMCBJtXX\nrFmT//Af/kNGjRqVmTNn5r777uv62p133pnTTjstI0eOzKRJk/I3f/M3SZIXX3wxF1xwQTZt2pQ3\nvelNGTlyZDZv3pzLL788n/3sZ7sef99992X8+PFd90866aR84QtfyJlnnpkRI0aks7MzTz/9dObN\nm5e3vvWtOeWUU/KlL33pgD28+vivHPvmm29OU1NTmpub09rampUrV+bUU0/N6NGj87nPfa7rsTfe\neGMuuuiizJ8/PyNHjsysWbPy4x//uOvrDz/8cN797ndn1KhRmT59er71rW/tM+4f/dEf5Xd/93fz\npje9KX/7t3+br3/96/nCF76QkSNH5sILL0ySLFmyJJMmTcrIkSNzxhln5K677uo6xle/+tW8613v\nyp/8yZ/k+OOPzymnnJJVq1Z1fX3r1q35+Mc/nubm5rz5zW/Ohz70oa6vffvb387MmTMzatSovPOd\n78z/+3//r+trS5Ysybhx4zJy5MhMmzYt3//+9w/4/AFwYEINQKE2bdqU3/u938tnP/vZbN26NX/1\nV3+VD3/4w/nlL3+ZJGlqakpbW1u2b9+eO+64I9dcc03WrVuXY445JitXrszYsWN7nPl57WzOihUr\nsnLlyjz//POpVCr5wAc+kJkzZ+bpp5/O6tWrs3Tp0nz3u9/tVf2bN2/Orl27smnTptx44435xCc+\nka9//et56KGH8r//9//On//5n2fDhg1d+9999925+OKLs3Xr1lxyySX5/d///ezZsye7d+/OBz7w\ngZx//vn5xS9+kVtvvTWXXnppHn300a7HLl++PP/lv/yX7NixIwsWLMill16a6667Ltu3b09ra2uS\nZNKkSXnggQeyffv23HDDDfnoRz+aLVu2dB1j7dq1mTZtWn75y1/mT/7kT7Jw4cKur330ox/NSy+9\nlJ/97Gd55plncs011yRJHnrooSxcuDBf/vKX89xzz+UP/uAPMnfu3Pz617/OI488kttvvz0//OEP\ns3379tx7772ZOHFir547APYl1AAU4Pd///dz/PHH5/jjj++aBfgf/+N/5Hd/93fzvve9L0nyO7/z\nO5k1a1ba2tqSJO9///u7fkl+17velfPOOy/333//YdXxyU9+MmPHjs1RRx2V//t//2+effbZ/Omf\n/mmGDh2aiRMn5oorrsiKFSt6daw3vOEN+U//6T9l6NChmT9/fp599tl86lOfyjHHHJPTTjstp512\nWn70ox917X/WWWflgx/8YIYOHZprr702v/rVr7JmzZqsWbMmL7zwQq6//voMGzYs7373u/N7v/d7\nWb58eddjL7zwwrzjHe9Ikhx11FHd1vPhD384TU1NSZKLLrookydPztq1a7u+PmHChHz84x9PpVLJ\nxz72sTz99NN55plnsnnz5tx7771ZtmxZRo4cmaFDh+Zd73pXkuTLX/5yrrzyysyaNSuVSiWXXXZZ\njjrqqKxZsyZDhw7Nrl278pOf/CS7d+/OiSeemJNOOungviEAJBFqAIrQ2tqa5557Ls8991z+8R//\nMUmyYcOG/M//+T+7ws6oUaPywAMP5Omnn06SrFy5Mv/+3//7vPnNb86oUaOycuXKPPvss4dVx7hx\n47q2N2zYkI6Ojn3G/9znPpdnnnmmV8d685vf3DUTdPTRRydJ3vrWt3Z9/eijj87OnTu77r/6VLhK\npZLm5uZs2rQpmzZt2udryd4A0tHR0e1jD+RrX/ta12lio0aNyvr16/d5vl49m/VKvTt37sxTTz2V\n448/PiNHjtzvmBs2bMhf//Vf7/Mctbe3Z9OmTTnllFPyX//rf83ixYvT1NSUj3zkI13fOwAOjlAD\nUIDuPlMzfvz4LFiwoCvsbN26NTt27Mh1112XXbt2Zd68ebnuuuvyi1/8Ilu3bs373//+ruN0t0jA\nsccemxdffLHrfne/YL/6cePHj8/JJ5+8z/jbtm3b5/Ms9fTqldpqtVra29szduzYjB07Nhs3btxn\n340bN6a5ubnburu7v3HjxixatCj/7b/9t2zdujVbt27N6aeffsDPMr3a+PHj89xzz2X79u3dfu1P\n//RP93mOdu7cmYsvvjhJMn/+/Nx///1dp9l95jOf6XE8APYn1AAU6qMf/Wi+9a1v5Tvf+U46Ozvz\n8ssv57777sumTZuya9eu7Nq1K6NHj86QIUOycuXKfOc73+l6bFNTU375y1/u84v4jBkz0tbWlq1b\nt2bz5s1ZunTp645/9tln501velO+8IUv5OWXX86ePXuyfv36/Mu//Euf9PvDH/4wd911V/bs2ZMv\nfvGLeeMb35h3vOMdmTNnTo499th84QtfyO7du1OtVvPtb387l1xyyQGP1dTUlCeeeKLr/gsvvJAh\nQ4Zk9OjR6ezszB133JGf/OQnvaprzJgxef/7358/+qM/yvPPP5/du3d3neb3iU98Iv/9v//3rtPY\nXnjhhbS1teWFF17II488ku9///vZtWtX3vCGN+Too4/OkCH+WwY4FH56AvRzB1p6edy4cWltbc1N\nN92Ut7zlLZkwYUL+6q/+Kp2dnRkxYkRuvfXWXHTRRTn++OOzYsWKrlW+kuTUU0/NJZdckpNPPjnH\nH398Nm/enMsuuyxve9vbMnHixJx//vmZP3/+69YxZMiQfPvb3866dety0kkn5a1vfWs+8YlPdDtj\ncSh9vvb+hRdemL//+7/PqFGj8vWvfz3f/OY3M3To0AwfPjzf+ta30tbWltGjR+eqq67K3/3d32Xy\n5MkHfP4WLlyY9evXd31Gadq0abn22mvzjne8I2PGjMn69evzzne+s9f1/t3f/V2GDRuWqVOnpqmp\nqSsQnnXWWfnyl7+cq666Kscff3ymTJmSr371q0mSX/3qV/nMZz6Tt7zlLRk7dmx+8Ytf7LPiGwC9\n16uLb65atSqf+tSn0tnZmYULF+b666/f5+s///nPc/nll+fBBx/MTTfdlGuvvXafr3d2dmbWrFkZ\nN25c7r777vp2AMCAd+ONN+bxxx/P1772tUaXAkA/NKynHTo7O3PVVVdl9erVGTt2bGbPnp0LL7ww\nU6dO7drnzW9+c770pS/ts6b/qy1dujSnnXbaIf/1DgAA4EB6PP1s7dq1mTx5ciZMmJDhw4dn/vz5\nXWv6v2L06NE566yzMmzY/hmpvb09bW1tueKKK+pXNQAAwL/pcaamo6Njn6Uwx40bt8+6/T255ppr\ncvPNN2fbtm2HViEAg94NN9zQ6BIA6Mf6dKGAe+65J01NTZkxY0ZqtVqvlsYEAAA4GD3O1DQ3N++z\n/n97e/s+a/+/ngceeCB333132tra8tJLL2XHjh1ZsGBBtx/0PNDqPgAAAK/obqKkx5ma2bNn57HH\nHsuGDRuya9eurFixInPnzu3VIDfddFM2btyYJ554IitWrMi55577uivXvDKbU+LthhtuaHgNetBD\nf7npofG30uvXQ/+56aHxt9Lr10P/uQ2EHg6kx5maoUOH5rbbbst5553XtaTztGnTsmzZslQqlSxa\ntChbtmzJrFmzsmPHjgwZMiRLly7NT3/604wYMaKnwwMAAByWHkNNkpx//vn5+c9/vs+//cEf/EHX\ndlNTU5566qnXPcY555yTc8455xBKBAAAOLA+XShgMGlpaWl0CYdND/2DHvqH0nsovf5ED/2FHhqv\n9PoTPfQXA6GHA6nUXu/ktCOoUqm87nlyAADA4HagzGCmBgAAKJpQAwAAFE2oAQAAiibUAAAARRNq\nAACAogk1AABA0YQaAACgaEINAABQNKEGAAAomlADAAAUTagBAACKJtQAAABFE2oAAICiCTUAAEDR\nhBoAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGEGgAAoGhCDQAA\nUDShBgAAKJpQAwAAFE2oAQAAiibUAAAARRNqAACAogk1AABA0YQaAACgaEINAABQNKEGAAAomlAD\nAAAUTagBAACKJtQAAABFE2oAAICiCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom\n1AAAAEUTagAAgKIJNQAAQNGGNbqAklWre2+vbLe07N1uafnNNgAA0LcqtVqt1ugikqRSqaSflHJI\nKpWk4PIBAKDfO1BmcPoZAABQNKEGAAAomlADAAAUTagBAACKJtQAAABFE2oAAICiCTUAAEDRhBoA\nAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGEGgAAoGhCDQAAUDSh\nBgAAKJpQAwAAFE2oAQAAiibUAAAARetVqFm1alWmTp2aKVOmZMmSJft9/ec//3l+67d+K2984xtz\nyy23dP17e3t7zj333Jx++umZPn16br311vpVDgAAkKRSq9Vqr7dDZ2dnpkyZktWrV2fs2LGZPXt2\nVqxYkalTp3bt8+yzz2bDhg256667MmrUqFx77bVJks2bN2fz5s2ZMWNGdu7cmbPOOiutra37PLar\nkEolPZTSr1UqScHlAwBAv3egzNDjTM3atWszefLkTJgwIcOHD8/8+fPT2tq6zz6jR4/OWWedlWHD\nhu3z72PGjMmMGTOSJCNGjMi0adPS0dFxOH0AAADso8dQ09HRkfHjx3fdHzdu3CEFkyeffDLr1q3L\nnDlzDvqxAAAAB3JEFgrYuXNn5s2bl6VLl2bEiBFHYkgAAGCQGNbTDs3Nzdm4cWPX/fb29jQ3N/d6\ngN27d2fevHm57LLLcuGFF77uvosXL+7abmlpSUtLS6/HAQAABpZqtZpqtdrjfj0uFLBnz56ceuqp\nWb16dU444YScffbZWb58eaZNm7bfvjfeeGNGjBiRT3/6013/tmDBgowePXqfVdG6LcRCAQ1Rre69\nvbL9So5safnNNgAA9AcHygw9hppk75LOn/zkJ9PZ2ZmFCxfmM5/5TJYtW5ZKpZJFixZly5YtmTVr\nVnbs2JEhQ4ZkxIgR+elPf5of/ehH+e3f/u1Mnz49lUollUolN910U84///xeF1iKUkPNqw2EHgAA\nGLgOK9QcCUJN4w2EHgAAGLgOeUlnAACA/kyoAQAAiibUAAAARRNqAACAogk1AABA0YQaAACgaEIN\nAABQNKEGAAAomlADAAAUTagBAACKNqzRBcDhqlb33l7ZbmnZu93S8pttAAAGrkqtVqs1uogkqVQq\n6SelHJJKJSm4/CR6AACgfztQZnD6GQAAUDShBgAAKJrP1EA/4HNBAACHzmdq6mQgfJZDD/3DQOgB\nAKAv+EwNAAAwIAk1AABA0YQaAACgaEINAABQNKEGAAAomlADAAAUTagBAACKJtQAAABFE2oAAICi\nCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGEGgAA\noGhCDQAAUDShBgAAKJpQAwAAFE2oAQAAiibUAAAARRNqAACAogk1AABA0YQaAACgaEINAABQNKEG\nAAAomlADAAAUTagBAACKJtQAAABFE2oAAICiCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUAMAABRN\nqAEAAIom1AAAAEUTagAAgKIJNQAAQNGGNboAYGCoVvfeXtluadm73dLym20AgL5QqdVqtUYXkSSV\nSiX9pJRDUqkkBZefRA/9hR4AALp3oMzg9DMAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAA\nAEUTagAAgKIJNQAAQNGEGgAAoGiDOtRMHDMmlUqlLrckdTtWpVLJxDFjGvzsAABAGSq1Wq3W6CKS\nvYHgSJdSqVRSrxErqaWWSp2OllSSBjwfSf94NRw6PfQPA6EHAKD/OVBm6NVMzapVqzJ16tRMmTIl\nS5Ys2e/rP//5z/Nbv/VbeeMb35hbbrnloB4LAABwOHqcqens7MyUKVOyevXqjB07NrNnz86KFSsy\nderUrn2effbZbNiwIXfddVdGjRqVa6+9tteP7SrETM1rjmem5lDooX8YCD0AAP3PIc/UrF27NpMn\nT86ECRMyfPjwzJ8/P62trfvsM3r06Jx11lkZNmzYQT8WAADgcPQYajo6OjJ+/Piu++PGjUtHR0ev\nDn44jwUAAOiNYT3vcuQsXry4a7ulpSUtLS0NqwUAAGisarWaarXa4349hprm5uZs3Lix6357e3ua\nm5t7VcTBPvbVoQYAABjcXjvRceONN3a7X4+nn82ePTuPPfZYNmzYkF27dmXFihWZO3fuAfd/9Qd3\nDvaxAAAAB6vHmZqhQ4fmtttuy3nnnZfOzs4sXLgw06ZNy7Jly1KpVLJo0aJs2bIls2bNyo4dOzJk\nyJAsXbo0P/3pTzNixIhuHwsAAFAvLr5Zr2NZ0rlf0EP/MBB6AAD6n8O6+CYAAEB/JdQAAABFE2oA\nAICiCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGE\nGgAAoGhCDQAAUDShBgAAKJpQAwAAFE2oAQAAiibUAAAARRNqAACAogk1AABA0YQaAACgaEINAABQ\nNKEGAAAomlADAAAUTagBAACKJtQAAABFE2oAAICiCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUFO4\niWPGpFKp1OWWpG7HqlQqmThmTIOfHQAABoNKrVarNbqIZO8v00e6lEqlknqNWEkttVTqdLSkkvTq\n+RgIPdRTpZL0j1f0odND41Sre2+vbLe07N1uafnNNgDQOAfKDEJNvY4l1HRzPKHmUOihfxgIPQDA\nQHOgzOD0MwAAoGhCDQAAUDShBgAAKJpQAwAAFE2oAQAAiibUAAAARRNqAACAogk1NNzEMWNSqVTq\ncktSt2NVKpVMHDPmiNZf7x56Uz8AQOlcfLNex3LxzW6ONzh6qGf9e8esXw+NuABqMjAuXDkQegCA\ngcbFNwEAgAFJqAEAAIo2rNEFAFAf1ere2yvbLS17t1tafrMNAAORz9TU61g+j9LN8QZHDz5T0824\nA+DzKKX3UHr9ANAdn6kBAAAGJKEGAAAomlADAAAUTagBAACKJtQAAABFE2oAAICiCTUAAEDRhBoA\nAKBoQg0AAFA0oQYAACiaUAMAABRtWKMLAIBXVKt7b69st7Ts3W5p+c02ALxWpVar1RpdRJJUKpUc\n6VIqlUrqNWIltdRSqdPRkkrSq+dDD68d88j3UM/6945Zvx56+z2ot0ol6R8/WQ5d6T2UXn8yMHoA\noL4OlBmcfgYAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGEGgAAoGhCDQAAUDShBgAAKJpQAwAA\nFE2oAQAAitarULNq1apMnTo1U6ZMyZIlS7rd5+qrr87kyZMzY8aMrFu3ruvfv/jFL+aMM87I2972\ntlx66aXZtWtXfSoHAABIL0JNZ2dnrrrqqtx7771Zv359li9fnocffniffVauXJnHH388jz76aJYt\nW5Yrr7wySbJp06Z86UtfyoMPPpgf//jH2b17d1asWNE3nQAAAINSj6Fm7dq1mTx5ciZMmJDhw4dn\n/vz5aW1t3Wef1tbWLFiwIEkyZ86cbNu2LVu2bEmS7NmzJy+88EJ2796dF198MWPHju2DNgAAgMGq\nx1DT0dGR8ePHd90fN25cOjo6Xnef5ubmdHR0ZOzYsfn0pz+dE088Mc3NzTnuuOPynve8p47lAwAA\ng12fLhTw/PPPp7W1NRs2bMimTZuyc+fOfOMb3+jLIQEAgEFmWE87NDc3Z+PGjV3329vb09zcvN8+\nTz311H77fO9738vJJ5+c448/PknyoQ99KP/8z/+cj3zkI92OtXjx4q7tlpaWtLS0HEwvAADAAFKt\nVlOtVnvcr8dQM3v27Dz22GPZsGFDTjjhhKxYsSLLly/fZ5+5c+fm9ttvz8UXX5w1a9bkuOOOS1NT\nU0488cSsWbMmL7/8co466qisXr06s2fPPuBYrw41AADA4PbaiY4bb7yx2/16DDVDhw7NbbfdlvPO\nOy+dnZ1ZuHBhpk2blmXLlqVSqWTRokW54IIL0tbWlkmTJuXYY4/NHXfckSQ5++yzM2/evMycOTPD\nhw/PzJkzs2jRovp0CNTNxDFjsuHfFveoj1oqlUpdjjShqSlPbt5cl2MBAANTpVar1RpdRJJUKpUc\n6VIqlUrqNWIltdRSn1/i9h4vvXo+9PDaMY98D/Wsf++Y9euhEd+DveMe+R7qrVJJ+sdPx0NTev3J\nwOgBgPo6UGbo04UCAAAA+ppQAwAAFE2oAQAAitbjQgEAQO9Vq3tvr2y/smhPS8tvtgGoLwsF1OtY\nPmTfzfEGRw8D4UP2A6GHeiv9Q+ql15/oAYD9WSgAAAAYkIQaAACgaEINAABQNKEGAAAomlADAAAU\nTagBAACKZknnw3h8NeekmpZ/225JS6pJkpZU05L7Dq+2DI7lkJPyexgIyyEPhB7qrfSleEuvP9ED\nAPs7UGYQao7oiL03WAJBUn4PAyEQDIQe6q30X0ZLrz/RAwD7O1BmGNaAWgCAfqxa3Xt7ZbulZe92\nS8tvtgH6EzM1R3TE3hsssxxJ+T0MhFmOgdBDvZX+F/bS60/00F8MhB6AgeNAmcFCAQAAQNGEGgAA\noGhCDQAAUDShBgAAKJpQAwAAFE2oAQAAiibUAAAARXPxTQBgwHEBURhcXHzziI7Ye4PlwpVJ+T0M\nhAtXDoQe6q30Cw6WXn+ih/5CD0B/cqDMYKYGAAC6YcavHGZqjuiIvTdYZjmS8nsYCLMcA6GHeiv9\nL7ul15/oob/QA+zlddQ/HCgzWCgAAAAomlADAAAUTagBAACKZqEAAADqzofsOZIsFHBER+y9wfIh\n+6T8HgbCh+wHQg/1VvoHQkuvP9FDf6EH6mEgfA8GQg8DgSWdAQAKYZYDDo6ZmiM6Yu8NllmOpPwe\nBsIsx0Dood5K/4tc6fUneugv9NB4pdef6IH6saQzAAAwIAk1AABA0YQaAACgaEINAABQNKEGAAAo\nmlADAAAUTagBBoSJY8akUqnU7ZakbseaOGZMg58dABjYXKfmiI7Ye4PlGi9J+T0MhGu86KG7ccu+\n1s5AuJ6CHvoHPTRe6fUneqB+XKcGAAAYkIQaAACgaEINAABQNKEGAAAomlADAAAUTagBAACKJtQA\nAABFE2oAAICiCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAAAEUTagAAgKIJ\nNQD9xMQxY1KpVOpyS1K3Y1UqlUwcM6bBzw4AHFilVqvVGl1Esvc/3yNdSqVSSb9ovhuVpFfPRz17\nqKSWWip1Otrg6aHer6N69tCI78HecfWw77EGx3uh3iqVpH/8D3Xo9NA/lN5D6fUneqB+DpQZzNQA\nUDdmmwBohGGNLgCAgWPDli11nG1KfWfftmyp49EA6E/M1AAAAEUTagAAgKIJNQAAQNGEGgAAoGhC\nDQAAUDSrnw1y1ZyTalqSJOekmsW5IUnSkmpacl8DKwMAgN5x8c0jOmLvNeqCg/U0WC446KKP3Y2r\nh32PNTjeC8nA6KGeBsLF+vTQeKXXn+iB+nHxTQAAYEASagAAgKI5/eyIjth7Tj871DHLPP3s1Z9t\nqqYlLakmOfzPNjl163DGLbsHp591dzynnx0KPTRe6fUneqB+DpQZehVqVq1alU996lPp7OzMwoUL\nc/311++3z9VXX52VK1fm2GOPzZ133pkZM2YkSbZt25YrrrgiP/nJTzJkyJB85StfyZw5c3pdYF8S\nCPrWYPklyPegu3HLDgR7xy27B6Gmu+MJNYdCD41Xev2JHqifQ/5MTWdnZ6666qrce++9Wb9+fZYv\nX56HH354n31WrlyZxx9/PI8++miWLVuWK6+8sutrn/zkJ3PBBRfkZz/7WX70ox9l2rRpdWgHAABg\nrx6XdF67dm0mT56cCRMmJEnmz5+f1tbWTJ06tWuf1tbWLFiwIEkyZ86cbNu2LVu2bMnRRx+d+++/\nP3feeefewYYNy8iRI/ugDQYzy1IDAAxuPYaajo6OjB8/vuv+uHHjsnbt2tfdp7m5OR0dHRk6dGhG\njx6dyy+/PD/60Y8ya9asLF26NEcffXQdW2Cwa8l9rwovNza0FgAAjrw+Xf1s9+7defDBB/PHf/zH\nefDBB3PMMcfk85//fF8OCQAADDI9ztQ0Nzdn48aNXffb29vT3Ny83z5PPfVUt/uMHz8+s2bNSpLM\nmzcvS5YsOeBYixcv7tpuaWlJS0tLr5oAAAAGnmq1mmq12uN+PYaa2bNn57HHHsuGDRtywgknZMWK\nFVm+fPk++8ydOze33357Lr744qxZsybHHXdcmpqakuwNNY888kimTJmS1atX57TTTjvgWK8ONQAA\nwOD22omOG2/s/qMGPYaaoUOH5rbbbst5553XtaTztGnTsmzZslQqlSxatCgXXHBB2traMmnSpBx7\n7LG54447uh5/66235tJLL82vf/3rnHzyyft8DQAA4HC5+OYRHbH3XKemf3CdmkMdt+xrvOwdt+we\nXKemu+NS+WXzAAAY/ElEQVS5Ts2h0EPjlV5/ogfq55CvUwMAANCfCTUAAEDRhBoAAKBoQg0AAFA0\noQYAACiaUAMAABRNqAGAfzNxzJhUKpW63ZLU9XgTx4xp8DME0D/1ePFNABgsNmzZUufrHaW+x9uy\npY5HAxg4zNQAAABFE2oAYADpz6fQOX0O6CtOPwOAAaQ/n0Ln9Dmgr5ipAQAAiibUAAAARRNqAACA\nogk1AEC/YrED4GBZKAAA6FcsdgAcLDM1AABA0YQaAACgaE4/A+qimnNSTUuS5JxUszg3JElaUk1L\n7mtgZQDAQFep1Wr1PG31kFUqlRzpUiqVSl3P2a2nStKr50MPfas3PZRef6KH7setpZZKnY515Huo\nZ/17jzc4eujPr6O9xyu7h8H0fq6nSiXpH7+tHTo9UC8HygxOPwMAqLN6ruCW1G/1Niu4MVA5/QwA\noM7quYJbPVdvS6zgxsBkpgYAgP2YbaIkZmoAANiP2SZKYqYGAIAByWzT4GGmBgCAAcls0+BhpgYA\nACiaUAMAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGEGgAAoGhCDQAAUDShBgAAKJpQAwAAFE2o\nAQAAiibUAAAARRNqAACAogk1AABA0YQaAACgaEINAABQNKEGAAAomlADAAAUTagBAACKNqzRBQD0\nF9Wck2pakiTnpJrFuSFJ0pJqWnJfAysDAF5PpVar1RpdRJJUKpUc6VIqlUr6RfPdqCS9ej700Ld6\n00Pp9Sd66GuN6KGSWmqp1Olog6eHer+O9PDaYzXm/Vx6D97P3R1v8PTAvg6UGZx+BgAAFE2oAQAA\niibUAAAARRNqAACAogk1AABA0YQaAACgaEINAABQNKEGAAAomlADAAAUTagBAACKJtQAAABFE2oA\nAICiCTUAAEDRhjW6AAB4RTXnpJqWJMk5qWZxbkiStKSaltzXwMoA6M8qtVqt1ugikqRSqeRIl1Kp\nVNIvmu9GJenV86GHvtWbHkqvP9FDX2tED5XUUkulTkcbPN+HetffiO9Df+6hUa+j0nsYCO9nPXR3\nvN71wL4OlBmcfgYAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGEGgAAoGiuUwMwQLjGCwCDlevU\nHNERe2+wXBMiKb+H0utP9NDX9NA/lH6Nl73HK7sH16k51DFd42X/4w2eHtiX69QAAAADUq9CzapV\nqzJ16tRMmTIlS5Ys6Xafq6++OpMnT86MGTOybt26fb7W2dmZt7/97Zk7d+7hVwwAAPAqPYaazs7O\nXHXVVbn33nuzfv36LF++PA8//PA++6xcuTKPP/54Hn300SxbtixXXnnlPl9funRpTjvttPpWDgAA\nkF6EmrVr12by5MmZMGFChg8fnvnz56e1tXWffVpbW7NgwYIkyZw5c7Jt27Zs2bIlSdLe3p62trZc\nccUVfVA+AAAw2PUYajo6OjJ+/Piu++PGjUtHR8fr7tPc3Ny1zzXXXJObb745lUr9PlgFAADwij5d\n0vmee+5JU1NTZsyYkWq12uMKD4sXL+7abmlpSUtLS1+WBwAA9GPVajXVarXH/XoMNc3Nzdm4cWPX\n/fb29jQ3N++3z1NPPbXfPv/wD/+Qu+++O21tbXnppZeyY8eOLFiwIF/72te6HevVoQYAABjcXjvR\nceONN3a7X4+nn82ePTuPPfZYNmzYkF27dmXFihX7rWI2d+7crqCyZs2aHHfccWlqaspNN92UjRs3\n5oknnsiKFSty7rnnHjDQAAAAHIoeZ2qGDh2a2267Leedd146OzuzcOHCTJs2LcuWLUulUsmiRYty\nwQUXpK2tLZMmTcqxxx6bO+6440jUDgAAkEqtn1zK9EBXB+3zMY/oiL03WK7enZTfQ+n1J3roa3ro\nHxrxfm7EFcj7cw+Neh2V3kOjrmSvh9eO2Zge2NeBMkOvLr4JAADQXwk1AABA0YQaAACgaEINAABQ\nNKEGAAAoWo9LOgMAlKaac1JNS5LknFSzODckSVpSTUvua2BlQF+wpPMRHbH3BsvyqUn5PZRef6KH\nvqaH/qH05ZD3Hu/I9PDqQFBNS1pSTXL4gWCwvI6SgbGUsB5eO6YlnfuDA2UGoeaIjth7fvD3D0JN\n4+mhfxgsPQg1fWuwvI6SgfHLtB5eO6ZQ0x8cKDM4/QwA6shpTwBHnpmaIzpi7/lrVv/gr6KNp4f+\nYbD00J/rT8rvYbC8jpLD76GvTgFMBtcsx0DogX05/exAYx7REXvPD/7+wS8QjaeH/mGw9NCf60/K\n72GwvI4SPew/plCz//GEmkNxoMxgSWcAAKBoQg0AAFA0oQYAACiaUAMAABTNks4AANSd5c05kqx+\ndkRH7D2rq/QPVhpqPD30D4Olh/5cf1J+D4PldZTooa9Z/WzwsvoZAAAwIAk1AABA0YQaAACgaEIN\nAABQNKEGAAAomlADAAAUTagBAACK5uKbAADQDRcQLYeLbx7REXvPhbH6Bxe6azw99A+DpYf+XH9S\nfg+D5XWU6KGvDaYe2JeLbwIAAAOS088AAGCAqlb33l7ZbmnZu93S8pvtgcDpZ0d0xN4bTNOqpfdQ\nev2JHvqaHvqH0t/PSfk9DJbXUaKHvjaYeqjrmJWkf/zmf+icfgYAAAxIQg0AAFA0oQYAACiaUAMA\nABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGEGgAAoGhCDQAAUDShBgAAKJpQAwAAFE2oAQAAiibU\nAAAARRNqAACAogk1AABA0YQaAACgaEINAABQNKEGAAAomlADAAAUTagBAACKJtQAAABFE2oAAICi\nCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNGEGgAA\noGhCDQAAUDShBgAA+qGJY8akUqnU7ZakbseaOGZMg5+dfQ1rdAEAAMD+NmzZklodj1dJ6na8ypYt\ndTpSfZipAQAAiibUAAAARRNqAACAogk1AABA0YQaAACgaL0KNatWrcrUqVMzZcqULFmypNt9rr76\n6kyePDkzZszIunXrkiTt7e0599xzc/rpp2f69Om59dZb61c5AABAehFqOjs7c9VVV+Xee+/N+vXr\ns3z58jz88MP77LNy5co8/vjjefTRR7Ns2bJceeWVSZJhw4bllltuyfr16/N//s//ye23377fYwEA\nAA5Hj6Fm7dq1mTx5ciZMmJDhw4dn/vz5aW1t3Wef1tbWLFiwIEkyZ86cbNu2LVu2bMmYMWMyY8aM\nJMmIESMybdq0dHR09EEbAADAYNVjqOno6Mj48eO77o8bN26/YPLafZqbm/fb58knn8y6desyZ86c\nw60ZAACgyxFZKGDnzp2ZN29eli5dmhEjRhyJIQEAgEFiWE87NDc3Z+PGjV3329vb09zcvN8+Tz31\nVLf77N69O/Pmzctll12WCy+88HXHWrx4cdd2S0tLWlpaetMDAAAwAFWr1VSr1R73q9Rqtdrr7bBn\nz56ceuqpWb16dU444YScffbZWb58eaZNm9a1T1tbW26//fbcc889WbNmTT71qU9lzZo1SZIFCxZk\n9OjRueWWW16/kEolPZRSd5VKJUd2xN6rJL16PvTQt3rTQ+n1J3roa3roH0p/Pyfl9zBYXkeJHvra\nYOmh3vVXUkstlTodq3ffg3o7UGbocaZm6NChue2223Leeeels7MzCxcuzLRp07Js2bJUKpUsWrQo\nF1xwQdra2jJp0qQce+yxufPOO5MkDzzwQL7+9a9n+vTpmTlzZiqVSm666aacf/75dW8QAAAYnHqc\nqTlSzNTsa7D8BSIpv4fS60/00Nf00D+U/n5Oyu9hsLyOEj30tcHSg5mabsY9QGY4IgsFAAAA9BWh\nBgAAKJpQAwAAFE2oAQAAiibUAAAARRNqAACAogk1AABA0YQaAACgaEINAABQNKEGAAAomlADAAAU\nTagBAACKJtQAAABFE2oAAICiCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIo2rNEF\nAAAAfaOac1JNS5LknFSzODckSVpSTUvua2Bl9VWp1Wq1RheRJJVKJUe6lEqlkn7RfDcqSa+eDz30\nrd70UHr9iR76mh76h9Lfz0n5PQyW11Gih742WHoovf4+GfcAmcHpZwAAQNGEGgAAoGhCDQAAUDSh\nBgAAKJpQAwAAFE2oAQAAiibUAAAARRNqAACAogk1AABA0YQaAACgaEINAABQNKEGAAAomlADAAAU\nTagBAACKJtQAAABFE2oAAICiCTUAAEDRhBoAAKBoQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAA\nAEUTagAAgKIJNQAAQNGEGgAAoGhCDQAAUDShBgAAKJpQAwAAFE2oAQAAiibUAAAARRNqAACAogk1\nAABA0YQaAACgaEINAABQNKEGAAAomlADAAAUTagBAACKJtQAAABFE2oAAICiCTUAAEDRhBoAAKBo\nQg0AAFA0oQYAACiaUAMAABRNqAEAAIom1AAAAEUTagAAgKIJNQAAQNF6FWpWrVqVqVOnZsqUKVmy\nZEm3+1x99dWZPHlyZsyYkXXr1h3UYwEAAA5Vj6Gms7MzV111Ve69996sX78+y5cvz8MPP7zPPitX\nrszjjz+eRx99NMuWLcuVV17Z68cOFNVGF1AH1UYXUAfVRhdQB9VGF1AH1UYXUAfVRhdwmKqNLqAO\nqo0uoA6qjS6gDqqNLqAOqo0u4DBVG11AHVQbXUAdVBtdQB1UG11AH+ox1KxduzaTJ0/OhAkTMnz4\n8MyfPz+tra377NPa2poFCxYkSebMmZNt27Zly5YtvXrsQFFtdAF1UG10AXVQbXQBdVBtdAF1UG10\nAXVQbXQBh6na6ALqoNroAuqg2ugC6qDa6ALqoNroAg5TtdEF1EG10QXUQbXRBdRBtdEF9KEeQ01H\nR0fGjx/fdX/cuHHp6Ojo1T69eSwAAMDh6JOFAmq1Wl8cFgAAYD/Detqhubk5Gzdu7Lrf3t6e5ubm\n/fZ56qmn9ttn165dPT721SqVykEVXw/1HPHGOh4r6f3zoYd9NaKHer9y69lDI74HiR5ea7C8F5Ly\ne+jPr6Ok/B68nw+N9/P+BksPA+G9cCT0GGpmz56dxx57LBs2bMgJJ5yQFStWZPny5fvsM3fu3Nx+\n++25+OKLs2bNmhx33HFpamrK6NGje3zsK8zuAAAAh6LHUDN06NDcdtttOe+889LZ2ZmFCxdm2rRp\nWbZsWSqVShYtWpQLLrggbW1tmTRpUo499tjccccdr/tYAACAeqnUTJEAAAAF65OFAgajzs7OvP3t\nb8/cuXMbXcpB+9WvfpU5c+Zk5syZmT59em68sd5njPa99vb2nHvuuTn99NMzffr03HrrrY0u6aBt\n27YtF110UaZNm5bTTz89P/jBDxpd0kFbuHBhmpqa8ra3va3RpRySRx55JDNnzszb3/72zJw5M//u\n3/27Il9LX/ziF3PGGWfkbW97Wy699NLs2rWr0SX16PVeO3/913+dIUOG5LnnnmtAZYdu4sSJOfPM\nMzNz5sycffbZjS6nV7r7PmzdujXnnXdeTj311Lzvfe/Ltm3bGljh6+uu/uuuuy7Tpk3LjBkz8uEP\nfzjbt29vYIU9666Hz372s12vpfPPPz+bN29uYIU9666Hf/iHf8gZZ5yRoUOH5sEHH2xgdYdm6dKl\nmT59erG/Y6xatSpTp07NlClTsmTJkkaX0zdq1MUtt9xSu/TSS2sf+MAHGl3KIXnhhRdqtVqttnv3\n7tqcOXNqP/jBDxpc0cF5+umnaw899FCtVqvVduzYUZsyZUrtZz/7WYOrOjgf+9jHal/5yldqtVqt\n9utf/7q2bdu2Bld08O6///7aQw89VJs+fXqjSzlse/bsqZ1wwgm1jRs3NrqUg9LR0VE76aSTar/6\n1a9qtVqt9h//43+sffWrX21wVT070Gvnqaeeqr3vfe+rTZw4sfbLX/6yQdUdmpNOOqn23HPPNbqM\ng9Ld9+G6666rLVmypFar1Wqf//zna9dff32jyutRd/V/97vfre3Zs6dWq9Vq119/fe0zn/lMo8rr\nle562LFjR9f2rbfeWrvyyisbUVqvddfDww8/XHvkkUdq7373u2s//OEPG1jdwfvJT35Smz59eu3l\nl1+u7d69u/be97639vjjjze6rF7bs2dP7ZRTTqk9+eSTtV27dtXOPPPM4n5H6g0zNXXQ3t6etra2\nXHHFFY0u5ZAdc8wxSfbO2uzevbtfrWbRG2PGjMmMGTOSJCNGjMi0adOKuibS9u3bc//99+fyyy9P\nkgwbNiwjR45scFUH753vfGdGjRrV6DLq4nvf+15OOeWUfa61VYo9e/bkhRdeyO7du/Piiy9m7Nix\njS6pRwd67VxzzTW5+eabG1DR4avVauns7Gx0GQelu+9Da2trPvaxjyVJPvaxj+Wuu+5qRGm90l39\n73nPezJkyN5fd97xjnekvb29EaX1Wnc9jBgxomv7hRde6Oqnv+quh1NPPTWTJ08ucmGon/3sZ5kz\nZ06OOuqoDB06NL/927+df/zHf2x0Wb22du3aTJ48ORMmTMjw4cMzf/78tLa2Nrqsuuvf74pCvPKf\nbmlB4NU6Ozszc+bMjBkzJu9973sze/bsRpd0yJ588smsW7cuc+bMaXQpvfav//qvGT16dC6//PK8\n/e1vz6JFi/LSSy81uqxB7e///u9zySWXNLqMgzZ27Nh8+tOfzoknnpjm5uYcd9xxec973tPosg7J\n3XffnfHjx2f69OmNLuWQVCqVrp+nX/7ylxtdziF75pln0tTUlGTvH5CeeeaZBld06L7yla/k/e9/\nf6PLOCT/+T//55x44on5xje+kT/7sz9rdDmDyhlnnJH7778/W7duzYsvvpi2trZ9LmXS33V0dOzz\nB7px48YV9Yff3hJqDtM999yTpqamzJgxI7Varci/QCTJkCFD8tBDD6W9vT0/+MEP8tOf/rTRJR2S\nnTt3Zt68eVm6dOk+f9nq73bv3p0HH3wwf/zHf5wHH3wwxxxzTD7/+c83uqxB69e//nXuvvvuXHTR\nRY0u5aA9//zzaW1tzYYNG7Jp06bs3Lkz3/jGNxpd1kF76aWXctNNN+3zGb/Sfr4+8MADefDBB9PW\n1pbbb789//RP/9Tokuqi1D/g/eVf/mWGDx+ej3zkI40u5ZD8xV/8RTZu3JhLL700X/rSlxpdzqAy\nderUXH/99Xnve9+bCy64IDNnzszQoUMbXRavIdQcpgceeCB33313Tj755FxyySX5/ve/nwULFjS6\nrEM2cuTIvPvd786qVasaXcpB2717d+bNm5fLLrssF154YaPLOSjjxo3L+PHjM2vWrCTJvHnzivwg\n5UCxcuXKnHXWWXnLW97S6FIO2ve+972cfPLJOf744zN06NB86EMfyj//8z83uqyD9vjjj+fJJ5/M\nmWeemZNOOint7e0566yzipolOOGEE5Ikb3nLW/LBD34wa9eubXBFh6apqSlbtmxJkmzevDlvfetb\nG1zRwbvzzjvT1tZWZMB/rY985CP5X//rfzW6jEHn8ssvz7/8y7+kWq3muOOOy5QpUxpdUq81Nzdn\n48aNXffb29vT3NzcwIr6hlBzmG666aZs3LgxTzzxRFasWJFzzz03X/va1xpd1kF59tlnu1azeeml\nl/Ld7343U6dObXBVB+/jH/94TjvttHzyk59sdCkHrampKePHj88jjzySJFm9enVOO+20Bld1aEqe\nsXzF8uXLizz1LElOPPHErFmzJi+//HJqtVpWr15dzPXBXv3aOeOMM7J58+Y88cQT+dd//deMGzcu\nDz30UDG/UL/44ovZuXNnkr2fgfjOd76TM844o8FV9c5r38Nz587NnXfemST56le/2u//aPTa+let\nWpWbb745d999d4466qgGVtZ7r+3hscce69q+6667inhPv97/BSX+H/GLX/wiSbJx48Z885vfLGrG\nb/bs2XnssceyYcOG7Nq1KytWrChytd4eHeGFCQa0arVa5OpnP/7xj2szZ86snXnmmbXp06fX/uIv\n/qLRJR20f/qnf6oNGTKkduaZZ9ZmzJhRmzlzZm3lypWNLuugrFu3rjZr1qzamWeeWfvgBz9Ye/75\n5xtd0kG75JJLaieccELtDW94Q238+PFdq7mV5IUXXqiNHj26tn379kaXcsgWL15cmzp1am369Om1\nBQsW1Hbt2tXoknrU02vnpJNOKmr1syeeeKLr59EZZ5xR+9znPtfoknqlu+/Dc889V/ud3/md2pQp\nU2rvfe97a1u3bm10mQfUXf2TJk2qnXjiibWZM2fWZs6cWfvDP/zDRpf5urrr4cMf/nDtjDPOqJ15\n5pm1uXPn1jZt2tToMl9Xdz1885vfrI0bN672xje+sTZmzJja+eef3+gyD8q73vWu2umnn16bMWNG\n7fvf/36jyzloK1eurE2ZMqU2adKkYn4eHSwX3wQAAIrm9DMAAKBoQg0AAFA0oQYAACiaUAMAABRN\nqAEAAIom1AAAAEUTagAAgKIJNQAAQNH+P3HFLaD0KkaJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x9c6bf98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "# Build a classification task using 3 informative features\n",
    "\n",
    "# Build a forest and compute the feature importances\n",
    "forest = ExtraTreesClassifier(n_estimators=250,\n",
    "                              random_state=0)\n",
    "\n",
    "forest.fit(X, y)\n",
    "importances = forest.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in forest.estimators_],\n",
    "             axis=0)\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X.shape[1]):\n",
    "    print(\"%d. feature %d - %s (%f) \" % (f + 1, indices[f], features[indices[f]], importances[indices[f]]))\n",
    "\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure(num=None, figsize=(14, 10), dpi=80, facecolor='w', edgecolor='k')\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X.shape[1]), importances[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.xticks(range(X.shape[1]), indices)\n",
    "plt.xlim([-1, X.shape[1]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.10156511,  0.09102479,  0.08849443,  0.08848106,  0.07941827])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "importances[indices[:5]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. feature 4 - Solidity (0.101565)\n",
      "2. feature 3 - Elongation (0.091025)\n",
      "3. feature 2 - Aspect-Ratio (0.088494)\n",
      "4. feature 6 - Isoperimetric-Factor (0.088481)\n",
      "5. feature 1 - Eccentricity (0.079418)\n"
     ]
    }
   ],
   "source": [
    "for f in range(5):\n",
    "    print(\"%d. feature %d - %s (%f)\" % (f + 1, indices[f], features[indices[f]] ,importances[indices[f]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "best_features = []\n",
    "for i in indices[:5]:\n",
    "    best_features.append(features[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAF6CAYAAAAj9ZDJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8FPW9//H3JCBiIIWYEsxFbglJkJDNjSAKLFipUoGq\nUGPVCIZS64+iYiu2tS30tFS0UlGsTVsFtDTReixBCbTH6CKHkuIRsQWUqwQSbnIxBBBCst/fH5Qp\ngVw2XJJvwuv5eOTx2Ml+Z+Y7n9nNe7+zMxPHGGMEAACsEdTcHQAAADURzgAAWIZwBgDAMoQzAACW\nIZwBALAM4QwAgGUIZ6CV+c53vqNf/OIXzd0NAOfB4Tpn4KTu3btr7969atOmjYwxchxHGzduVNeu\nXc95mcuWLdPdd9+tHTt2XMCethzjx49XTEyMfvaznzV3V4AWhZEz8G+O42jx4sU6dOiQKioqdOjQ\nofMKZkluyJ+r6urq81p/c/L7/c3dBaDFIpyB09R1IKm4uFjXXXedOnfurJSUFC1btsx9bt68eerT\np49CQ0MVGxur3/3ud5Kko0ePasSIEdq5c6c6duyo0NBQ7d69W+PHj9dPfvITd/5ly5YpJibGne7R\no4eefPJJJScnq0OHDvL7/dq1a5fGjBmjLl26qFevXnruuefq3IbTl39q2U899ZQiIiIUFRWlgoIC\nLVmyRPHx8QoPD9cvf/lLd97p06dr7NixysrKUmhoqNLT0/XPf/7Tff6TTz7R0KFD1blzZyUlJenN\nN9+ssd4HHnhAX/va19SxY0e9+OKLWrBggZ588kmFhoZq9OjRkqSZM2cqNjZWoaGh6tu3rxYuXOgu\nY/78+Ro0aJC+//3vKywsTL169dLSpUvd5w8ePKj77rtPUVFRuvLKK3Xbbbe5z7311ltKSUlR586d\ndf311+tf//qX+9zMmTMVHR2t0NBQJSYm6t13362zfoAVDABjjDHdu3c3RUVFZ/2+rKzMXHnllWbp\n0qXGGGPefvttc+WVV5p9+/YZY4wpLCw0n376qTHGmPfee89cccUV5sMPPzTGGOPz+UxMTEyN5Y0b\nN878+Mc/dqfPbNO9e3eTkpJiysrKzLFjx4zf7zdpaWnm5z//uamqqjKffvqp6dWrl/nb3/5W63ac\nvnyfz2fatGnjzvv73//efPnLXzZ33XWXOXLkiFm3bp1p37692bZtmzHGmGnTppnLLrvMvPHGG6aq\nqsr86le/Mj169DBVVVXmxIkTJjY21jzxxBPmxIkT5p133jEdO3Y0GzdudNfbqVMns3LlSmOMMceO\nHTtrW40x5vXXXze7d+82xhjz2muvmZCQEHd63rx55rLLLjMvvvii8fv95oUXXjCRkZHuvCNGjDBZ\nWVmmvLzcVFVVmffee88YY8zq1atNly5dzPvvv2/8fr95+eWXTffu3U1lZaXZsGGDiYmJcddRUlJi\ntm7dWmvtAFswcgZO8/Wvf11hYWEKCwtzR2V//OMf9bWvfU1f/epXJUk33HCD0tPTVVhYKEm6+eab\n1b17d0nSoEGDNHz4cC1fvvy8+vHggw8qMjJS7dq10/vvv699+/bpRz/6kYKDg9W9e3dNmDBB+fn5\nAS3rsssu0w9/+EMFBwcrKytL+/bt00MPPaQrrrhCffr0UZ8+ffTRRx+57dPS0nTrrbcqODhYU6ZM\n0fHjx1VcXKzi4mIdOXJEU6dOVZs2bTR06FDdcsstysvLc+cdPXq0BgwYIElq165drf25/fbbFRER\nIUkaO3as4uLitGrVKvf5bt266b777pPjOLr33nu1a9cu7d27V7t379Zf//pX5ebmKjQ0VMHBwRo0\naJAk6fe//73uv/9+paeny3Ec3XPPPWrXrp2Ki4sVHBysyspKrV27VlVVVbr66qvVo0ePxu0QoIkR\nzsBpCgoKdODAAR04cEBvvPGGJKmkpESvvfaaG9qdO3fWihUrtGvXLknSkiVLdO211+rKK69U586d\ntWTJEu3bt++8+hEdHe0+LikpUVlZWY31//KXv9TevXsDWtaVV17pfu/dvn17SVKXLl3c59u3b6/D\nhw+706cfYnccR1FRUdq5c6d27txZ4znpZJCWlZXVOm9dXn75Zffwc+fOnbVu3boa9Tr9e/5T/T18\n+LB27NihsLAwhYaGnrXMkpISPf300zVqVFpaqp07d6pXr1565plnNG3aNEVEROib3/ymu+8AWxHO\nwGlMLd85x8TEKDs72w3tgwcPqqKiQo8++qgqKys1ZswYPfroo/rss8908OBB3Xzzze5yajsZLCQk\nREePHnWnawuK0+eLiYlRz549a6y/vLy8xve9F9LpZ5YbY1RaWqrIyEhFRkZq+/btNdpu375dUVFR\ntfa7tunt27dr4sSJ+s1vfqODBw/q4MGDuuaaa+r8rv90MTExOnDggA4dOlTrcz/60Y9q1Ojw4cO6\n4447JElZWVlavny5SkpKJEmPPfZYg+sDmhPhDDTg7rvv1ptvvqm//e1v8vv9OnbsmJYtW6adO3eq\nsrJSlZWVCg8PV1BQkJYsWaK//e1v7rwRERHav39/jUDxeDwqLCzUwYMHtXv3bs2ePbve9ffv318d\nO3bUk08+qWPHjqm6ulrr1q3T//3f/12U7f3ggw+0cOFCVVdX69e//rUuv/xyDRgwQJmZmQoJCdGT\nTz6pqqoq+Xw+vfXWW7rzzjvrXFZERIS2bt3qTh85ckRBQUEKDw+X3+/X3LlztXbt2oD61bVrV918\n88164IEH9Pnnn6uqqsr9+uBb3/qWfvvb37qHx48cOaLCwkIdOXJEGzdu1LvvvqvKykpddtllat++\nvYKC+NMHu/EKBf6trkueoqOjVVBQoBkzZujLX/6yunXrpl/96lfy+/3q0KGDnn32WY0dO1ZhYWHK\nz893z0qWpPj4eN15553q2bOnwsLCtHv3bt1zzz3q16+funfvrptuuklZWVn19iMoKEhvvfWW1qxZ\nox49eqhLly761re+VesI8ly288zp0aNH69VXX1Xnzp21YMEC/eUvf1FwcLDatm2rN998U4WFhQoP\nD9ekSZP0yiuvKC4urs765eTkaN26de53+ImJiZoyZYoGDBigrl27at26dbr++usD7u8rr7yiNm3a\nKCEhQREREe4Hm7S0NP3+97/XpEmTFBYWpt69e2v+/PmSpOPHj+uxxx7Tl7/8ZUVGRuqzzz6rcYY6\nYKOAbkKydOlSPfTQQ/L7/crJydHUqVNrPL9hwwaNHz9eq1ev1owZMzRlypQaz/v9fqWnpys6OlqL\nFi26sFsA4IKZPn26tmzZopdffrm5uwJc0to01MDv92vSpEkqKipSZGSkMjIyNHr0aCUkJLhtrrzy\nSj333HM1rlc83ezZs9WnT59z/qQPAMClpMHD2qtWrVJcXJy6deumtm3bKisrSwUFBTXahIeHKy0t\nTW3anJ31paWlKiws1IQJEy5crwEAaMUaHDmXlZXVuDwiOjq6xjWJDXn44Yf11FNPqby8/Nx6CKDJ\n/PSnP23uLgDQRT4hbPHixYqIiJDH45ExJqDLJQAAuNQ1OHKOioqqcW1jaWlpjesa67NixQotWrRI\nhYWF+uKLL1RRUaHs7OxaTzY5n38OAABAS1TXoLXBkXNGRoY2b96skpISVVZWKj8/X6NGjQpoRTNm\nzND27du1detW5efna9iwYfWeBXpqdM1P3T8//elPm70PLeGHOlEr6kStbP+pT4Mj5+DgYM2ZM0fD\nhw93L6VKTExUbm6uHMfRxIkTtWfPHqWnp6uiokJBQUGaPXu21q9frw4dOjS0eAAAcIYGw1mSbrrp\nJm3YsKHG77797W+7jyMiIhr8Z/JDhgzRkCFDzqGLAABcWrhDWAvj9XqbuwstAnUKHLUKDHUKHLU6\nfwHdIawpOI7T4DF4AABai/pyj5EzAACWIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLEM4AAFiG\ncAYAwDKEMwAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAyxDOAABYhnAGAMAyhDMAAJYhnAEA\nsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWadPcHUDDfL6TP6cee70nH3u9\n/3kMAGg9HGOMae5OSJLjOLKkK1ZzHIkyAUDLV1/ucVgbAADLEM4AAFiGcAYAwDKEMwAAliGcAQCw\nDOEMAIBlCGcAACxDOAMAYBnCGQAAyxDOAABYJqBwXrp0qRISEtS7d2/NnDnzrOc3bNiggQMH6vLL\nL9esWbPc35eWlmrYsGG65pprlJSUpGefffbC9RwAgFaqwXtr+/1+9e7dW0VFRYqMjFRGRoby8/OV\nkJDgttm3b59KSkq0cOFCde7cWVOmTJEk7d69W7t375bH49Hhw4eVlpamgoKCGvO6HeHe2gHh3toA\n0Dqc1721V61apbi4OHXr1k1t27ZVVlaWCgoKarQJDw9XWlqa2rSp+U+uunbtKo/HI0nq0KGDEhMT\nVVZWdq7bAQDAJaHBcC4rK1NMTIw7HR0dfU4Bu23bNq1Zs0aZmZmNnhcAgEtJk5wQdvjwYY0ZM0az\nZ89Whw4dmmKVAAC0WG0aahAVFaXt27e706WlpYqKigp4BVVVVRozZozuuecejR49ut6206ZNcx97\nvV55vd6A1wMAgM18Pp98Pl9AbRs8Iay6ulrx8fEqKirSVVddpf79+ysvL0+JiYlntZ0+fbo6dOig\nRx55xP1ddna2wsPDa5zFXWtHOCEsIJwQVjef7+TPqcenPtt5vf95DAC2qC/3Ggxn6eSlVA8++KD8\nfr9ycnL02GOPKTc3V47jaOLEidqzZ4/S09NVUVGhoKAgdejQQevXr9dHH32kwYMHKykpSY7jyHEc\nzZgxQzfddFOjOon/IJwDQ50A2O68w7kpEM6BIXQCQ50A2O68LqUCAABNi3AGAMAyhDMAAJYhnAEA\nsAzhDACAZRq8CQmA1odrwgG7cSlVC8MlQoGhToGjVkDzqC/3GDkDQB04woDmwsi5hWGUExjqFDhq\nFRjqhAuNm5AAANCCEM4AAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAyxDO\nAABYhnAGAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACW\nIZwBALAM4QwAgGUIZwAALEM4AwBgmTbN3QEAQMvn8538OfXY6z352Ov9z2MEzjHGmObuhCQ5jiNL\numI1x5EoU8OoU+CoVWCoU+CoVWDqyz0OawMAYBnCGQAAyxDOAABYhnAGAMAyAYXz0qVLlZCQoN69\ne2vmzJlnPb9hwwYNHDhQl19+uWbNmtWoeQEAQE0Nnq3t9/vVu3dvFRUVKTIyUhkZGcrPz1dCQoLb\nZt++fSopKdHChQvVuXNnTZkyJeB53Y60sLO1u3ftqpI9e5phzUaS0+Rr7RYRoW27dzf5es8VZ4sG\njloFhjoFjloF5rzO1l61apXi4uLUrVs3tW3bVllZWSooKKjRJjw8XGlpaWrTpk2j522pSvbskZGa\n/EfNsE7z7+0FADSNBsO5rKxMMTEx7nR0dLTKysoCWvj5zAsAwKWKE8IAALBMg7fvjIqK0vbt293p\n0tJSRUVFBbTwxs47bdo097HX65WXe74BAFoJn88n36l7nDagwRPCqqurFR8fr6KiIl111VXq37+/\n8vLylJiYeFbb6dOnq0OHDnrkkUcaPW9LOyHMcRw1R28dGZlmOCHMkVrY/uGElEBRq8BQp8BRq8DU\nl3sNjpyDg4M1Z84cDR8+XH6/Xzk5OUpMTFRubq4cx9HEiRO1Z88epaenq6KiQkFBQZo9e7bWr1+v\nDh061DovAACoG//44hwxcrYbn9wDR60CQ50CR60Cwz++AACgBSGcAQCwDOEMAIBlCGcAACxDOAMA\nYBnCGQAAyxDOAABYhnAGAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZw\nBgDAMoQzAACWadPcHUDr171rV5Xs2dPEazVyHKeJ1yl1i4jQtt27m3y9AFoXxxhjmrsTkuQ4jizp\nSkAcx1Fz9NaRkVHTh44jnfP+aY5atcQ6NRfHkVpYl5sFdQoctQpMfbnHYW0AACxDOAMAYBnCGQAA\nyxDOAABYhrO1AUs0z1ntUks7s5064VLA2drniLO1GzEvZ2sHNi+vqcDmo07W42ztwHC2NgAALQjh\nDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGUIZwAALEM4AwBg\nGcIZAADLEM4AAFgmoHBeunSpEhIS1Lt3b82cObPWNpMnT1ZcXJw8Ho/WrFnj/v7Xv/61+vbtq379\n+umuu+5SZWXlhek5AACtVIPh7Pf7NWnSJP31r3/VunXrlJeXp08++aRGmyVLlmjLli3atGmTcnNz\ndf/990uSdu7cqeeee06rV6/WP//5T1VVVSk/P//ibAkAAK1Eg+G8atUqxcXFqVu3bmrbtq2ysrJU\nUFBQo01BQYGys7MlSZmZmSovL9eePXskSdXV1Tpy5Iiqqqp09OhRRUZGXoTNAACg9WgwnMvKyhQT\nE+NOR0dHq6ysrN42UVFRKisrU2RkpB555BFdffXVioqKUqdOnfSVr3zlAnYfAIDWp83FXPjnn3+u\ngoIClZSU6Etf+pLGjBmjP/3pT/rmN79Za/tp06a5j71er7xe78XsHgAATcbn88nn8wXUtsFwjoqK\n0vbt293p0tJSRUVFndVmx44dZ7V5++231bNnT4WFhUmSbrvtNv39738PKJwBAGhNzhx0Tp8+vc62\nDR7WzsjI0ObNm1VSUqLKykrl5+dr1KhRNdqMGjVKL7/8siSpuLhYnTp1UkREhK6++moVFxfr2LFj\nMsaoqKhIiYmJ57hZAIDG6N61qxzHafIfSU2+zu5duzZztS+sBkfOwcHBmjNnjoYPHy6/36+cnBwl\nJiYqNzdXjuNo4sSJGjFihAoLCxUbG6uQkBDNnTtXktS/f3+NGTNGKSkpatu2rVJSUjRx4sSLvlEA\nAKlkzx6ZZlivIzX5ep1/n4TcWjjGmObYd2dxHEeWdCUgjuM004veyMhphvXqnPdPc9SKOjVivS2s\nVtSpEfNeQrU6nzo1l/pyjzuEAQBgGcIZAADLEM4AAFiGcAYAwDKcEHaOmvJEC5+GyCfvvx975ZVP\nkuSVT14ta5I+tLSTUjh5pxHrbWG1ok6NmPcSqlVrOyGMcD5HzfWiby4t7Q8Ef0gbsd4WVivq1Ih5\nL6FatbZw5rA2AACWIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAliGc\nAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAyxDOAABYhnAGAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAs\nQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLEM4A\nAFiGcAYAwDKOMcY0dyckyXEcWdKVgDiOo5bT2/PnSOe8f5qqVj4NkU/efz/2yiufJMkrn7xa1gQ9\naBl1Omu9MjJymmG951Yr6tSIeS+hWp1PnZpLfbkXUDgvXbpUDz30kPx+v3JycjR16tSz2kyePFlL\nlixRSEiI5s2bJ4/HI0kqLy/XhAkTtHbtWgUFBemll15SZmZmozppI8K5EfNeQrVqiXVqaaFDnRox\n7yVUq9YWzm0amtnv92vSpEkqKipSZGSkMjIyNHr0aCUkJLhtlixZoi1btmjTpk36xz/+ofvvv1/F\nxcWSpAcffFAjRozQn//8Z1VVVeno0aMXaLMA4OI6/WjMEPk0TT+V1LRHY3BpajCcV61apbi4OHXr\n1k2SlJWVpYKCghrhXFBQoOzsbElSZmamysvLtWfPHrVv317Lly/XvHnzTq6sTRuFhoZehM0AgAvP\nq2WnhfD0Zu0LLi0NhnNZWZliYmLc6ejoaK1atareNlFRUSorK1NwcLDCw8M1fvx4ffTRR0pPT9fs\n2bPVvn37C7gJABqLESFgtwbD+XxUVVVp9erVev7555Wenq6HHnpITzzxhKZPr/0T6LRp09zHXq9X\nXq/3YnYPuGQxIgSans/nk8/nC6htg+EcFRWl7du3u9OlpaWKioo6q82OHTtqbRMTE6P09HRJ0pgx\nYzRz5sw613V6OAMA0JqcOeisa6AqBXCdc0ZGhjZv3qySkhJVVlYqPz9fo0aNqtFm1KhRevnllyVJ\nxcXF6tSpkyIiIhQREaGYmBht3LhRklRUVKQ+ffqcyzYBAHDJaHDkHBwcrDlz5mj48OHupVSJiYnK\nzc2V4ziaOHGiRowYocLCQsXGxiokJERz585153/22Wd111136cSJE+rZs2eN5wAAwNm4Cck5upSu\n3ZVa5rWWzYE6Ba6lXefcXFria4rrnANTX+5x+04AACxDOAMAYBnCGQAAyxDOAABYhnAGAMAyhDMA\nAJYhnAEAsAzhDACAZQhnAAAswx3CzhF3KWrEvJdQrahT4LhDWGBaymvq9H9D6pNXXvkkNd2/IW1t\ndwgjnM8RfyAaMe8lVCvqFDjCOTC8pgLT2sKZw9oAAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBlCGcA\nACxDOAMAYBnCGQAAyxDOAABYhnAGAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQ\nzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAA\nlgkonJcuXaqEhAT17t1bM2fOrLXN5MmTFRcXJ4/HozVr1tR4zu/3KzU1VaNGjTr/HgMA0Mo1GM5+\nv1+TJk3SX//6V61bt055eXn65JNParRZsmSJtmzZok2bNik3N1f3339/jednz56tPn36XNieAwDQ\nSjUYzqtWrVJcXJy6deumtm3bKisrSwUFBTXaFBQUKDs7W5KUmZmp8vJy7dmzR5JUWlqqwsJCTZgw\n4SJ0HwCA1qfBcC4rK1NMTIw7HR0drbKysnrbREVFuW0efvhhPfXUU3Ic50L1GQCAVu2inhC2ePFi\nRUREyOPxyBgjY8zFXB0AAK1Cm4YaREVFafv27e50aWmpoqKizmqzY8eOs9q8/vrrWrRokQoLC/XF\nF1+ooqJC2dnZevnll2td17Rp09zHXq9XXq+3kZsDAICdfD6ffD5fYI1NA6qqqkyvXr3Mtm3bzPHj\nx01ycrJZv359jTaLFy82I0aMMMYYs3LlSpOZmXnWcnw+nxk5cmSd6wmgK1aRZMwl9HM+++dSqhV1\nuvi1ok7U6kLXqbnU1+cGR87BwcGaM2eOhg8fLr/fr5ycHCUmJio3N1eO42jixIkaMWKECgsLFRsb\nq5CQEM2dO/ccPlMAAABJcv6d3s3OcRxZ0pWAOI6jltPb8+dI57x/LqVaUafAnWutqFMj5r2EanU+\ndWou9eUedwgDAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQz\nAACWIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBl\nCGcAACxDOAMAYBnCGQAAyxDOAABYhnAGAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkA\nAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLBBTOS5cuVUJCgnr3\n7q2ZM2fW2mby5MmKi4uTx+PRmjVrJEmlpaUaNmyYrrnmGiUlJenZZ5+9cD0HAKC1Mg2orq42vXr1\nMtu2bTOVlZUmOTnZfPzxxzXaFBYWmhEjRhhjjCkuLjaZmZnGGGN27dplPvzwQ2OMMRUVFaZ3795n\nzXtKAF2xiiRjLqGf89k/l1KtqNPFrxV1olYXuk7Npb4+NzhyXrVqleLi4tStWze1bdtWWVlZKigo\nqNGmoKBA2dnZkqTMzEyVl5drz5496tq1qzwejySpQ4cOSkxMVFlZ2QX6WAEAQOvUYDiXlZUpJibG\nnY6Ojj4rYM9sExUVdVabbdu2ac2aNcrMzDzfPgMA0Ko1yQlhhw8f1pgxYzR79mx16NChKVYJAECL\n1aahBlFRUdq+fbs7XVpaqqioqLPa7Nixo9Y2VVVVGjNmjO655x6NHj263nVNmzbNfez1euX1egPZ\nBgAArOfz+eTz+QJq6/z7S+k6VVdXKz4+XkVFRbrqqqvUv39/5eXlKTEx0W1TWFio559/XosXL1Zx\ncbEeeughFRcXS5Kys7MVHh6uWbNm1d8Rx1EDXbGK4zhqOb09f450zvvnUqoVdQrcudaKOjVi3kuo\nVudTp+ZSX+41OHIODg7WnDlzNHz4cPn9fuXk5CgxMVG5ublyHEcTJ07UiBEjVFhYqNjYWIWEhGje\nvHmSpBUrVmjBggVKSkpSSkqKHMfRjBkzdNNNN13QDQQAoDVpcOTcVBg5241P74GhToFj5BwYXlOB\naW0jZ+4QBgCAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGUIZwAA\nLEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAyxDO\nAABYhnAGAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACW\nIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAlgkonJcuXaqEhAT17t1b\nM2fOrLXN5MmTFRcXJ4/HozVr1jRqXgAAcBrTgOrqatOrVy+zbds2U1lZaZKTk83HH39co01hYaEZ\nMWKEMcaY4uJik5mZGfC8pwTQFatIMqYZft5tpvWez/5pjlpRp9ZbK+pErS50nZpLfX1ucOS8atUq\nxcXFqVv2vWOSAAAR7UlEQVS3bmrbtq2ysrJUUFBQo01BQYGys7MlSZmZmSovL9eePXsCmheN42vu\nDrQQvubuQAvia+4OtBC+5u5AC+Jr7g60Ag2Gc1lZmWJiYtzp6OholZWVBdQmkHkBAEBNF+WEsJOj\ndQAAcC7aNNQgKipK27dvd6dLS0sVFRV1VpsdO3ac1aaysrLBeU/nOE6jOt/cmqu305tpveezf5qj\nVtQpcC2tVtSpEfNewH40RnPUqqVlSH0aDOeMjAxt3rxZJSUluuqqq5Sfn6+8vLwabUaNGqXnn39e\nd9xxh4qLi9WpUydFREQoPDy8wXlPYbQNAMBJDYZzcHCw5syZo+HDh8vv9ysnJ0eJiYnKzc2V4zia\nOHGiRowYocLCQsXGxiokJERz586td14AAFA3xzBkBQDAKtwhrAn94he/UN++fZWcnKzU1FS9//77\ndbYdOnSoVq9eLUm65ZZbdOjQobPaTJ8+XbNmzZIk/fSnP9U777wjSZo9e7aOHTt2EbbgwggODlZq\naqpSUlKUmpqqJ598UlLNbb7YysvL9cILL7jTu3bt0je+8Y0mWXddFi5cqKCgIG3cuPGir+vM7T/T\n9OnTFR0drdTUVPXt21f5+fkNLrOgoECffPKJO336a7Jjx47n3+lz8Oabb7qvr/PV0Ptq4sSJNba/\nIUOHDlVCQoL7PnjjjTca3adly5Zp5cqV9bap6/12MX300UdasmRJnc9/8MEHeuihh+pdxqm/ew29\nVlutJrrW+pK3cuVKM3DgQHPixAljjDH79+83u3btqrO91+s1H3zwQb3LnDZtmnn66afP+n337t3N\n/v37z6/DF1HHjh1r/X0g23yhfPrpp6Zv375Nsq5A3XHHHWbw4MFm2rRpF31dDW3/6a+tTZs2mdDQ\nUFNVVVXvMseNG2def/31Wp+ra59fTA31t7Hqe19VV1c3enler9esXr36vPo0bdo086tf/areNmfW\n/kLXpTbz5s0zkyZNqvW5xq7fxvdqU2Dk3ER27dql8PBwtWlz8mv+sLAwde3aVUVFRUpNTVVycrIm\nTJigEydOnDVvjx49dODAAUknR9/x8fEaPHiwNmzY4LYZP3683njjDT333HPauXOnhg4dqhtuuEFz\n587Vww8/7Lb7wx/+oEceeeQib239TADfpOTl5alfv37q16+fHnvsMff3HTt21OOPPy6Px6OBAwfq\ns88+kyRt3bpV1157rZKTk/XjH//YHakdOXJEX/nKV5Senq7k5GS9+eabkqQf/OAH2rp1q1JTUzV1\n6lSVlJQoKSlJknT8+HHdd9996tevn9LS0uTz+SRJ8+fP1+23366bb75Z8fHxmjp16gWryZEjR7Ri\nxQq9+OKL7kmTu3fv1pAhQ5Samqp+/fppxYoVbg2mTJmivn376sYbb9T+/fvdGtx8883KyMjQkCFD\n3BH43r17ddttt8nj8SglJUXFxcX6wQ9+oC1btrjbX59T55IcPHhQ0snXUP/+/ZWSkqKxY8fq2LFj\nWrlypRYtWqRHH31Uqamp+vTTT93XpCRVV1crNTVVffr00VVXXaWUlJQa21Tf/m7Mto4fP17f+c53\ndO2112rq1KmaP3++vvvd77rPPfDAA7r22msVGxurZcuWKScnR3369NF9993nrvN//ud/NHDgQKWn\np+uOO+7QkSNHznpfnerb9773PaWkpGjlypU1jvwsXbpUaWlpSklJ0Y033lhnbf1+/1m/u/XWW5WR\nkaGkpCT94Q9/cH9/5jJLSkr029/+Vs8884xSU1O1YsUKlZSU6IYbbpDH49GNN96o0tJSGWPcugwY\nMMDd3++//76uu+46eTweDRgwQEeOHJHf79f3v/99JSUlyePx6Pnnn5ckrV69Wl6vVxkZGbr55pu1\nZ88eSSdH/4899pgyMzOVkJCgFStW6MSJE/rJT36i1157Tampqfrzn/+s6dOnKzs7W9dff72ys7O1\nbNkyjRw5UtLJ1/6p95vH49Ff/vIXSf/5u3fma/Xee++tcTOru+++231ftyrN/engUnH48GHj8XhM\nfHy8eeCBB8yyZcvMsWPHTExMjNm8ebMxxpjs7Gwze/ZsY0zNUWSPHj3M/v37zQcffGD69etnjh07\nZg4dOmRiY2Pd0c24cePMf//3fxtjTn7CP3DggLve2NhY99PqwIEDzdq1a5t0288UHBxsUlJSjMfj\nMSkpKea1114zxvxnm3fu3Gmuvvpqs3//flNdXW2GDRtmCgoKjDHGOI5jFi9ebIwx5tFHHzW/+MUv\njDHG3HLLLebVV181xhjz29/+1h0tVFVVmYqKCmOMMfv27TOxsbHGGGO2bdtmkpKS3D6dPv3000+b\nnJwcY4wxn3zyibn66qvN8ePHzbx580yvXr1MRUWFOXbsmOnWrZspLS29IDVZsGCBmTBhgjHGmOuu\nu86sXr3aPP3002bGjBnGGGP8fr85fPiwW4O8vDxjjDE/+9nPzHe/+11jjDE33HCD+1r6xz/+YYYN\nG2aMOTkiP/W68vv95tChQ2dt/5lOHzl/8MEHZvDgwe5zp15bxhjz+OOPmzlz5hhjar4GT58+duyY\ncRzHbN682Tz99NMmJSXFzJ49292mhvZ3Y7Z13LhxZuTIkW4f5s2b584zbtw4c+eddxpjjCkoKDCh\noaFm3bp1xhhj0tLSzEcffWT27dtnBg8ebI4ePWqMMWbmzJnmv/7rv4wxNd9Xp/p2+pGCU6/fzz77\nzMTExJiSkhJjjDEHDx6stcZer9ckJCS474NTyz7V/osvvjB9+/Y1Bw4cqHOZZx49GzlypHnllVeM\nMca89NJL5utf/7oJDg42YWFhJjQ01H2/VVZWmp49e7p/YyoqKkxVVZV54YUXzNixY43f73fXc+LE\nCTNw4ECzb98+Y4wxr776qrnvvvvcbfje975njDl5G+evfOUrZ9X9VD/T09PN8ePHjTHG+Hw+dz9N\nnTrVPPzww27bzz//3Bjzn797Z75Wly1bZr7+9a8bY4wpLy83PXv2PKcjF7Zr8GxtXBghISFavXq1\nli9frnfeeUdZWVl67LHH1LNnT/Xq1UuSdO+99+o3v/mNJk+eXOsyli9frltvvVXt2rVTu3btNGrU\nqDrXZ/49Og0JCdGwYcP01ltvKSEhQVVVVbrmmmsu/AY2whVXXFHvd8vvv/++hg4dqrCwMEnSXXfd\npffee0+jRo3SZZddphEjRkiS0tLS9Pbbb0uSVq5c6X6a/uY3v6nvf//7kk7W4Qc/+IHee+89BQUF\naefOndq7d2+9/fvf//1fdx/Ex8ere/fu7sjshhtuUIcOHSRJffr0UUlJSb3X7gcqLy/P/Q7ujjvu\n0J/+9CeNHj1a48eP14kTJzR69GglJydLOvkd4qnvx++++27dfvvtOnLkiP7+979r7Nix7r4/dRTm\nnXfe0SuvvCLp5HWgHTt2dI/E1GfWrFl66aWXtGnTphojk3/96196/PHH9fnnn+vIkSP66le/Wu9y\nNmzYoKCgIPXq1UsZGRmaNWuWcnNzNWTIECUnJ6uoqKjO/R0UFNSobZWksWPH1tmXU6O1pKQkde3a\nVX369JEkXXPNNdq2bZt27Nih9evX67rrrpMxRidOnNDAgQPd+c1pR33atGmj22677ax1FBcXa8iQ\nIbr66qslSZ06daqzP3/605+UkpJS43fPPPOMFi5cKOnkvSE2bdqkvXv3BrTMlStXuiPPe+65R1On\nTtUVV1yhUaNGadiwYbrnnnskSWvXrlVkZKRSU1MlyX1Nv/322/rOd77jXi/cqVMnrVu3TmvXrtWN\nN94oY4z8fr8iIyPddZ6qQVpamkpKSurc1lPv3zO9/fbbevXVV93pL33pS5LqPsI2ePBg/b//9/+0\nf/9+vf7667r99tsVFNT6DgITzk3IcRwNHjxYgwcPVlJSknvI6GLLycnRjBkzlJCQoPHjxzfJOs9X\nXW/Mtm3buo+Dg4NVVVUlqebNB06fd8GCBdq3b58+/PBDBQUFqUePHo0+We705bVr167W9Z+PgwcP\n6p133tHatWvlOI6qq6vlOI6eeuopLV++XIsXL9a4ceP0yCOP6O677z6rNo7jyO/3q3PnzrV+6Ank\nxgyPP/64Fi9eLMdx3GVMmTJFU6ZM0Ztvvqn77rtPW7du1WWXXaZx48Zp0aJF6tu3r+bPn69ly5YF\nvK2DBg3SM888o1/+8pcaP368pkyZotDQ0IDvc9DQtkonP5DW5dT+CwoKqrEvg4KCVFVVpaCgIA0f\nPlwLFixosC+XX355nbWtbXtuuukm7d27V+np6frd735Xa7tly5bpnXfe0T/+8Q+1a9dOQ4cOdV+v\ngdSovn19Zl0CrbkxRn379nW/gjjTqTo29H6ob780VnZ2tl555RXl5+dr3rx5F2y5Nml9HzcstXHj\nRm3evNmdXrNmjWJjY7Vt2zZt3bpVkvTKK6/I6/WeNe+pN9HgwYO1cOFCHT9+XBUVFXV+zxIaGlrj\n7O7+/ftrx44dysvL05133nkBt+rcNPRHoX///nrvvfd04MABVVdXKy8vr9a6nG7AgAF6/fXXJanG\nmcXl5eXq0qWLgoKC9O6777qf7Dt27KiKiopalzVo0CD3j/PGjRu1Y8cOxcfHB7p5jfbnP/9Z2dnZ\n+vTTT7V161aVlJSoR48eeu+999SlSxfl5ORowoQJbhj5/X53WxcsWKDrr79eHTt2VI8ePdzfS9I/\n//lPSSdH+7/5zW/ceQ8dOnTW9v/85z/Xhx9+WGvgjRw5UhkZGZo/f74k6fDhw+ratatOnDhRI8Q6\nduxY61UF8fHx8vv92rp1q7Zv36633npL48aNU05OjlavXl3v/m7stjZGba/DAQMGaMWKFdqyZYsk\n6ejRo9q0aZOks99Xdb2OBwwYoOXLl7uvtVPf1S9dulSrV692g7k25eXl6ty5s9q1a6dPPvlExcXF\n9S7zzJoPHDjQPWfhj3/8owYNGlRrP+Pj47V792598MEHkk7u0+rqat14443Kzc1VdXW1u574+Hh9\n9tlnbl+qqqq0fv36Wvt/al11vRZqc+ONN9YYqHz++ec1nq/tvXrvvffqmWeekeM4SkhICGg9LQ3h\n3EQOHz6se++9V3379pXH49HHH3+sJ554QnPnztWYMWOUnJys4OBgffvb35ZU8xPwqccpKSm64447\n1K9fP33ta19T//79z2ojSd/61rd00003uSeuSNI3vvENXXfdde4ho+Z07NixGpd2/PCHP5T0n23o\n2rWrnnjiCXm9XqWkpCg9PV233HJLjTZn+vWvf61Zs2bJ4/Foy5Yt7nbeddddev/995WcnKw//vGP\n7k1wwsLCdN1116lfv35nnRD1wAMPqLq6Wv369dOdd96p+fPn1xixn3KhbhX46quv6tZbb63xu9tu\nu03jx4+Xx+NRamqqXnvtNfewd0hIiFatWqWkpCT5fD795Cc/kXQyvF588UV5PB717dtXixYtknTy\nMOm7776rfv36KT09XR9//LHCwsI0cODAWre/Nj/+8Y/dy/Z+9rOfqX///ho0aFCNmwplZWXpqaee\nUlpamj799FO3Pu3atVP79u01ZswYXX/99Vq0aJFeeuklvfbaa3rwwQfr3d+N3db69smZz9X2HgsP\nD9e8efN05513Kjk5WQMHDnRPvDzzfVXX8sLDw/W73/1Ot956q1JSUpSVlRVQf6STo+sTJ07ommuu\n0Q9/+ENde+219S5z5MiR+stf/uKeEPbcc89p7ty58ng8WrBggXv516JFi/S9733Pfb+1bdtW+fn5\nmjRpkjwej4YPH67jx49rwoQJiomJUb9+/ZSSkqK8vDy1bdtWr7/+uqZOneqeVHjq8q26ajB06FCt\nX7/ePSGsvv3y+OOP68CBA0pKSlJKSop7AuapeWp7r3bp0kWJiYkt5kjgueAmJJeIkSNHasqUKRo6\ndGhzd+Wi+OKLL9S+fXtJJ8MuPz/f/e6ttalv1N/aXErbisAdPXpUycnJWr16dbNdQ3+xMXJu5crL\nyxUfH6+QkJBWG8zSyZsaeDweJScn64UXXtDTTz/d3F26aFrTzf0bciltKwJTVFSkPn36aPLkya02\nmCVGzgAAWIeRMwAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAy/x/s+0kGbPyugQAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x8d5a7f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the top 5 feature importances of the forest\n",
    "plt.figure(num=None, figsize=(8, 6), dpi=80, facecolor='w', edgecolor='k')\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(5), importances[indices][:5], \n",
    "       color=\"r\",  yerr=std[indices][:5], align=\"center\")\n",
    "plt.xticks(range(5), best_features)\n",
    "plt.xlim([-1, 5])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree accuracy and time elapsed caculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTree\n",
      "Acurracy:  0.389705882353\n",
      "time elapsed:  0.0139999389648\n"
     ]
    }
   ],
   "source": [
    "t0=time()\n",
    "print \"DecisionTree\"\n",
    "\n",
    "dt = DecisionTreeClassifier(min_samples_split=20,random_state=99)\n",
    "# dt = DecisionTreeClassifier(min_samples_split=20,max_depth=5,random_state=99)\n",
    "\n",
    "clf_dt=dt.fit(X_train,y_train)\n",
    "\n",
    "print \"Acurracy: \", clf_dt.score(X_test,y_test)\n",
    "t1=time()\n",
    "print \"time elapsed: \", t1-t0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross validation for DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross result========\n",
      "[ 0.45528455  0.38596491  0.49514563]\n",
      "0.445465032065\n",
      "time elapsed:  0.121000051498\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tt0=time()\n",
    "print \"cross result========\"\n",
    "scores = cross_validation.cross_val_score(dt, X, y, cv=3)\n",
    "print scores\n",
    "print scores.mean()\n",
    "tt1=time()\n",
    "print \"time elapsed: \", tt1-tt0\n",
    "print \"\\n\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Tuning our hyperparameters using GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:   17.1s\n",
      "[Parallel(n_jobs=-1)]: Done  81 out of  81 | elapsed:   17.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 27 candidates, totalling 81 fits\n",
      "Best score: 0.551\n",
      "Best parameters set:\n",
      "\tclf__max_depth: 25\n",
      "\tclf__min_samples_leaf: 3\n",
      "\tclf__min_samples_split: 5\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       0.67      0.40      0.50         5\n",
      "          2       0.33      0.40      0.36         5\n",
      "          3       0.40      0.50      0.44         4\n",
      "          4       0.50      0.33      0.40         3\n",
      "          5       0.43      1.00      0.60         3\n",
      "          6       0.40      0.40      0.40         5\n",
      "          7       0.00      0.00      0.00         6\n",
      "          8       1.00      0.50      0.67         4\n",
      "          9       0.50      0.40      0.44         5\n",
      "         10       1.00      1.00      1.00         6\n",
      "         11       1.00      0.75      0.86         4\n",
      "         12       0.80      0.67      0.73         6\n",
      "         13       0.46      0.75      0.57         8\n",
      "         14       0.80      0.67      0.73         6\n",
      "         15       1.00      0.40      0.57         5\n",
      "         22       0.40      0.40      0.40         5\n",
      "         23       0.67      1.00      0.80         2\n",
      "         24       1.00      0.29      0.44         7\n",
      "         25       0.40      1.00      0.57         2\n",
      "         26       0.67      0.33      0.44         6\n",
      "         27       0.56      0.71      0.63         7\n",
      "         28       1.00      1.00      1.00         3\n",
      "         29       0.67      0.67      0.67         3\n",
      "         30       0.44      0.80      0.57         5\n",
      "         31       0.80      0.80      0.80         5\n",
      "         32       0.67      0.50      0.57         4\n",
      "         33       0.60      0.75      0.67         4\n",
      "         34       0.20      0.50      0.29         2\n",
      "         35       1.00      0.67      0.80         3\n",
      "         36       0.50      1.00      0.67         3\n",
      "\n",
      "avg / total       0.64      0.59      0.58       136\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Miniconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:1074: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('clf', DecisionTreeClassifier(criterion='entropy'))\n",
    "])\n",
    "\n",
    "parameters = {\n",
    "    'clf__max_depth': (5, 25 , 50),\n",
    "    'clf__min_samples_split': (1, 5, 10),\n",
    "    'clf__min_samples_leaf': (1, 2, 3)\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='f1')\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "print 'Best score: %0.3f' % grid_search.best_score_\n",
    "print 'Best parameters set:'\n",
    "\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print '\\t%s: %r' % (param_name, best_parameters[param_name])\n",
    "\n",
    "predictions = grid_search.predict(X_test)\n",
    "\n",
    "print classification_report(y_test, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest accuracy and time elapsed caculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForest\n",
      "Acurracy:  0.683823529412\n",
      "time elapsed:  1.16700005531\n"
     ]
    }
   ],
   "source": [
    "t2=time()\n",
    "print \"RandomForest\"\n",
    "rf = RandomForestClassifier(n_estimators=100,n_jobs=-1)\n",
    "clf_rf = rf.fit(X_train,y_train)\n",
    "print \"Acurracy: \", clf_rf.score(X_test,y_test)\n",
    "t3=time()\n",
    "print \"time elapsed: \", t3-t2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross validation for RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross result========\n",
      "[ 0.66666667  0.79824561  0.76699029]\n",
      "0.743967523988\n",
      "time elapsed:  3.51099991798\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tt2=time()\n",
    "print \"cross result========\"\n",
    "scores = cross_validation.cross_val_score(rf, X, y, cv=3)\n",
    "print scores\n",
    "print scores.mean()\n",
    "tt3=time()\n",
    "print \"time elapsed: \", tt3-tt2\n",
    "print \"\\n\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuning Models using GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:   19.3s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   29.0s\n",
      "[Parallel(n_jobs=-1)]: Done 324 out of 324 | elapsed:   37.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 108 candidates, totalling 324 fits\n",
      "Best score: 0.716\n",
      "Best parameters set:\n",
      "\tclf__max_depth: 50\n",
      "\tclf__min_samples_leaf: 1\n",
      "\tclf__min_samples_split: 1\n",
      "\tclf__n_estimators: 50\n",
      "Accuracy: 0.698529411765\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       1.00      0.80      0.89         5\n",
      "          2       0.67      0.40      0.50         5\n",
      "          3       1.00      0.50      0.67         4\n",
      "          4       0.67      0.67      0.67         3\n",
      "          5       0.60      1.00      0.75         3\n",
      "          6       1.00      0.60      0.75         5\n",
      "          7       1.00      0.50      0.67         6\n",
      "          8       1.00      1.00      1.00         4\n",
      "          9       0.38      0.60      0.46         5\n",
      "         10       0.71      0.83      0.77         6\n",
      "         11       0.67      1.00      0.80         4\n",
      "         12       0.55      1.00      0.71         6\n",
      "         13       0.78      0.88      0.82         8\n",
      "         14       1.00      0.50      0.67         6\n",
      "         15       1.00      1.00      1.00         5\n",
      "         22       0.33      0.20      0.25         5\n",
      "         23       0.67      1.00      0.80         2\n",
      "         24       1.00      0.43      0.60         7\n",
      "         25       0.67      1.00      0.80         2\n",
      "         26       0.67      0.67      0.67         6\n",
      "         27       0.67      0.57      0.62         7\n",
      "         28       0.33      0.33      0.33         3\n",
      "         29       0.75      1.00      0.86         3\n",
      "         30       0.80      0.80      0.80         5\n",
      "         31       0.75      0.60      0.67         5\n",
      "         32       0.43      0.75      0.55         4\n",
      "         33       0.60      0.75      0.67         4\n",
      "         34       0.33      0.50      0.40         2\n",
      "         35       1.00      0.67      0.80         3\n",
      "         36       1.00      1.00      1.00         3\n",
      "\n",
      "avg / total       0.75      0.70      0.69       136\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pipeline2 = Pipeline([\n",
    "('clf', RandomForestClassifier(criterion='entropy'))\n",
    "])\n",
    "\n",
    "parameters = {\n",
    "    'clf__n_estimators': (5, 25, 50, 100),\n",
    "    'clf__max_depth': (5, 25 , 50),\n",
    "    'clf__min_samples_split': (1, 5, 10),\n",
    "    'clf__min_samples_leaf': (1, 2, 3)\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(pipeline2, parameters, n_jobs=-1, verbose=1, scoring='accuracy', cv=3)\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "print 'Best score: %0.3f' % grid_search.best_score_\n",
    "\n",
    "print 'Best parameters set:'\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print '\\t%s: %r' % (param_name, best_parameters[param_name])\n",
    "\n",
    "predictions = grid_search.predict(X_test)\n",
    "print 'Accuracy:', accuracy_score(y_test, predictions)\n",
    "print classification_report(y_test, predictions)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes accuracy and time elapsed caculation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NaiveBayes\n",
      "Acurracy:  0.0294117647059\n",
      "time elapsed:  0.018000125885\n"
     ]
    }
   ],
   "source": [
    "t4=time()\n",
    "print \"NaiveBayes\"\n",
    "nb = BernoulliNB()\n",
    "clf_nb=nb.fit(X_train,y_train)\n",
    "print \"Acurracy: \", clf_nb.score(X_test,y_test)\n",
    "t5=time()\n",
    "print \"time elapsed: \", t5-t4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross-validation for NB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross result========\n",
      "[ 0.04878049  0.04385965  0.04854369]\n",
      "0.047061275416\n",
      "time elapsed:  0.0650000572205\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tt4=time()\n",
    "print \"cross result========\"\n",
    "scores = cross_validation.cross_val_score(nb, X,y, cv=3)\n",
    "print scores\n",
    "print scores.mean()\n",
    "tt5=time()\n",
    "print \"time elapsed: \", tt5-tt4\n",
    "print \"\\n\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNN accuracy and time elapsed caculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN\n",
      "Acurracy:  0.0955882352941\n",
      "time elapsed:  0.00799989700317\n"
     ]
    }
   ],
   "source": [
    "t6=time()\n",
    "print \"KNN\"\n",
    "# knn = KNeighborsClassifier(n_neighbors=3)\n",
    "knn = KNeighborsClassifier(n_neighbors=3)\n",
    "clf_knn=knn.fit(X_train, y_train)\n",
    "print \"Acurracy: \", clf_knn.score(X_test,y_test) \n",
    "t7=time()\n",
    "print \"time elapsed: \", t7-t6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross validation for KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross result========\n",
      "[ 0.1686747   0.13513514  0.07692308  0.16666667  0.15517241]\n",
      "0.140514398263\n",
      "time elapsed:  0.0729999542236\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tt6=time()\n",
    "print \"cross result========\"\n",
    "scores = cross_validation.cross_val_score(knn, X,y, cv=5)\n",
    "print scores\n",
    "print scores.mean()\n",
    "tt7=time()\n",
    "print \"time elapsed: \", tt7-tt6\n",
    "print \"\\n\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine tuning the model using GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
      "Best score: 0.186\n",
      "Best parameters set:\n",
      "\tn_neighbors: 1\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       0.00      0.00      0.00         5\n",
      "          2       0.00      0.00      0.00         5\n",
      "          3       0.00      0.00      0.00         4\n",
      "          4       0.00      0.00      0.00         3\n",
      "          5       0.00      0.00      0.00         3\n",
      "          6       0.50      0.20      0.29         5\n",
      "          7       0.00      0.00      0.00         6\n",
      "          8       1.00      1.00      1.00         4\n",
      "          9       0.00      0.00      0.00         5\n",
      "         10       0.00      0.00      0.00         6\n",
      "         11       0.50      0.75      0.60         4\n",
      "         12       0.00      0.00      0.00         6\n",
      "         13       0.00      0.00      0.00         8\n",
      "         14       0.00      0.00      0.00         6\n",
      "         15       0.50      0.20      0.29         5\n",
      "         22       0.00      0.00      0.00         5\n",
      "         23       0.00      0.00      0.00         2\n",
      "         24       0.00      0.00      0.00         7\n",
      "         25       0.00      0.00      0.00         2\n",
      "         26       0.00      0.00      0.00         6\n",
      "         27       0.00      0.00      0.00         7\n",
      "         28       0.00      0.00      0.00         3\n",
      "         29       0.00      0.00      0.00         3\n",
      "         30       0.00      0.00      0.00         5\n",
      "         31       1.00      0.40      0.57         5\n",
      "         32       0.00      0.00      0.00         4\n",
      "         33       0.00      0.00      0.00         4\n",
      "         34       0.40      1.00      0.57         2\n",
      "         35       0.00      0.00      0.00         3\n",
      "         36       0.00      0.00      0.00         3\n",
      "\n",
      "avg / total       0.12      0.10      0.10       136\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   6 out of   6 | elapsed:   13.1s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn import grid_search\n",
    "\n",
    "knn = KNeighborsClassifier()\n",
    "\n",
    "parameters = {'n_neighbors':[1,10]}\n",
    "\n",
    "grid = grid_search.GridSearchCV(knn, parameters, n_jobs=-1, verbose=1, scoring='accuracy')\n",
    "\n",
    "\n",
    "grid.fit(X_train, y_train)\n",
    "\n",
    "print 'Best score: %0.3f' % grid.best_score_\n",
    "\n",
    "print 'Best parameters set:'\n",
    "best_parameters = grid.best_estimator_.get_params()\n",
    "\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print '\\t%s: %r' % (param_name, best_parameters[param_name])\n",
    "    \n",
    "predictions = grid.predict(X_test)\n",
    "print classification_report(y_test, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM accuracy and time elapsed caculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM\n",
      "Acurracy:  0.132352941176\n",
      "time elapsed:  0.0260000228882\n"
     ]
    }
   ],
   "source": [
    "t7=time()\n",
    "print \"SVM\"\n",
    "\n",
    "svc = SVC()\n",
    "clf_svc=svc.fit(X_train, y_train)\n",
    "print \"Acurracy: \", clf_svc.score(X_test,y_test) \n",
    "t8=time()\n",
    "print \"time elapsed: \", t8-t7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross validation for SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross result========\n",
      "[ 0.18072289  0.2027027   0.24615385  0.21666667  0.15517241]\n",
      "0.200283704177\n",
      "time elapsed:  34.7860000134\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tt7=time()\n",
    "print \"cross result========\"\n",
    "scores = cross_validation.cross_val_score(svc,X,y, cv=5)\n",
    "print scores\n",
    "print scores.mean()\n",
    "tt8=time()\n",
    "print \"time elapsed: \", tt7-tt6\n",
    "print \"\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
      "Best score: 0.515\n",
      "Best parameters set:\n",
      "\tC: 10\n",
      "\tkernel: 'linear'\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       0.62      1.00      0.77         5\n",
      "          2       0.50      0.20      0.29         5\n",
      "          3       1.00      0.75      0.86         4\n",
      "          4       0.20      0.33      0.25         3\n",
      "          5       0.38      1.00      0.55         3\n",
      "          6       0.57      0.80      0.67         5\n",
      "          7       0.62      0.83      0.71         6\n",
      "          8       1.00      1.00      1.00         4\n",
      "          9       0.60      0.60      0.60         5\n",
      "         10       1.00      0.67      0.80         6\n",
      "         11       0.75      0.75      0.75         4\n",
      "         12       0.33      0.17      0.22         6\n",
      "         13       0.67      0.25      0.36         8\n",
      "         14       0.20      0.17      0.18         6\n",
      "         15       0.71      1.00      0.83         5\n",
      "         22       1.00      0.20      0.33         5\n",
      "         23       1.00      1.00      1.00         2\n",
      "         24       0.50      0.29      0.36         7\n",
      "         25       0.50      0.50      0.50         2\n",
      "         26       0.22      0.33      0.27         6\n",
      "         27       0.75      0.43      0.55         7\n",
      "         28       0.50      0.33      0.40         3\n",
      "         29       1.00      0.67      0.80         3\n",
      "         30       0.67      0.80      0.73         5\n",
      "         31       0.80      0.80      0.80         5\n",
      "         32       0.40      0.50      0.44         4\n",
      "         33       0.20      0.50      0.29         4\n",
      "         34       0.50      0.50      0.50         2\n",
      "         35       0.00      0.00      0.00         3\n",
      "         36       1.00      0.33      0.50         3\n",
      "\n",
      "avg / total       0.61      0.54      0.53       136\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  12 out of  12 | elapsed:   18.0s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn import grid_search\n",
    "\n",
    "svc = SVC()\n",
    "\n",
    "parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}\n",
    "\n",
    "grid = grid_search.GridSearchCV(svc, parameters, n_jobs=-1, verbose=1, scoring='accuracy')\n",
    "\n",
    "\n",
    "grid.fit(X_train, y_train)\n",
    "\n",
    "print 'Best score: %0.3f' % grid.best_score_\n",
    "\n",
    "print 'Best parameters set:'\n",
    "best_parameters = grid.best_estimator_.get_params()\n",
    "\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print '\\t%s: %r' % (param_name, best_parameters[param_name])\n",
    "    \n",
    "predictions = grid.predict(X_test)\n",
    "print classification_report(y_test, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:   18.2s\n",
      "[Parallel(n_jobs=-1)]: Done  90 out of  90 | elapsed:   19.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 30 candidates, totalling 90 fits\n",
      "Best score: 0.525\n",
      "Best parameters set:\n",
      "\tclf__C: 30\n",
      "\tclf__gamma: 0.01\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       0.71      1.00      0.83         5\n",
      "          2       0.50      0.20      0.29         5\n",
      "          3       1.00      0.75      0.86         4\n",
      "          4       0.17      0.33      0.22         3\n",
      "          5       0.43      1.00      0.60         3\n",
      "          6       0.57      0.80      0.67         5\n",
      "          7       0.83      0.83      0.83         6\n",
      "          8       1.00      1.00      1.00         4\n",
      "          9       0.33      0.40      0.36         5\n",
      "         10       0.80      0.67      0.73         6\n",
      "         11       0.75      0.75      0.75         4\n",
      "         12       0.20      0.17      0.18         6\n",
      "         13       0.67      0.25      0.36         8\n",
      "         14       0.33      0.17      0.22         6\n",
      "         15       0.67      0.80      0.73         5\n",
      "         22       1.00      0.20      0.33         5\n",
      "         23       1.00      1.00      1.00         2\n",
      "         24       0.67      0.29      0.40         7\n",
      "         25       0.50      0.50      0.50         2\n",
      "         26       0.25      0.33      0.29         6\n",
      "         27       0.67      0.57      0.62         7\n",
      "         28       0.50      0.33      0.40         3\n",
      "         29       1.00      0.67      0.80         3\n",
      "         30       0.67      0.80      0.73         5\n",
      "         31       1.00      1.00      1.00         5\n",
      "         32       0.50      0.75      0.60         4\n",
      "         33       0.18      0.50      0.27         4\n",
      "         34       1.00      1.00      1.00         2\n",
      "         35       0.00      0.00      0.00         3\n",
      "         36       0.50      0.33      0.40         3\n",
      "\n",
      "avg / total       0.61      0.55      0.55       136\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pipeline = Pipeline([\n",
    "    ('clf', SVC(kernel='linear', gamma=0.01, C=10))\n",
    "])\n",
    "\n",
    "parameters = {\n",
    "    'clf__gamma': (0.01, 0.03, 0.1, 0.3, 1),\n",
    "    'clf__C': (0.1, 0.3, 1, 3, 10, 30),\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='accuracy')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "print 'Best score: %0.3f' % grid_search.best_score_\n",
    "\n",
    "print 'Best parameters set:'\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print '\\t%s: %r' % (param_name, best_parameters[param_name])\n",
    "    \n",
    "predictions = grid_search.predict(X_test)\n",
    "print classification_report(y_test, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}