KDD Panda Try.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "col_names = [\"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\n",
    "    \"dst_bytes\",\"land\",\"wrong_fragment\",\"urgent\",\"hot\",\"num_failed_logins\",\n",
    "    \"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\n",
    "    \"num_file_creations\",\"num_shells\",\"num_access_files\",\"num_outbound_cmds\",\n",
    "    \"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\n",
    "    \"srv_serror_rate\",\"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\n",
    "    \"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\",\n",
    "    \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\n",
    "    \"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\"dst_host_srv_serror_rate\",\n",
    "    \"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\"label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>494021.000000</td>\n",
       "      <td>4.940210e+05</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>47.979302</td>\n",
       "      <td>3.025610e+03</td>\n",
       "      <td>868.532425</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>0.006433</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.034519</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>0.148247</td>\n",
       "      <td>0.010212</td>\n",
       "      <td>...</td>\n",
       "      <td>232.470778</td>\n",
       "      <td>188.665670</td>\n",
       "      <td>0.753780</td>\n",
       "      <td>0.030906</td>\n",
       "      <td>0.601935</td>\n",
       "      <td>0.006684</td>\n",
       "      <td>0.176754</td>\n",
       "      <td>0.176443</td>\n",
       "      <td>0.058118</td>\n",
       "      <td>0.057412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>707.746472</td>\n",
       "      <td>9.882181e+05</td>\n",
       "      <td>33040.001252</td>\n",
       "      <td>0.006673</td>\n",
       "      <td>0.134805</td>\n",
       "      <td>0.005510</td>\n",
       "      <td>0.782103</td>\n",
       "      <td>0.015520</td>\n",
       "      <td>0.355345</td>\n",
       "      <td>1.798326</td>\n",
       "      <td>...</td>\n",
       "      <td>64.745380</td>\n",
       "      <td>106.040437</td>\n",
       "      <td>0.410781</td>\n",
       "      <td>0.109259</td>\n",
       "      <td>0.481309</td>\n",
       "      <td>0.042133</td>\n",
       "      <td>0.380593</td>\n",
       "      <td>0.380919</td>\n",
       "      <td>0.230590</td>\n",
       "      <td>0.230140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.500000e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>0.410000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.200000e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.032000e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.040000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>58329.000000</td>\n",
       "      <td>6.933756e+08</td>\n",
       "      <td>5155468.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>884.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            duration     src_bytes       dst_bytes           land  \\\n",
       "count  494021.000000  4.940210e+05   494021.000000  494021.000000   \n",
       "mean       47.979302  3.025610e+03      868.532425       0.000045   \n",
       "std       707.746472  9.882181e+05    33040.001252       0.006673   \n",
       "min         0.000000  0.000000e+00        0.000000       0.000000   \n",
       "25%         0.000000  4.500000e+01        0.000000       0.000000   \n",
       "50%         0.000000  5.200000e+02        0.000000       0.000000   \n",
       "75%         0.000000  1.032000e+03        0.000000       0.000000   \n",
       "max     58329.000000  6.933756e+08  5155468.000000       1.000000   \n",
       "\n",
       "       wrong_fragment         urgent            hot  num_failed_logins  \\\n",
       "count   494021.000000  494021.000000  494021.000000      494021.000000   \n",
       "mean         0.006433       0.000014       0.034519           0.000152   \n",
       "std          0.134805       0.005510       0.782103           0.015520   \n",
       "min          0.000000       0.000000       0.000000           0.000000   \n",
       "25%          0.000000       0.000000       0.000000           0.000000   \n",
       "50%          0.000000       0.000000       0.000000           0.000000   \n",
       "75%          0.000000       0.000000       0.000000           0.000000   \n",
       "max          3.000000       3.000000      30.000000           5.000000   \n",
       "\n",
       "           logged_in  num_compromised            ...             \\\n",
       "count  494021.000000    494021.000000            ...              \n",
       "mean        0.148247         0.010212            ...              \n",
       "std         0.355345         1.798326            ...              \n",
       "min         0.000000         0.000000            ...              \n",
       "25%         0.000000         0.000000            ...              \n",
       "50%         0.000000         0.000000            ...              \n",
       "75%         0.000000         0.000000            ...              \n",
       "max         1.000000       884.000000            ...              \n",
       "\n",
       "       dst_host_count  dst_host_srv_count  dst_host_same_srv_rate  \\\n",
       "count   494021.000000       494021.000000           494021.000000   \n",
       "mean       232.470778          188.665670                0.753780   \n",
       "std         64.745380          106.040437                0.410781   \n",
       "min          0.000000            0.000000                0.000000   \n",
       "25%        255.000000           46.000000                0.410000   \n",
       "50%        255.000000          255.000000                1.000000   \n",
       "75%        255.000000          255.000000                1.000000   \n",
       "max        255.000000          255.000000                1.000000   \n",
       "\n",
       "       dst_host_diff_srv_rate  dst_host_same_src_port_rate  \\\n",
       "count           494021.000000                494021.000000   \n",
       "mean                 0.030906                     0.601935   \n",
       "std                  0.109259                     0.481309   \n",
       "min                  0.000000                     0.000000   \n",
       "25%                  0.000000                     0.000000   \n",
       "50%                  0.000000                     1.000000   \n",
       "75%                  0.040000                     1.000000   \n",
       "max                  1.000000                     1.000000   \n",
       "\n",
       "       dst_host_srv_diff_host_rate  dst_host_serror_rate  \\\n",
       "count                494021.000000         494021.000000   \n",
       "mean                      0.006684              0.176754   \n",
       "std                       0.042133              0.380593   \n",
       "min                       0.000000              0.000000   \n",
       "25%                       0.000000              0.000000   \n",
       "50%                       0.000000              0.000000   \n",
       "75%                       0.000000              0.000000   \n",
       "max                       1.000000              1.000000   \n",
       "\n",
       "       dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "count             494021.000000         494021.000000   \n",
       "mean                   0.176443              0.058118   \n",
       "std                    0.380919              0.230590   \n",
       "min                    0.000000              0.000000   \n",
       "25%                    0.000000              0.000000   \n",
       "50%                    0.000000              0.000000   \n",
       "75%                    0.000000              0.000000   \n",
       "max                    1.000000              1.000000   \n",
       "\n",
       "       dst_host_srv_rerror_rate  \n",
       "count             494021.000000  \n",
       "mean                   0.057412  \n",
       "std                    0.230140  \n",
       "min                    0.000000  \n",
       "25%                    0.000000  \n",
       "50%                    0.000000  \n",
       "75%                    0.000000  \n",
       "max                    1.000000  \n",
       "\n",
       "[8 rows x 38 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kdd_data_10percent = pandas.read_csv(\"kddcup.data_10_percent_corrected\", header=None, names = col_names)\n",
    "kdd_data_10percent.describe()\n",
    "#Training Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "smurf.              280790\n",
       "neptune.            107201\n",
       "normal.              97278\n",
       "back.                 2203\n",
       "satan.                1589\n",
       "ipsweep.              1247\n",
       "portsweep.            1040\n",
       "warezclient.          1020\n",
       "teardrop.              979\n",
       "pod.                   264\n",
       "nmap.                  231\n",
       "guess_passwd.           53\n",
       "buffer_overflow.        30\n",
       "land.                   21\n",
       "warezmaster.            20\n",
       "imap.                   12\n",
       "rootkit.                10\n",
       "loadmodule.              9\n",
       "ftp_write.               8\n",
       "multihop.                7\n",
       "phf.                     4\n",
       "perl.                    3\n",
       "spy.                     2\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kdd_data_10percent['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>494021.000000</td>\n",
       "      <td>4.940210e+05</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>47.979302</td>\n",
       "      <td>3.025610e+03</td>\n",
       "      <td>868.532425</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>0.006433</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.034519</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>0.148247</td>\n",
       "      <td>0.010212</td>\n",
       "      <td>...</td>\n",
       "      <td>232.470778</td>\n",
       "      <td>188.665670</td>\n",
       "      <td>0.753780</td>\n",
       "      <td>0.030906</td>\n",
       "      <td>0.601935</td>\n",
       "      <td>0.006684</td>\n",
       "      <td>0.176754</td>\n",
       "      <td>0.176443</td>\n",
       "      <td>0.058118</td>\n",
       "      <td>0.057412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>707.746472</td>\n",
       "      <td>9.882181e+05</td>\n",
       "      <td>33040.001252</td>\n",
       "      <td>0.006673</td>\n",
       "      <td>0.134805</td>\n",
       "      <td>0.005510</td>\n",
       "      <td>0.782103</td>\n",
       "      <td>0.015520</td>\n",
       "      <td>0.355345</td>\n",
       "      <td>1.798326</td>\n",
       "      <td>...</td>\n",
       "      <td>64.745380</td>\n",
       "      <td>106.040437</td>\n",
       "      <td>0.410781</td>\n",
       "      <td>0.109259</td>\n",
       "      <td>0.481309</td>\n",
       "      <td>0.042133</td>\n",
       "      <td>0.380593</td>\n",
       "      <td>0.380919</td>\n",
       "      <td>0.230590</td>\n",
       "      <td>0.230140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.500000e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>0.410000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.200000e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.032000e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.040000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>58329.000000</td>\n",
       "      <td>6.933756e+08</td>\n",
       "      <td>5155468.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>884.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            duration     src_bytes       dst_bytes           land  \\\n",
       "count  494021.000000  4.940210e+05   494021.000000  494021.000000   \n",
       "mean       47.979302  3.025610e+03      868.532425       0.000045   \n",
       "std       707.746472  9.882181e+05    33040.001252       0.006673   \n",
       "min         0.000000  0.000000e+00        0.000000       0.000000   \n",
       "25%         0.000000  4.500000e+01        0.000000       0.000000   \n",
       "50%         0.000000  5.200000e+02        0.000000       0.000000   \n",
       "75%         0.000000  1.032000e+03        0.000000       0.000000   \n",
       "max     58329.000000  6.933756e+08  5155468.000000       1.000000   \n",
       "\n",
       "       wrong_fragment         urgent            hot  num_failed_logins  \\\n",
       "count   494021.000000  494021.000000  494021.000000      494021.000000   \n",
       "mean         0.006433       0.000014       0.034519           0.000152   \n",
       "std          0.134805       0.005510       0.782103           0.015520   \n",
       "min          0.000000       0.000000       0.000000           0.000000   \n",
       "25%          0.000000       0.000000       0.000000           0.000000   \n",
       "50%          0.000000       0.000000       0.000000           0.000000   \n",
       "75%          0.000000       0.000000       0.000000           0.000000   \n",
       "max          3.000000       3.000000      30.000000           5.000000   \n",
       "\n",
       "           logged_in  num_compromised            ...             \\\n",
       "count  494021.000000    494021.000000            ...              \n",
       "mean        0.148247         0.010212            ...              \n",
       "std         0.355345         1.798326            ...              \n",
       "min         0.000000         0.000000            ...              \n",
       "25%         0.000000         0.000000            ...              \n",
       "50%         0.000000         0.000000            ...              \n",
       "75%         0.000000         0.000000            ...              \n",
       "max         1.000000       884.000000            ...              \n",
       "\n",
       "       dst_host_count  dst_host_srv_count  dst_host_same_srv_rate  \\\n",
       "count   494021.000000       494021.000000           494021.000000   \n",
       "mean       232.470778          188.665670                0.753780   \n",
       "std         64.745380          106.040437                0.410781   \n",
       "min          0.000000            0.000000                0.000000   \n",
       "25%        255.000000           46.000000                0.410000   \n",
       "50%        255.000000          255.000000                1.000000   \n",
       "75%        255.000000          255.000000                1.000000   \n",
       "max        255.000000          255.000000                1.000000   \n",
       "\n",
       "       dst_host_diff_srv_rate  dst_host_same_src_port_rate  \\\n",
       "count           494021.000000                494021.000000   \n",
       "mean                 0.030906                     0.601935   \n",
       "std                  0.109259                     0.481309   \n",
       "min                  0.000000                     0.000000   \n",
       "25%                  0.000000                     0.000000   \n",
       "50%                  0.000000                     1.000000   \n",
       "75%                  0.040000                     1.000000   \n",
       "max                  1.000000                     1.000000   \n",
       "\n",
       "       dst_host_srv_diff_host_rate  dst_host_serror_rate  \\\n",
       "count                494021.000000         494021.000000   \n",
       "mean                      0.006684              0.176754   \n",
       "std                       0.042133              0.380593   \n",
       "min                       0.000000              0.000000   \n",
       "25%                       0.000000              0.000000   \n",
       "50%                       0.000000              0.000000   \n",
       "75%                       0.000000              0.000000   \n",
       "max                       1.000000              1.000000   \n",
       "\n",
       "       dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "count             494021.000000         494021.000000   \n",
       "mean                   0.176443              0.058118   \n",
       "std                    0.380919              0.230590   \n",
       "min                    0.000000              0.000000   \n",
       "25%                    0.000000              0.000000   \n",
       "50%                    0.000000              0.000000   \n",
       "75%                    0.000000              0.000000   \n",
       "max                    1.000000              1.000000   \n",
       "\n",
       "       dst_host_srv_rerror_rate  \n",
       "count             494021.000000  \n",
       "mean                   0.057412  \n",
       "std                    0.230140  \n",
       "min                    0.000000  \n",
       "25%                    0.000000  \n",
       "50%                    0.000000  \n",
       "75%                    0.000000  \n",
       "max                    1.000000  \n",
       "\n",
       "[8 rows x 38 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_features = [\n",
    "    \"duration\",\"src_bytes\",\n",
    "    \"dst_bytes\",\"land\",\"wrong_fragment\",\"urgent\",\"hot\",\"num_failed_logins\",\n",
    "    \"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\n",
    "    \"num_file_creations\",\"num_shells\",\"num_access_files\",\"num_outbound_cmds\",\n",
    "    \"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\n",
    "    \"srv_serror_rate\",\"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\n",
    "    \"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\",\n",
    "    \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\n",
    "    \"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\"dst_host_srv_serror_rate\",\n",
    "    \"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\"\n",
    "]\n",
    "features = kdd_data_10percent[num_features].astype(float)\n",
    "features.describe()\n",
    "#Use only features which have discrete values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    396743\n",
       "0     97278\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "labels = kdd_data_10percent['label'].copy()\n",
    "labels[labels!='normal.'] = 1\n",
    "labels[labels!=1] = 0\n",
    "\n",
    "labels.value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>494021.000000</td>\n",
       "      <td>4.940210e+05</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>47.979302</td>\n",
       "      <td>3.025610e+03</td>\n",
       "      <td>868.532425</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>0.006433</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.034519</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>0.148247</td>\n",
       "      <td>0.010212</td>\n",
       "      <td>...</td>\n",
       "      <td>232.470778</td>\n",
       "      <td>188.665670</td>\n",
       "      <td>0.753780</td>\n",
       "      <td>0.030906</td>\n",
       "      <td>0.601935</td>\n",
       "      <td>0.006684</td>\n",
       "      <td>0.176754</td>\n",
       "      <td>0.176443</td>\n",
       "      <td>0.058118</td>\n",
       "      <td>0.057412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>707.746472</td>\n",
       "      <td>9.882181e+05</td>\n",
       "      <td>33040.001252</td>\n",
       "      <td>0.006673</td>\n",
       "      <td>0.134805</td>\n",
       "      <td>0.005510</td>\n",
       "      <td>0.782103</td>\n",
       "      <td>0.015520</td>\n",
       "      <td>0.355345</td>\n",
       "      <td>1.798326</td>\n",
       "      <td>...</td>\n",
       "      <td>64.745380</td>\n",
       "      <td>106.040437</td>\n",
       "      <td>0.410781</td>\n",
       "      <td>0.109259</td>\n",
       "      <td>0.481309</td>\n",
       "      <td>0.042133</td>\n",
       "      <td>0.380593</td>\n",
       "      <td>0.380919</td>\n",
       "      <td>0.230590</td>\n",
       "      <td>0.230140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.500000e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>0.410000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.200000e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.032000e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.040000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>58329.000000</td>\n",
       "      <td>6.933756e+08</td>\n",
       "      <td>5155468.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>884.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            duration     src_bytes       dst_bytes           land  \\\n",
       "count  494021.000000  4.940210e+05   494021.000000  494021.000000   \n",
       "mean       47.979302  3.025610e+03      868.532425       0.000045   \n",
       "std       707.746472  9.882181e+05    33040.001252       0.006673   \n",
       "min         0.000000  0.000000e+00        0.000000       0.000000   \n",
       "25%         0.000000  4.500000e+01        0.000000       0.000000   \n",
       "50%         0.000000  5.200000e+02        0.000000       0.000000   \n",
       "75%         0.000000  1.032000e+03        0.000000       0.000000   \n",
       "max     58329.000000  6.933756e+08  5155468.000000       1.000000   \n",
       "\n",
       "       wrong_fragment         urgent            hot  num_failed_logins  \\\n",
       "count   494021.000000  494021.000000  494021.000000      494021.000000   \n",
       "mean         0.006433       0.000014       0.034519           0.000152   \n",
       "std          0.134805       0.005510       0.782103           0.015520   \n",
       "min          0.000000       0.000000       0.000000           0.000000   \n",
       "25%          0.000000       0.000000       0.000000           0.000000   \n",
       "50%          0.000000       0.000000       0.000000           0.000000   \n",
       "75%          0.000000       0.000000       0.000000           0.000000   \n",
       "max          3.000000       3.000000      30.000000           5.000000   \n",
       "\n",
       "           logged_in  num_compromised            ...             \\\n",
       "count  494021.000000    494021.000000            ...              \n",
       "mean        0.148247         0.010212            ...              \n",
       "std         0.355345         1.798326            ...              \n",
       "min         0.000000         0.000000            ...              \n",
       "25%         0.000000         0.000000            ...              \n",
       "50%         0.000000         0.000000            ...              \n",
       "75%         0.000000         0.000000            ...              \n",
       "max         1.000000       884.000000            ...              \n",
       "\n",
       "       dst_host_count  dst_host_srv_count  dst_host_same_srv_rate  \\\n",
       "count   494021.000000       494021.000000           494021.000000   \n",
       "mean       232.470778          188.665670                0.753780   \n",
       "std         64.745380          106.040437                0.410781   \n",
       "min          0.000000            0.000000                0.000000   \n",
       "25%        255.000000           46.000000                0.410000   \n",
       "50%        255.000000          255.000000                1.000000   \n",
       "75%        255.000000          255.000000                1.000000   \n",
       "max        255.000000          255.000000                1.000000   \n",
       "\n",
       "       dst_host_diff_srv_rate  dst_host_same_src_port_rate  \\\n",
       "count           494021.000000                494021.000000   \n",
       "mean                 0.030906                     0.601935   \n",
       "std                  0.109259                     0.481309   \n",
       "min                  0.000000                     0.000000   \n",
       "25%                  0.000000                     0.000000   \n",
       "50%                  0.000000                     1.000000   \n",
       "75%                  0.040000                     1.000000   \n",
       "max                  1.000000                     1.000000   \n",
       "\n",
       "       dst_host_srv_diff_host_rate  dst_host_serror_rate  \\\n",
       "count                494021.000000         494021.000000   \n",
       "mean                      0.006684              0.176754   \n",
       "std                       0.042133              0.380593   \n",
       "min                       0.000000              0.000000   \n",
       "25%                       0.000000              0.000000   \n",
       "50%                       0.000000              0.000000   \n",
       "75%                       0.000000              0.000000   \n",
       "max                       1.000000              1.000000   \n",
       "\n",
       "       dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "count             494021.000000         494021.000000   \n",
       "mean                   0.176443              0.058118   \n",
       "std                    0.380919              0.230590   \n",
       "min                    0.000000              0.000000   \n",
       "25%                    0.000000              0.000000   \n",
       "50%                    0.000000              0.000000   \n",
       "75%                    0.000000              0.000000   \n",
       "max                    1.000000              1.000000   \n",
       "\n",
       "       dst_host_srv_rerror_rate  \n",
       "count             494021.000000  \n",
       "mean                   0.057412  \n",
       "std                    0.230140  \n",
       "min                    0.000000  \n",
       "25%                    0.000000  \n",
       "50%                    0.000000  \n",
       "75%                    0.000000  \n",
       "max                    1.000000  \n",
       "\n",
       "[8 rows x 38 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "features.apply(lambda x: MinMaxScaler().fit_transform(x))\n",
    "features.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier trained in 856.191 seconds\n"
     ]
    }
   ],
   "source": [
    "clf = KNeighborsClassifier(n_neighbors = 5, algorithm = 'ball_tree', leaf_size=500)\n",
    "t0 = time()\n",
    "clf.fit(features,labels)\n",
    "tt = time()-t0\n",
    "print \"Classifier trained in {} seconds\".format(round(tt,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "smurf.              2807886\n",
       "neptune.            1072017\n",
       "normal.              972781\n",
       "satan.                15892\n",
       "ipsweep.              12481\n",
       "portsweep.            10413\n",
       "nmap.                  2316\n",
       "back.                  2203\n",
       "warezclient.           1020\n",
       "teardrop.               979\n",
       "pod.                    264\n",
       "guess_passwd.            53\n",
       "buffer_overflow.         30\n",
       "land.                    21\n",
       "warezmaster.             20\n",
       "imap.                    12\n",
       "rootkit.                 10\n",
       "loadmodule.               9\n",
       "ftp_write.                8\n",
       "multihop.                 7\n",
       "phf.                      4\n",
       "perl.                     3\n",
       "spy.                      2\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kdd_data_corrected = pandas.read_csv(\"kddcup.data.corrected\", header=None, names = col_names)\n",
    "kdd_data_corrected['label'].value_counts()\n",
    "#Testing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Manish\\Anaconda2\\lib\\site-packages\\ipykernel\\__main__.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "C:\\Users\\Manish\\Anaconda2\\lib\\site-packages\\ipykernel\\__main__.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1    3925650\n",
       "0     972781\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kdd_data_corrected['label'][kdd_data_corrected['label']!='normal.'] = 1\n",
    "kdd_data_corrected['label'][kdd_data_corrected['label']!=1] = 0\n",
    "kdd_data_corrected['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.558012e-07</td>\n",
       "      <td>3.441080e-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.173944e-07</td>\n",
       "      <td>3.456654e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.710190e-07</td>\n",
       "      <td>9.374494e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.688450e-07</td>\n",
       "      <td>1.551219e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011765</td>\n",
       "      <td>0.011765</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>3.710101e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015686</td>\n",
       "      <td>0.015686</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>1.724683e-07</td>\n",
       "      <td>9.786727e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>1.702943e-07</td>\n",
       "      <td>1.020659e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>1.695697e-07</td>\n",
       "      <td>1.041271e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.027451</td>\n",
       "      <td>0.027451</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>9.885969e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>1.311629e-07</td>\n",
       "      <td>4.160504e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035294</td>\n",
       "      <td>0.035294</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>1.333368e-07</td>\n",
       "      <td>9.466101e-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>1.340615e-07</td>\n",
       "      <td>6.885825e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043137</td>\n",
       "      <td>0.043137</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>9.885969e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047059</td>\n",
       "      <td>0.047059</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>1.311629e-07</td>\n",
       "      <td>4.160504e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.050980</td>\n",
       "      <td>0.050980</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>1.710190e-07</td>\n",
       "      <td>9.374494e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.054902</td>\n",
       "      <td>0.054902</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>1.688450e-07</td>\n",
       "      <td>1.551219e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>1.724683e-07</td>\n",
       "      <td>9.786727e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.062745</td>\n",
       "      <td>0.062745</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>1.702943e-07</td>\n",
       "      <td>1.020659e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>1.695697e-07</td>\n",
       "      <td>1.041271e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.070588</td>\n",
       "      <td>0.070588</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>3.710101e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.074510</td>\n",
       "      <td>0.074510</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>1.340615e-07</td>\n",
       "      <td>6.885825e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.078431</td>\n",
       "      <td>0.078431</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0</td>\n",
       "      <td>1.333368e-07</td>\n",
       "      <td>9.466101e-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.082353</td>\n",
       "      <td>0.082353</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>1.311629e-07</td>\n",
       "      <td>4.160504e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086275</td>\n",
       "      <td>0.086275</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>9.885969e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.090196</td>\n",
       "      <td>0.090196</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0</td>\n",
       "      <td>1.710190e-07</td>\n",
       "      <td>9.374494e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.094118</td>\n",
       "      <td>0.094118</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0</td>\n",
       "      <td>1.688450e-07</td>\n",
       "      <td>1.551219e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>3.710101e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0</td>\n",
       "      <td>1.724683e-07</td>\n",
       "      <td>9.786727e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.105882</td>\n",
       "      <td>0.105882</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0</td>\n",
       "      <td>1.695697e-07</td>\n",
       "      <td>1.041271e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.109804</td>\n",
       "      <td>0.109804</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0</td>\n",
       "      <td>1.702943e-07</td>\n",
       "      <td>1.020659e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898401</th>\n",
       "      <td>0</td>\n",
       "      <td>2.050778e-07</td>\n",
       "      <td>3.557422e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898402</th>\n",
       "      <td>0</td>\n",
       "      <td>2.094258e-07</td>\n",
       "      <td>8.366812e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035294</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898403</th>\n",
       "      <td>0</td>\n",
       "      <td>2.094258e-07</td>\n",
       "      <td>1.421442e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898404</th>\n",
       "      <td>0</td>\n",
       "      <td>1.471053e-07</td>\n",
       "      <td>1.847417e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043137</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898405</th>\n",
       "      <td>0</td>\n",
       "      <td>1.420327e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047059</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898406</th>\n",
       "      <td>0</td>\n",
       "      <td>1.036259e-07</td>\n",
       "      <td>1.333117e-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.050980</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898407</th>\n",
       "      <td>0</td>\n",
       "      <td>1.463806e-07</td>\n",
       "      <td>3.066559e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.054902</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898408</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.786345e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898409</th>\n",
       "      <td>0</td>\n",
       "      <td>1.471053e-07</td>\n",
       "      <td>9.160743e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.062745</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898410</th>\n",
       "      <td>0</td>\n",
       "      <td>1.434820e-07</td>\n",
       "      <td>1.655804e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898411</th>\n",
       "      <td>0</td>\n",
       "      <td>1.427574e-07</td>\n",
       "      <td>3.557422e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.070588</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898412</th>\n",
       "      <td>0</td>\n",
       "      <td>1.471053e-07</td>\n",
       "      <td>1.421442e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.074510</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898413</th>\n",
       "      <td>0</td>\n",
       "      <td>1.471053e-07</td>\n",
       "      <td>8.366812e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.078431</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898414</th>\n",
       "      <td>0</td>\n",
       "      <td>2.058025e-07</td>\n",
       "      <td>1.745122e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.082353</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898415</th>\n",
       "      <td>0</td>\n",
       "      <td>2.101504e-07</td>\n",
       "      <td>3.066559e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086275</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898416</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>1.786345e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.090196</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898417</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>1.847417e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.094118</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898418</th>\n",
       "      <td>0</td>\n",
       "      <td>1.673957e-07</td>\n",
       "      <td>1.333117e-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898419</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>9.160743e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898420</th>\n",
       "      <td>0</td>\n",
       "      <td>2.065271e-07</td>\n",
       "      <td>3.557422e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.105882</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898421</th>\n",
       "      <td>0</td>\n",
       "      <td>2.072518e-07</td>\n",
       "      <td>1.655804e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.109804</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898422</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>8.366812e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898423</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>1.421442e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898424</th>\n",
       "      <td>0</td>\n",
       "      <td>1.152204e-07</td>\n",
       "      <td>1.206775e-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898425</th>\n",
       "      <td>0</td>\n",
       "      <td>1.586998e-07</td>\n",
       "      <td>1.862684e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898426</th>\n",
       "      <td>0</td>\n",
       "      <td>1.536272e-07</td>\n",
       "      <td>1.746648e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011765</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898427</th>\n",
       "      <td>0</td>\n",
       "      <td>1.586998e-07</td>\n",
       "      <td>1.801613e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015686</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898428</th>\n",
       "      <td>0</td>\n",
       "      <td>1.579751e-07</td>\n",
       "      <td>2.755857e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898429</th>\n",
       "      <td>0</td>\n",
       "      <td>1.586998e-07</td>\n",
       "      <td>9.420297e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898430</th>\n",
       "      <td>0</td>\n",
       "      <td>1.586998e-07</td>\n",
       "      <td>8.382080e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.027451</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4898431 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         duration     src_bytes     dst_bytes  land  wrong_fragment  urgent  \\\n",
       "0               0  1.558012e-07  3.441080e-05     0               0       0   \n",
       "1               0  1.173944e-07  3.456654e-06     0               0       0   \n",
       "2               0  1.710190e-07  9.374494e-07     0               0       0   \n",
       "3               0  1.688450e-07  1.551219e-06     0               0       0   \n",
       "4               0  1.731929e-07  3.710101e-07     0               0       0   \n",
       "5               0  1.724683e-07  9.786727e-07     0               0       0   \n",
       "6               0  1.702943e-07  1.020659e-06     0               0       0   \n",
       "7               0  1.695697e-07  1.041271e-06     0               0       0   \n",
       "8               0  1.731929e-07  9.885969e-07     0               0       0   \n",
       "9               0  1.311629e-07  4.160504e-06     0               0       0   \n",
       "10              0  1.333368e-07  9.466101e-08     0               0       0   \n",
       "11              0  1.340615e-07  6.885825e-06     0               0       0   \n",
       "12              0  1.731929e-07  9.885969e-07     0               0       0   \n",
       "13              0  1.311629e-07  4.160504e-06     0               0       0   \n",
       "14              0  1.710190e-07  9.374494e-07     0               0       0   \n",
       "15              0  1.688450e-07  1.551219e-06     0               0       0   \n",
       "16              0  1.724683e-07  9.786727e-07     0               0       0   \n",
       "17              0  1.702943e-07  1.020659e-06     0               0       0   \n",
       "18              0  1.695697e-07  1.041271e-06     0               0       0   \n",
       "19              0  1.731929e-07  3.710101e-07     0               0       0   \n",
       "20              0  1.340615e-07  6.885825e-06     0               0       0   \n",
       "21              0  1.333368e-07  9.466101e-08     0               0       0   \n",
       "22              0  1.311629e-07  4.160504e-06     0               0       0   \n",
       "23              0  1.731929e-07  9.885969e-07     0               0       0   \n",
       "24              0  1.710190e-07  9.374494e-07     0               0       0   \n",
       "25              0  1.688450e-07  1.551219e-06     0               0       0   \n",
       "26              0  1.731929e-07  3.710101e-07     0               0       0   \n",
       "27              0  1.724683e-07  9.786727e-07     0               0       0   \n",
       "28              0  1.695697e-07  1.041271e-06     0               0       0   \n",
       "29              0  1.702943e-07  1.020659e-06     0               0       0   \n",
       "...           ...           ...           ...   ...             ...     ...   \n",
       "4898401         0  2.050778e-07  3.557422e-07     0               0       0   \n",
       "4898402         0  2.094258e-07  8.366812e-07     0               0       0   \n",
       "4898403         0  2.094258e-07  1.421442e-06     0               0       0   \n",
       "4898404         0  1.471053e-07  1.847417e-07     0               0       0   \n",
       "4898405         0  1.420327e-07  0.000000e+00     0               0       0   \n",
       "4898406         0  1.036259e-07  1.333117e-05     0               0       0   \n",
       "4898407         0  1.463806e-07  3.066559e-06     0               0       0   \n",
       "4898408         0  0.000000e+00  1.786345e-07     0               0       0   \n",
       "4898409         0  1.471053e-07  9.160743e-07     0               0       0   \n",
       "4898410         0  1.434820e-07  1.655804e-06     0               0       0   \n",
       "4898411         0  1.427574e-07  3.557422e-07     0               0       0   \n",
       "4898412         0  1.471053e-07  1.421442e-06     0               0       0   \n",
       "4898413         0  1.471053e-07  8.366812e-07     0               0       0   \n",
       "4898414         0  2.058025e-07  1.745122e-06     0               0       0   \n",
       "4898415         0  2.101504e-07  3.066559e-06     0               0       0   \n",
       "4898416         0  2.108751e-07  1.786345e-07     0               0       0   \n",
       "4898417         0  2.108751e-07  1.847417e-07     0               0       0   \n",
       "4898418         0  1.673957e-07  1.333117e-05     0               0       0   \n",
       "4898419         0  2.108751e-07  9.160743e-07     0               0       0   \n",
       "4898420         0  2.065271e-07  3.557422e-07     0               0       0   \n",
       "4898421         0  2.072518e-07  1.655804e-06     0               0       0   \n",
       "4898422         0  2.108751e-07  8.366812e-07     0               0       0   \n",
       "4898423         0  2.108751e-07  1.421442e-06     0               0       0   \n",
       "4898424         0  1.152204e-07  1.206775e-05     0               0       0   \n",
       "4898425         0  1.586998e-07  1.862684e-07     0               0       0   \n",
       "4898426         0  1.536272e-07  1.746648e-06     0               0       0   \n",
       "4898427         0  1.586998e-07  1.801613e-07     0               0       0   \n",
       "4898428         0  1.579751e-07  2.755857e-06     0               0       0   \n",
       "4898429         0  1.586998e-07  9.420297e-07     0               0       0   \n",
       "4898430         0  1.586998e-07  8.382080e-07     0               0       0   \n",
       "\n",
       "         hot  num_failed_logins  logged_in  num_compromised  \\\n",
       "0          0                  0          1                0   \n",
       "1          0                  0          1                0   \n",
       "2          0                  0          1                0   \n",
       "3          0                  0          1                0   \n",
       "4          0                  0          1                0   \n",
       "5          0                  0          1                0   \n",
       "6          0                  0          1                0   \n",
       "7          0                  0          1                0   \n",
       "8          0                  0          1                0   \n",
       "9          0                  0          1                0   \n",
       "10         0                  0          1                0   \n",
       "11         0                  0          1                0   \n",
       "12         0                  0          1                0   \n",
       "13         0                  0          1                0   \n",
       "14         0                  0          1                0   \n",
       "15         0                  0          1                0   \n",
       "16         0                  0          1                0   \n",
       "17         0                  0          1                0   \n",
       "18         0                  0          1                0   \n",
       "19         0                  0          1                0   \n",
       "20         0                  0          1                0   \n",
       "21         0                  0          1                0   \n",
       "22         0                  0          1                0   \n",
       "23         0                  0          1                0   \n",
       "24         0                  0          1                0   \n",
       "25         0                  0          1                0   \n",
       "26         0                  0          1                0   \n",
       "27         0                  0          1                0   \n",
       "28         0                  0          1                0   \n",
       "29         0                  0          1                0   \n",
       "...      ...                ...        ...              ...   \n",
       "4898401    0                  0          1                0   \n",
       "4898402    0                  0          1                0   \n",
       "4898403    0                  0          1                0   \n",
       "4898404    0                  0          1                0   \n",
       "4898405    0                  0          1                0   \n",
       "4898406    0                  0          1                0   \n",
       "4898407    0                  0          1                0   \n",
       "4898408    0                  0          0                0   \n",
       "4898409    0                  0          1                0   \n",
       "4898410    0                  0          1                0   \n",
       "4898411    0                  0          1                0   \n",
       "4898412    0                  0          1                0   \n",
       "4898413    0                  0          1                0   \n",
       "4898414    0                  0          1                0   \n",
       "4898415    0                  0          1                0   \n",
       "4898416    0                  0          1                0   \n",
       "4898417    0                  0          1                0   \n",
       "4898418    0                  0          1                0   \n",
       "4898419    0                  0          1                0   \n",
       "4898420    0                  0          1                0   \n",
       "4898421    0                  0          1                0   \n",
       "4898422    0                  0          1                0   \n",
       "4898423    0                  0          1                0   \n",
       "4898424    0                  0          1                0   \n",
       "4898425    0                  0          1                0   \n",
       "4898426    0                  0          1                0   \n",
       "4898427    0                  0          1                0   \n",
       "4898428    0                  0          1                0   \n",
       "4898429    0                  0          1                0   \n",
       "4898430    0                  0          1                0   \n",
       "\n",
       "                   ...             dst_host_count  dst_host_srv_count  \\\n",
       "0                  ...                   0.000000            0.000000   \n",
       "1                  ...                   0.003922            0.003922   \n",
       "2                  ...                   0.007843            0.007843   \n",
       "3                  ...                   0.011765            0.011765   \n",
       "4                  ...                   0.015686            0.015686   \n",
       "5                  ...                   0.019608            0.019608   \n",
       "6                  ...                   0.023529            0.023529   \n",
       "7                  ...                   0.027451            0.027451   \n",
       "8                  ...                   0.031373            0.031373   \n",
       "9                  ...                   0.035294            0.035294   \n",
       "10                 ...                   0.039216            0.039216   \n",
       "11                 ...                   0.043137            0.043137   \n",
       "12                 ...                   0.047059            0.047059   \n",
       "13                 ...                   0.050980            0.050980   \n",
       "14                 ...                   0.054902            0.054902   \n",
       "15                 ...                   0.058824            0.058824   \n",
       "16                 ...                   0.062745            0.062745   \n",
       "17                 ...                   0.066667            0.066667   \n",
       "18                 ...                   0.070588            0.070588   \n",
       "19                 ...                   0.074510            0.074510   \n",
       "20                 ...                   0.078431            0.078431   \n",
       "21                 ...                   0.082353            0.082353   \n",
       "22                 ...                   0.086275            0.086275   \n",
       "23                 ...                   0.090196            0.090196   \n",
       "24                 ...                   0.094118            0.094118   \n",
       "25                 ...                   0.098039            0.098039   \n",
       "26                 ...                   0.101961            0.101961   \n",
       "27                 ...                   0.105882            0.105882   \n",
       "28                 ...                   0.109804            0.109804   \n",
       "29                 ...                   0.113725            0.113725   \n",
       "...                ...                        ...                 ...   \n",
       "4898401            ...                   0.031373            1.000000   \n",
       "4898402            ...                   0.035294            1.000000   \n",
       "4898403            ...                   0.039216            1.000000   \n",
       "4898404            ...                   0.043137            1.000000   \n",
       "4898405            ...                   0.047059            1.000000   \n",
       "4898406            ...                   0.050980            1.000000   \n",
       "4898407            ...                   0.054902            1.000000   \n",
       "4898408            ...                   0.058824            1.000000   \n",
       "4898409            ...                   0.062745            1.000000   \n",
       "4898410            ...                   0.066667            1.000000   \n",
       "4898411            ...                   0.070588            1.000000   \n",
       "4898412            ...                   0.074510            1.000000   \n",
       "4898413            ...                   0.078431            1.000000   \n",
       "4898414            ...                   0.082353            1.000000   \n",
       "4898415            ...                   0.086275            1.000000   \n",
       "4898416            ...                   0.090196            1.000000   \n",
       "4898417            ...                   0.094118            1.000000   \n",
       "4898418            ...                   0.098039            1.000000   \n",
       "4898419            ...                   0.101961            1.000000   \n",
       "4898420            ...                   0.105882            1.000000   \n",
       "4898421            ...                   0.109804            1.000000   \n",
       "4898422            ...                   0.113725            1.000000   \n",
       "4898423            ...                   0.117647            1.000000   \n",
       "4898424            ...                   0.003922            1.000000   \n",
       "4898425            ...                   0.007843            1.000000   \n",
       "4898426            ...                   0.011765            1.000000   \n",
       "4898427            ...                   0.015686            1.000000   \n",
       "4898428            ...                   0.019608            1.000000   \n",
       "4898429            ...                   0.023529            1.000000   \n",
       "4898430            ...                   0.027451            1.000000   \n",
       "\n",
       "         dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "0                             0                       0   \n",
       "1                             1                       0   \n",
       "2                             1                       0   \n",
       "3                             1                       0   \n",
       "4                             1                       0   \n",
       "5                             1                       0   \n",
       "6                             1                       0   \n",
       "7                             1                       0   \n",
       "8                             1                       0   \n",
       "9                             1                       0   \n",
       "10                            1                       0   \n",
       "11                            1                       0   \n",
       "12                            1                       0   \n",
       "13                            1                       0   \n",
       "14                            1                       0   \n",
       "15                            1                       0   \n",
       "16                            1                       0   \n",
       "17                            1                       0   \n",
       "18                            1                       0   \n",
       "19                            1                       0   \n",
       "20                            1                       0   \n",
       "21                            1                       0   \n",
       "22                            1                       0   \n",
       "23                            1                       0   \n",
       "24                            1                       0   \n",
       "25                            1                       0   \n",
       "26                            1                       0   \n",
       "27                            1                       0   \n",
       "28                            1                       0   \n",
       "29                            1                       0   \n",
       "...                         ...                     ...   \n",
       "4898401                       1                       0   \n",
       "4898402                       1                       0   \n",
       "4898403                       1                       0   \n",
       "4898404                       1                       0   \n",
       "4898405                       1                       0   \n",
       "4898406                       1                       0   \n",
       "4898407                       1                       0   \n",
       "4898408                       1                       0   \n",
       "4898409                       1                       0   \n",
       "4898410                       1                       0   \n",
       "4898411                       1                       0   \n",
       "4898412                       1                       0   \n",
       "4898413                       1                       0   \n",
       "4898414                       1                       0   \n",
       "4898415                       1                       0   \n",
       "4898416                       1                       0   \n",
       "4898417                       1                       0   \n",
       "4898418                       1                       0   \n",
       "4898419                       1                       0   \n",
       "4898420                       1                       0   \n",
       "4898421                       1                       0   \n",
       "4898422                       1                       0   \n",
       "4898423                       1                       0   \n",
       "4898424                       1                       0   \n",
       "4898425                       1                       0   \n",
       "4898426                       1                       0   \n",
       "4898427                       1                       0   \n",
       "4898428                       1                       0   \n",
       "4898429                       1                       0   \n",
       "4898430                       1                       0   \n",
       "\n",
       "         dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "0                               0.00                         0.00   \n",
       "1                               1.00                         0.00   \n",
       "2                               0.50                         0.00   \n",
       "3                               0.33                         0.00   \n",
       "4                               0.25                         0.00   \n",
       "5                               0.20                         0.00   \n",
       "6                               0.17                         0.00   \n",
       "7                               0.14                         0.00   \n",
       "8                               0.12                         0.00   \n",
       "9                               0.11                         0.00   \n",
       "10                              0.10                         0.00   \n",
       "11                              0.09                         0.00   \n",
       "12                              0.08                         0.00   \n",
       "13                              0.08                         0.00   \n",
       "14                              0.07                         0.00   \n",
       "15                              0.07                         0.00   \n",
       "16                              0.06                         0.00   \n",
       "17                              0.06                         0.00   \n",
       "18                              0.06                         0.00   \n",
       "19                              0.05                         0.00   \n",
       "20                              0.05                         0.00   \n",
       "21                              0.05                         0.00   \n",
       "22                              0.05                         0.00   \n",
       "23                              0.04                         0.00   \n",
       "24                              0.04                         0.00   \n",
       "25                              0.04                         0.00   \n",
       "26                              0.04                         0.00   \n",
       "27                              0.04                         0.00   \n",
       "28                              0.04                         0.00   \n",
       "29                              0.03                         0.00   \n",
       "...                              ...                          ...   \n",
       "4898401                         0.12                         0.05   \n",
       "4898402                         0.11                         0.05   \n",
       "4898403                         0.10                         0.05   \n",
       "4898404                         0.09                         0.05   \n",
       "4898405                         0.08                         0.05   \n",
       "4898406                         0.08                         0.05   \n",
       "4898407                         0.07                         0.05   \n",
       "4898408                         0.07                         0.05   \n",
       "4898409                         0.06                         0.05   \n",
       "4898410                         0.06                         0.05   \n",
       "4898411                         0.06                         0.05   \n",
       "4898412                         0.05                         0.05   \n",
       "4898413                         0.05                         0.05   \n",
       "4898414                         0.05                         0.05   \n",
       "4898415                         0.05                         0.05   \n",
       "4898416                         0.04                         0.05   \n",
       "4898417                         0.04                         0.05   \n",
       "4898418                         0.04                         0.05   \n",
       "4898419                         0.04                         0.05   \n",
       "4898420                         0.04                         0.05   \n",
       "4898421                         0.04                         0.05   \n",
       "4898422                         0.03                         0.05   \n",
       "4898423                         0.03                         0.05   \n",
       "4898424                         1.00                         0.05   \n",
       "4898425                         0.50                         0.05   \n",
       "4898426                         0.33                         0.05   \n",
       "4898427                         0.25                         0.05   \n",
       "4898428                         0.20                         0.05   \n",
       "4898429                         0.17                         0.05   \n",
       "4898430                         0.14                         0.05   \n",
       "\n",
       "         dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "0                        0.00                      0.00                     0   \n",
       "1                        0.00                      0.00                     0   \n",
       "2                        0.00                      0.00                     0   \n",
       "3                        0.00                      0.00                     0   \n",
       "4                        0.00                      0.00                     0   \n",
       "5                        0.00                      0.00                     0   \n",
       "6                        0.00                      0.00                     0   \n",
       "7                        0.00                      0.00                     0   \n",
       "8                        0.00                      0.00                     0   \n",
       "9                        0.00                      0.00                     0   \n",
       "10                       0.00                      0.00                     0   \n",
       "11                       0.00                      0.00                     0   \n",
       "12                       0.00                      0.00                     0   \n",
       "13                       0.00                      0.00                     0   \n",
       "14                       0.00                      0.00                     0   \n",
       "15                       0.00                      0.00                     0   \n",
       "16                       0.00                      0.00                     0   \n",
       "17                       0.00                      0.00                     0   \n",
       "18                       0.00                      0.00                     0   \n",
       "19                       0.00                      0.00                     0   \n",
       "20                       0.00                      0.00                     0   \n",
       "21                       0.00                      0.00                     0   \n",
       "22                       0.00                      0.00                     0   \n",
       "23                       0.00                      0.00                     0   \n",
       "24                       0.00                      0.00                     0   \n",
       "25                       0.00                      0.00                     0   \n",
       "26                       0.00                      0.00                     0   \n",
       "27                       0.00                      0.00                     0   \n",
       "28                       0.00                      0.00                     0   \n",
       "29                       0.00                      0.00                     0   \n",
       "...                       ...                       ...                   ...   \n",
       "4898401                  0.00                      0.01                     0   \n",
       "4898402                  0.00                      0.01                     0   \n",
       "4898403                  0.00                      0.01                     0   \n",
       "4898404                  0.00                      0.01                     0   \n",
       "4898405                  0.08                      0.01                     0   \n",
       "4898406                  0.08                      0.01                     0   \n",
       "4898407                  0.07                      0.01                     0   \n",
       "4898408                  0.07                      0.01                     0   \n",
       "4898409                  0.06                      0.01                     0   \n",
       "4898410                  0.06                      0.01                     0   \n",
       "4898411                  0.06                      0.01                     0   \n",
       "4898412                  0.05                      0.01                     0   \n",
       "4898413                  0.05                      0.01                     0   \n",
       "4898414                  0.05                      0.01                     0   \n",
       "4898415                  0.05                      0.01                     0   \n",
       "4898416                  0.04                      0.01                     0   \n",
       "4898417                  0.04                      0.01                     0   \n",
       "4898418                  0.04                      0.01                     0   \n",
       "4898419                  0.04                      0.01                     0   \n",
       "4898420                  0.04                      0.01                     0   \n",
       "4898421                  0.04                      0.01                     0   \n",
       "4898422                  0.03                      0.01                     0   \n",
       "4898423                  0.03                      0.01                     0   \n",
       "4898424                  0.00                      0.01                     0   \n",
       "4898425                  0.00                      0.01                     0   \n",
       "4898426                  0.00                      0.01                     0   \n",
       "4898427                  0.00                      0.01                     0   \n",
       "4898428                  0.00                      0.01                     0   \n",
       "4898429                  0.00                      0.01                     0   \n",
       "4898430                  0.00                      0.01                     0   \n",
       "\n",
       "         dst_host_srv_rerror_rate  \n",
       "0                               0  \n",
       "1                               0  \n",
       "2                               0  \n",
       "3                               0  \n",
       "4                               0  \n",
       "5                               0  \n",
       "6                               0  \n",
       "7                               0  \n",
       "8                               0  \n",
       "9                               0  \n",
       "10                              0  \n",
       "11                              0  \n",
       "12                              0  \n",
       "13                              0  \n",
       "14                              0  \n",
       "15                              0  \n",
       "16                              0  \n",
       "17                              0  \n",
       "18                              0  \n",
       "19                              0  \n",
       "20                              0  \n",
       "21                              0  \n",
       "22                              0  \n",
       "23                              0  \n",
       "24                              0  \n",
       "25                              0  \n",
       "26                              0  \n",
       "27                              0  \n",
       "28                              0  \n",
       "29                              0  \n",
       "...                           ...  \n",
       "4898401                         0  \n",
       "4898402                         0  \n",
       "4898403                         0  \n",
       "4898404                         0  \n",
       "4898405                         0  \n",
       "4898406                         0  \n",
       "4898407                         0  \n",
       "4898408                         0  \n",
       "4898409                         0  \n",
       "4898410                         0  \n",
       "4898411                         0  \n",
       "4898412                         0  \n",
       "4898413                         0  \n",
       "4898414                         0  \n",
       "4898415                         0  \n",
       "4898416                         0  \n",
       "4898417                         0  \n",
       "4898418                         0  \n",
       "4898419                         0  \n",
       "4898420                         0  \n",
       "4898421                         0  \n",
       "4898422                         0  \n",
       "4898423                         0  \n",
       "4898424                         0  \n",
       "4898425                         0  \n",
       "4898426                         0  \n",
       "4898427                         0  \n",
       "4898428                         0  \n",
       "4898429                         0  \n",
       "4898430                         0  \n",
       "\n",
       "[4898431 rows x 38 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "kdd_data_corrected[num_features] = kdd_data_corrected[num_features].astype(float)\n",
    "kdd_data_corrected[num_features].apply(lambda x: MinMaxScaler().fit_transform(x))\n",
    "#scaling features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### To do Svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier trained in 753.731 seconds\n"
     ]
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "clf1 =svm.SVC()\n",
    "t0 = time()\n",
    "clf1.fit(features[:50000],labels[:50000])\n",
    "tt = time()-t0\n",
    "print \"Classifier trained in {} seconds\".format(round(tt,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['KNNdump.ppkl',\n",
       " 'KNNdump.ppkl_01.npy',\n",
       " 'KNNdump.ppkl_02.npy',\n",
       " 'KNNdump.ppkl_03.npy',\n",
       " 'KNNdump.ppkl_04.npy',\n",
       " 'KNNdump.ppkl_05.npy',\n",
       " 'KNNdump.ppkl_06.npy',\n",
       " 'KNNdump.ppkl_07.npy',\n",
       " 'KNNdump.ppkl_08.npy',\n",
       " 'KNNdump.ppkl_09.npy']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#from sklearn.externals import joblib\n",
    "#joblib.dump(clf,'KNNdump.ppkl')\n",
    "#for saving model parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#clf = joblib.load('filename.pkl') \n",
    "#for loading model parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "Y = np.asarray(labels, dtype=\"|S6\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    37966\n",
       "1    12034\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[:50000].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier trained in 8.067 seconds\n"
     ]
    }
   ],
   "source": [
    "#SGD Classification\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "clf2 = SGDClassifier(loss=\"hinge\", penalty=\"l2\")\n",
    "t0 = time()\n",
    "clf2.fit(features,Y)\n",
    "tt = time()-t0\n",
    "print \"Classifier trained in {} seconds\".format(round(tt,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted in 0.581 seconds\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "pred = clf2.predict(features_test)\n",
    "tt = time() - t0\n",
    "print \"Predicted in {} seconds\".format(round(tt,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R squared is 0.9845.\n"
     ]
    }
   ],
   "source": [
    "a=pred.astype(int)\n",
    "b=labels_test.astype(int)\n",
    "from sklearn.metrics import accuracy_score\n",
    "acc = accuracy_score(b,a)\n",
    "print \"R squared is {}.\".format(round(acc,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.558012e-07</td>\n",
       "      <td>3.441080e-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.173944e-07</td>\n",
       "      <td>3.456654e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.710190e-07</td>\n",
       "      <td>9.374494e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.688450e-07</td>\n",
       "      <td>1.551219e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011765</td>\n",
       "      <td>0.011765</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>3.710101e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015686</td>\n",
       "      <td>0.015686</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>1.724683e-07</td>\n",
       "      <td>9.786727e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>1.702943e-07</td>\n",
       "      <td>1.020659e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>1.695697e-07</td>\n",
       "      <td>1.041271e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.027451</td>\n",
       "      <td>0.027451</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>9.885969e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>1.311629e-07</td>\n",
       "      <td>4.160504e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035294</td>\n",
       "      <td>0.035294</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>1.333368e-07</td>\n",
       "      <td>9.466101e-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>1.340615e-07</td>\n",
       "      <td>6.885825e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043137</td>\n",
       "      <td>0.043137</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>9.885969e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047059</td>\n",
       "      <td>0.047059</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>1.311629e-07</td>\n",
       "      <td>4.160504e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.050980</td>\n",
       "      <td>0.050980</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>1.710190e-07</td>\n",
       "      <td>9.374494e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.054902</td>\n",
       "      <td>0.054902</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>1.688450e-07</td>\n",
       "      <td>1.551219e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>1.724683e-07</td>\n",
       "      <td>9.786727e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.062745</td>\n",
       "      <td>0.062745</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>1.702943e-07</td>\n",
       "      <td>1.020659e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>1.695697e-07</td>\n",
       "      <td>1.041271e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.070588</td>\n",
       "      <td>0.070588</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>3.710101e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.074510</td>\n",
       "      <td>0.074510</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>1.340615e-07</td>\n",
       "      <td>6.885825e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.078431</td>\n",
       "      <td>0.078431</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0</td>\n",
       "      <td>1.333368e-07</td>\n",
       "      <td>9.466101e-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.082353</td>\n",
       "      <td>0.082353</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>1.311629e-07</td>\n",
       "      <td>4.160504e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086275</td>\n",
       "      <td>0.086275</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>9.885969e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.090196</td>\n",
       "      <td>0.090196</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0</td>\n",
       "      <td>1.710190e-07</td>\n",
       "      <td>9.374494e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.094118</td>\n",
       "      <td>0.094118</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0</td>\n",
       "      <td>1.688450e-07</td>\n",
       "      <td>1.551219e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0</td>\n",
       "      <td>1.731929e-07</td>\n",
       "      <td>3.710101e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0</td>\n",
       "      <td>1.724683e-07</td>\n",
       "      <td>9.786727e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.105882</td>\n",
       "      <td>0.105882</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0</td>\n",
       "      <td>1.695697e-07</td>\n",
       "      <td>1.041271e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.109804</td>\n",
       "      <td>0.109804</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0</td>\n",
       "      <td>1.702943e-07</td>\n",
       "      <td>1.020659e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898401</th>\n",
       "      <td>0</td>\n",
       "      <td>2.050778e-07</td>\n",
       "      <td>3.557422e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898402</th>\n",
       "      <td>0</td>\n",
       "      <td>2.094258e-07</td>\n",
       "      <td>8.366812e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035294</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898403</th>\n",
       "      <td>0</td>\n",
       "      <td>2.094258e-07</td>\n",
       "      <td>1.421442e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898404</th>\n",
       "      <td>0</td>\n",
       "      <td>1.471053e-07</td>\n",
       "      <td>1.847417e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043137</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898405</th>\n",
       "      <td>0</td>\n",
       "      <td>1.420327e-07</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047059</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898406</th>\n",
       "      <td>0</td>\n",
       "      <td>1.036259e-07</td>\n",
       "      <td>1.333117e-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.050980</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898407</th>\n",
       "      <td>0</td>\n",
       "      <td>1.463806e-07</td>\n",
       "      <td>3.066559e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.054902</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898408</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.786345e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898409</th>\n",
       "      <td>0</td>\n",
       "      <td>1.471053e-07</td>\n",
       "      <td>9.160743e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.062745</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898410</th>\n",
       "      <td>0</td>\n",
       "      <td>1.434820e-07</td>\n",
       "      <td>1.655804e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898411</th>\n",
       "      <td>0</td>\n",
       "      <td>1.427574e-07</td>\n",
       "      <td>3.557422e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.070588</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898412</th>\n",
       "      <td>0</td>\n",
       "      <td>1.471053e-07</td>\n",
       "      <td>1.421442e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.074510</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898413</th>\n",
       "      <td>0</td>\n",
       "      <td>1.471053e-07</td>\n",
       "      <td>8.366812e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.078431</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898414</th>\n",
       "      <td>0</td>\n",
       "      <td>2.058025e-07</td>\n",
       "      <td>1.745122e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.082353</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898415</th>\n",
       "      <td>0</td>\n",
       "      <td>2.101504e-07</td>\n",
       "      <td>3.066559e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086275</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898416</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>1.786345e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.090196</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898417</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>1.847417e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.094118</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898418</th>\n",
       "      <td>0</td>\n",
       "      <td>1.673957e-07</td>\n",
       "      <td>1.333117e-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898419</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>9.160743e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898420</th>\n",
       "      <td>0</td>\n",
       "      <td>2.065271e-07</td>\n",
       "      <td>3.557422e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.105882</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898421</th>\n",
       "      <td>0</td>\n",
       "      <td>2.072518e-07</td>\n",
       "      <td>1.655804e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.109804</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898422</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>8.366812e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898423</th>\n",
       "      <td>0</td>\n",
       "      <td>2.108751e-07</td>\n",
       "      <td>1.421442e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898424</th>\n",
       "      <td>0</td>\n",
       "      <td>1.152204e-07</td>\n",
       "      <td>1.206775e-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898425</th>\n",
       "      <td>0</td>\n",
       "      <td>1.586998e-07</td>\n",
       "      <td>1.862684e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898426</th>\n",
       "      <td>0</td>\n",
       "      <td>1.536272e-07</td>\n",
       "      <td>1.746648e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011765</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898427</th>\n",
       "      <td>0</td>\n",
       "      <td>1.586998e-07</td>\n",
       "      <td>1.801613e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015686</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898428</th>\n",
       "      <td>0</td>\n",
       "      <td>1.579751e-07</td>\n",
       "      <td>2.755857e-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898429</th>\n",
       "      <td>0</td>\n",
       "      <td>1.586998e-07</td>\n",
       "      <td>9.420297e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4898430</th>\n",
       "      <td>0</td>\n",
       "      <td>1.586998e-07</td>\n",
       "      <td>8.382080e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.027451</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4898431 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         duration     src_bytes     dst_bytes  land  wrong_fragment  urgent  \\\n",
       "0               0  1.558012e-07  3.441080e-05     0               0       0   \n",
       "1               0  1.173944e-07  3.456654e-06     0               0       0   \n",
       "2               0  1.710190e-07  9.374494e-07     0               0       0   \n",
       "3               0  1.688450e-07  1.551219e-06     0               0       0   \n",
       "4               0  1.731929e-07  3.710101e-07     0               0       0   \n",
       "5               0  1.724683e-07  9.786727e-07     0               0       0   \n",
       "6               0  1.702943e-07  1.020659e-06     0               0       0   \n",
       "7               0  1.695697e-07  1.041271e-06     0               0       0   \n",
       "8               0  1.731929e-07  9.885969e-07     0               0       0   \n",
       "9               0  1.311629e-07  4.160504e-06     0               0       0   \n",
       "10              0  1.333368e-07  9.466101e-08     0               0       0   \n",
       "11              0  1.340615e-07  6.885825e-06     0               0       0   \n",
       "12              0  1.731929e-07  9.885969e-07     0               0       0   \n",
       "13              0  1.311629e-07  4.160504e-06     0               0       0   \n",
       "14              0  1.710190e-07  9.374494e-07     0               0       0   \n",
       "15              0  1.688450e-07  1.551219e-06     0               0       0   \n",
       "16              0  1.724683e-07  9.786727e-07     0               0       0   \n",
       "17              0  1.702943e-07  1.020659e-06     0               0       0   \n",
       "18              0  1.695697e-07  1.041271e-06     0               0       0   \n",
       "19              0  1.731929e-07  3.710101e-07     0               0       0   \n",
       "20              0  1.340615e-07  6.885825e-06     0               0       0   \n",
       "21              0  1.333368e-07  9.466101e-08     0               0       0   \n",
       "22              0  1.311629e-07  4.160504e-06     0               0       0   \n",
       "23              0  1.731929e-07  9.885969e-07     0               0       0   \n",
       "24              0  1.710190e-07  9.374494e-07     0               0       0   \n",
       "25              0  1.688450e-07  1.551219e-06     0               0       0   \n",
       "26              0  1.731929e-07  3.710101e-07     0               0       0   \n",
       "27              0  1.724683e-07  9.786727e-07     0               0       0   \n",
       "28              0  1.695697e-07  1.041271e-06     0               0       0   \n",
       "29              0  1.702943e-07  1.020659e-06     0               0       0   \n",
       "...           ...           ...           ...   ...             ...     ...   \n",
       "4898401         0  2.050778e-07  3.557422e-07     0               0       0   \n",
       "4898402         0  2.094258e-07  8.366812e-07     0               0       0   \n",
       "4898403         0  2.094258e-07  1.421442e-06     0               0       0   \n",
       "4898404         0  1.471053e-07  1.847417e-07     0               0       0   \n",
       "4898405         0  1.420327e-07  0.000000e+00     0               0       0   \n",
       "4898406         0  1.036259e-07  1.333117e-05     0               0       0   \n",
       "4898407         0  1.463806e-07  3.066559e-06     0               0       0   \n",
       "4898408         0  0.000000e+00  1.786345e-07     0               0       0   \n",
       "4898409         0  1.471053e-07  9.160743e-07     0               0       0   \n",
       "4898410         0  1.434820e-07  1.655804e-06     0               0       0   \n",
       "4898411         0  1.427574e-07  3.557422e-07     0               0       0   \n",
       "4898412         0  1.471053e-07  1.421442e-06     0               0       0   \n",
       "4898413         0  1.471053e-07  8.366812e-07     0               0       0   \n",
       "4898414         0  2.058025e-07  1.745122e-06     0               0       0   \n",
       "4898415         0  2.101504e-07  3.066559e-06     0               0       0   \n",
       "4898416         0  2.108751e-07  1.786345e-07     0               0       0   \n",
       "4898417         0  2.108751e-07  1.847417e-07     0               0       0   \n",
       "4898418         0  1.673957e-07  1.333117e-05     0               0       0   \n",
       "4898419         0  2.108751e-07  9.160743e-07     0               0       0   \n",
       "4898420         0  2.065271e-07  3.557422e-07     0               0       0   \n",
       "4898421         0  2.072518e-07  1.655804e-06     0               0       0   \n",
       "4898422         0  2.108751e-07  8.366812e-07     0               0       0   \n",
       "4898423         0  2.108751e-07  1.421442e-06     0               0       0   \n",
       "4898424         0  1.152204e-07  1.206775e-05     0               0       0   \n",
       "4898425         0  1.586998e-07  1.862684e-07     0               0       0   \n",
       "4898426         0  1.536272e-07  1.746648e-06     0               0       0   \n",
       "4898427         0  1.586998e-07  1.801613e-07     0               0       0   \n",
       "4898428         0  1.579751e-07  2.755857e-06     0               0       0   \n",
       "4898429         0  1.586998e-07  9.420297e-07     0               0       0   \n",
       "4898430         0  1.586998e-07  8.382080e-07     0               0       0   \n",
       "\n",
       "         hot  num_failed_logins  logged_in  num_compromised  \\\n",
       "0          0                  0          1                0   \n",
       "1          0                  0          1                0   \n",
       "2          0                  0          1                0   \n",
       "3          0                  0          1                0   \n",
       "4          0                  0          1                0   \n",
       "5          0                  0          1                0   \n",
       "6          0                  0          1                0   \n",
       "7          0                  0          1                0   \n",
       "8          0                  0          1                0   \n",
       "9          0                  0          1                0   \n",
       "10         0                  0          1                0   \n",
       "11         0                  0          1                0   \n",
       "12         0                  0          1                0   \n",
       "13         0                  0          1                0   \n",
       "14         0                  0          1                0   \n",
       "15         0                  0          1                0   \n",
       "16         0                  0          1                0   \n",
       "17         0                  0          1                0   \n",
       "18         0                  0          1                0   \n",
       "19         0                  0          1                0   \n",
       "20         0                  0          1                0   \n",
       "21         0                  0          1                0   \n",
       "22         0                  0          1                0   \n",
       "23         0                  0          1                0   \n",
       "24         0                  0          1                0   \n",
       "25         0                  0          1                0   \n",
       "26         0                  0          1                0   \n",
       "27         0                  0          1                0   \n",
       "28         0                  0          1                0   \n",
       "29         0                  0          1                0   \n",
       "...      ...                ...        ...              ...   \n",
       "4898401    0                  0          1                0   \n",
       "4898402    0                  0          1                0   \n",
       "4898403    0                  0          1                0   \n",
       "4898404    0                  0          1                0   \n",
       "4898405    0                  0          1                0   \n",
       "4898406    0                  0          1                0   \n",
       "4898407    0                  0          1                0   \n",
       "4898408    0                  0          0                0   \n",
       "4898409    0                  0          1                0   \n",
       "4898410    0                  0          1                0   \n",
       "4898411    0                  0          1                0   \n",
       "4898412    0                  0          1                0   \n",
       "4898413    0                  0          1                0   \n",
       "4898414    0                  0          1                0   \n",
       "4898415    0                  0          1                0   \n",
       "4898416    0                  0          1                0   \n",
       "4898417    0                  0          1                0   \n",
       "4898418    0                  0          1                0   \n",
       "4898419    0                  0          1                0   \n",
       "4898420    0                  0          1                0   \n",
       "4898421    0                  0          1                0   \n",
       "4898422    0                  0          1                0   \n",
       "4898423    0                  0          1                0   \n",
       "4898424    0                  0          1                0   \n",
       "4898425    0                  0          1                0   \n",
       "4898426    0                  0          1                0   \n",
       "4898427    0                  0          1                0   \n",
       "4898428    0                  0          1                0   \n",
       "4898429    0                  0          1                0   \n",
       "4898430    0                  0          1                0   \n",
       "\n",
       "                   ...             dst_host_count  dst_host_srv_count  \\\n",
       "0                  ...                   0.000000            0.000000   \n",
       "1                  ...                   0.003922            0.003922   \n",
       "2                  ...                   0.007843            0.007843   \n",
       "3                  ...                   0.011765            0.011765   \n",
       "4                  ...                   0.015686            0.015686   \n",
       "5                  ...                   0.019608            0.019608   \n",
       "6                  ...                   0.023529            0.023529   \n",
       "7                  ...                   0.027451            0.027451   \n",
       "8                  ...                   0.031373            0.031373   \n",
       "9                  ...                   0.035294            0.035294   \n",
       "10                 ...                   0.039216            0.039216   \n",
       "11                 ...                   0.043137            0.043137   \n",
       "12                 ...                   0.047059            0.047059   \n",
       "13                 ...                   0.050980            0.050980   \n",
       "14                 ...                   0.054902            0.054902   \n",
       "15                 ...                   0.058824            0.058824   \n",
       "16                 ...                   0.062745            0.062745   \n",
       "17                 ...                   0.066667            0.066667   \n",
       "18                 ...                   0.070588            0.070588   \n",
       "19                 ...                   0.074510            0.074510   \n",
       "20                 ...                   0.078431            0.078431   \n",
       "21                 ...                   0.082353            0.082353   \n",
       "22                 ...                   0.086275            0.086275   \n",
       "23                 ...                   0.090196            0.090196   \n",
       "24                 ...                   0.094118            0.094118   \n",
       "25                 ...                   0.098039            0.098039   \n",
       "26                 ...                   0.101961            0.101961   \n",
       "27                 ...                   0.105882            0.105882   \n",
       "28                 ...                   0.109804            0.109804   \n",
       "29                 ...                   0.113725            0.113725   \n",
       "...                ...                        ...                 ...   \n",
       "4898401            ...                   0.031373            1.000000   \n",
       "4898402            ...                   0.035294            1.000000   \n",
       "4898403            ...                   0.039216            1.000000   \n",
       "4898404            ...                   0.043137            1.000000   \n",
       "4898405            ...                   0.047059            1.000000   \n",
       "4898406            ...                   0.050980            1.000000   \n",
       "4898407            ...                   0.054902            1.000000   \n",
       "4898408            ...                   0.058824            1.000000   \n",
       "4898409            ...                   0.062745            1.000000   \n",
       "4898410            ...                   0.066667            1.000000   \n",
       "4898411            ...                   0.070588            1.000000   \n",
       "4898412            ...                   0.074510            1.000000   \n",
       "4898413            ...                   0.078431            1.000000   \n",
       "4898414            ...                   0.082353            1.000000   \n",
       "4898415            ...                   0.086275            1.000000   \n",
       "4898416            ...                   0.090196            1.000000   \n",
       "4898417            ...                   0.094118            1.000000   \n",
       "4898418            ...                   0.098039            1.000000   \n",
       "4898419            ...                   0.101961            1.000000   \n",
       "4898420            ...                   0.105882            1.000000   \n",
       "4898421            ...                   0.109804            1.000000   \n",
       "4898422            ...                   0.113725            1.000000   \n",
       "4898423            ...                   0.117647            1.000000   \n",
       "4898424            ...                   0.003922            1.000000   \n",
       "4898425            ...                   0.007843            1.000000   \n",
       "4898426            ...                   0.011765            1.000000   \n",
       "4898427            ...                   0.015686            1.000000   \n",
       "4898428            ...                   0.019608            1.000000   \n",
       "4898429            ...                   0.023529            1.000000   \n",
       "4898430            ...                   0.027451            1.000000   \n",
       "\n",
       "         dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "0                             0                       0   \n",
       "1                             1                       0   \n",
       "2                             1                       0   \n",
       "3                             1                       0   \n",
       "4                             1                       0   \n",
       "5                             1                       0   \n",
       "6                             1                       0   \n",
       "7                             1                       0   \n",
       "8                             1                       0   \n",
       "9                             1                       0   \n",
       "10                            1                       0   \n",
       "11                            1                       0   \n",
       "12                            1                       0   \n",
       "13                            1                       0   \n",
       "14                            1                       0   \n",
       "15                            1                       0   \n",
       "16                            1                       0   \n",
       "17                            1                       0   \n",
       "18                            1                       0   \n",
       "19                            1                       0   \n",
       "20                            1                       0   \n",
       "21                            1                       0   \n",
       "22                            1                       0   \n",
       "23                            1                       0   \n",
       "24                            1                       0   \n",
       "25                            1                       0   \n",
       "26                            1                       0   \n",
       "27                            1                       0   \n",
       "28                            1                       0   \n",
       "29                            1                       0   \n",
       "...                         ...                     ...   \n",
       "4898401                       1                       0   \n",
       "4898402                       1                       0   \n",
       "4898403                       1                       0   \n",
       "4898404                       1                       0   \n",
       "4898405                       1                       0   \n",
       "4898406                       1                       0   \n",
       "4898407                       1                       0   \n",
       "4898408                       1                       0   \n",
       "4898409                       1                       0   \n",
       "4898410                       1                       0   \n",
       "4898411                       1                       0   \n",
       "4898412                       1                       0   \n",
       "4898413                       1                       0   \n",
       "4898414                       1                       0   \n",
       "4898415                       1                       0   \n",
       "4898416                       1                       0   \n",
       "4898417                       1                       0   \n",
       "4898418                       1                       0   \n",
       "4898419                       1                       0   \n",
       "4898420                       1                       0   \n",
       "4898421                       1                       0   \n",
       "4898422                       1                       0   \n",
       "4898423                       1                       0   \n",
       "4898424                       1                       0   \n",
       "4898425                       1                       0   \n",
       "4898426                       1                       0   \n",
       "4898427                       1                       0   \n",
       "4898428                       1                       0   \n",
       "4898429                       1                       0   \n",
       "4898430                       1                       0   \n",
       "\n",
       "         dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "0                               0.00                         0.00   \n",
       "1                               1.00                         0.00   \n",
       "2                               0.50                         0.00   \n",
       "3                               0.33                         0.00   \n",
       "4                               0.25                         0.00   \n",
       "5                               0.20                         0.00   \n",
       "6                               0.17                         0.00   \n",
       "7                               0.14                         0.00   \n",
       "8                               0.12                         0.00   \n",
       "9                               0.11                         0.00   \n",
       "10                              0.10                         0.00   \n",
       "11                              0.09                         0.00   \n",
       "12                              0.08                         0.00   \n",
       "13                              0.08                         0.00   \n",
       "14                              0.07                         0.00   \n",
       "15                              0.07                         0.00   \n",
       "16                              0.06                         0.00   \n",
       "17                              0.06                         0.00   \n",
       "18                              0.06                         0.00   \n",
       "19                              0.05                         0.00   \n",
       "20                              0.05                         0.00   \n",
       "21                              0.05                         0.00   \n",
       "22                              0.05                         0.00   \n",
       "23                              0.04                         0.00   \n",
       "24                              0.04                         0.00   \n",
       "25                              0.04                         0.00   \n",
       "26                              0.04                         0.00   \n",
       "27                              0.04                         0.00   \n",
       "28                              0.04                         0.00   \n",
       "29                              0.03                         0.00   \n",
       "...                              ...                          ...   \n",
       "4898401                         0.12                         0.05   \n",
       "4898402                         0.11                         0.05   \n",
       "4898403                         0.10                         0.05   \n",
       "4898404                         0.09                         0.05   \n",
       "4898405                         0.08                         0.05   \n",
       "4898406                         0.08                         0.05   \n",
       "4898407                         0.07                         0.05   \n",
       "4898408                         0.07                         0.05   \n",
       "4898409                         0.06                         0.05   \n",
       "4898410                         0.06                         0.05   \n",
       "4898411                         0.06                         0.05   \n",
       "4898412                         0.05                         0.05   \n",
       "4898413                         0.05                         0.05   \n",
       "4898414                         0.05                         0.05   \n",
       "4898415                         0.05                         0.05   \n",
       "4898416                         0.04                         0.05   \n",
       "4898417                         0.04                         0.05   \n",
       "4898418                         0.04                         0.05   \n",
       "4898419                         0.04                         0.05   \n",
       "4898420                         0.04                         0.05   \n",
       "4898421                         0.04                         0.05   \n",
       "4898422                         0.03                         0.05   \n",
       "4898423                         0.03                         0.05   \n",
       "4898424                         1.00                         0.05   \n",
       "4898425                         0.50                         0.05   \n",
       "4898426                         0.33                         0.05   \n",
       "4898427                         0.25                         0.05   \n",
       "4898428                         0.20                         0.05   \n",
       "4898429                         0.17                         0.05   \n",
       "4898430                         0.14                         0.05   \n",
       "\n",
       "         dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "0                        0.00                      0.00                     0   \n",
       "1                        0.00                      0.00                     0   \n",
       "2                        0.00                      0.00                     0   \n",
       "3                        0.00                      0.00                     0   \n",
       "4                        0.00                      0.00                     0   \n",
       "5                        0.00                      0.00                     0   \n",
       "6                        0.00                      0.00                     0   \n",
       "7                        0.00                      0.00                     0   \n",
       "8                        0.00                      0.00                     0   \n",
       "9                        0.00                      0.00                     0   \n",
       "10                       0.00                      0.00                     0   \n",
       "11                       0.00                      0.00                     0   \n",
       "12                       0.00                      0.00                     0   \n",
       "13                       0.00                      0.00                     0   \n",
       "14                       0.00                      0.00                     0   \n",
       "15                       0.00                      0.00                     0   \n",
       "16                       0.00                      0.00                     0   \n",
       "17                       0.00                      0.00                     0   \n",
       "18                       0.00                      0.00                     0   \n",
       "19                       0.00                      0.00                     0   \n",
       "20                       0.00                      0.00                     0   \n",
       "21                       0.00                      0.00                     0   \n",
       "22                       0.00                      0.00                     0   \n",
       "23                       0.00                      0.00                     0   \n",
       "24                       0.00                      0.00                     0   \n",
       "25                       0.00                      0.00                     0   \n",
       "26                       0.00                      0.00                     0   \n",
       "27                       0.00                      0.00                     0   \n",
       "28                       0.00                      0.00                     0   \n",
       "29                       0.00                      0.00                     0   \n",
       "...                       ...                       ...                   ...   \n",
       "4898401                  0.00                      0.01                     0   \n",
       "4898402                  0.00                      0.01                     0   \n",
       "4898403                  0.00                      0.01                     0   \n",
       "4898404                  0.00                      0.01                     0   \n",
       "4898405                  0.08                      0.01                     0   \n",
       "4898406                  0.08                      0.01                     0   \n",
       "4898407                  0.07                      0.01                     0   \n",
       "4898408                  0.07                      0.01                     0   \n",
       "4898409                  0.06                      0.01                     0   \n",
       "4898410                  0.06                      0.01                     0   \n",
       "4898411                  0.06                      0.01                     0   \n",
       "4898412                  0.05                      0.01                     0   \n",
       "4898413                  0.05                      0.01                     0   \n",
       "4898414                  0.05                      0.01                     0   \n",
       "4898415                  0.05                      0.01                     0   \n",
       "4898416                  0.04                      0.01                     0   \n",
       "4898417                  0.04                      0.01                     0   \n",
       "4898418                  0.04                      0.01                     0   \n",
       "4898419                  0.04                      0.01                     0   \n",
       "4898420                  0.04                      0.01                     0   \n",
       "4898421                  0.04                      0.01                     0   \n",
       "4898422                  0.03                      0.01                     0   \n",
       "4898423                  0.03                      0.01                     0   \n",
       "4898424                  0.00                      0.01                     0   \n",
       "4898425                  0.00                      0.01                     0   \n",
       "4898426                  0.00                      0.01                     0   \n",
       "4898427                  0.00                      0.01                     0   \n",
       "4898428                  0.00                      0.01                     0   \n",
       "4898429                  0.00                      0.01                     0   \n",
       "4898430                  0.00                      0.01                     0   \n",
       "\n",
       "         dst_host_srv_rerror_rate  \n",
       "0                               0  \n",
       "1                               0  \n",
       "2                               0  \n",
       "3                               0  \n",
       "4                               0  \n",
       "5                               0  \n",
       "6                               0  \n",
       "7                               0  \n",
       "8                               0  \n",
       "9                               0  \n",
       "10                              0  \n",
       "11                              0  \n",
       "12                              0  \n",
       "13                              0  \n",
       "14                              0  \n",
       "15                              0  \n",
       "16                              0  \n",
       "17                              0  \n",
       "18                              0  \n",
       "19                              0  \n",
       "20                              0  \n",
       "21                              0  \n",
       "22                              0  \n",
       "23                              0  \n",
       "24                              0  \n",
       "25                              0  \n",
       "26                              0  \n",
       "27                              0  \n",
       "28                              0  \n",
       "29                              0  \n",
       "...                           ...  \n",
       "4898401                         0  \n",
       "4898402                         0  \n",
       "4898403                         0  \n",
       "4898404                         0  \n",
       "4898405                         0  \n",
       "4898406                         0  \n",
       "4898407                         0  \n",
       "4898408                         0  \n",
       "4898409                         0  \n",
       "4898410                         0  \n",
       "4898411                         0  \n",
       "4898412                         0  \n",
       "4898413                         0  \n",
       "4898414                         0  \n",
       "4898415                         0  \n",
       "4898416                         0  \n",
       "4898417                         0  \n",
       "4898418                         0  \n",
       "4898419                         0  \n",
       "4898420                         0  \n",
       "4898421                         0  \n",
       "4898422                         0  \n",
       "4898423                         0  \n",
       "4898424                         0  \n",
       "4898425                         0  \n",
       "4898426                         0  \n",
       "4898427                         0  \n",
       "4898428                         0  \n",
       "4898429                         0  \n",
       "4898430                         0  \n",
       "\n",
       "[4898431 rows x 38 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "kdd_data_corrected[num_features] = kdd_data_corrected[num_features].astype(float)\n",
    "kdd_data_corrected[num_features].apply(lambda x: MinMaxScaler().fit_transform(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features_train, features_test, labels_train, labels_test = train_test_split(\n",
    "    kdd_data_corrected[num_features], \n",
    "    kdd_data_corrected['label'], \n",
    "    test_size=0.1, \n",
    "    random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted in 5511.78 seconds\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "pred = clf.predict(features_test)\n",
    "tt = time() - t0\n",
    "print \"Predicted in {} seconds\".format(round(tt,3))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R squared is 0.9995.\n"
     ]
    }
   ],
   "source": [
    "a=pred.astype(int)\n",
    "b=labels_test.astype(int)\n",
    "from sklearn.metrics import accuracy_score\n",
    "acc = accuracy_score(a,b)\n",
    "print \"R squared is {}.\".format(round(acc,4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KMeans Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "k = 30\n",
    "km = KMeans(n_clusters = k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clustered in 120.703 seconds\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "km.fit(features)\n",
    "tt = time()-t0\n",
    "print \"Clustered in {} seconds\".format(round(tt,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     289759\n",
       "23    170786\n",
       "24     19077\n",
       "16      6955\n",
       "8       2308\n",
       "10      1974\n",
       "25      1231\n",
       "21      1003\n",
       "20       572\n",
       "14       100\n",
       "2         59\n",
       "28        33\n",
       "7         32\n",
       "4         23\n",
       "15        21\n",
       "3         16\n",
       "17        12\n",
       "26        11\n",
       "12        10\n",
       "13         9\n",
       "18         7\n",
       "27         6\n",
       "5          4\n",
       "19         3\n",
       "6          3\n",
       "9          2\n",
       "22         2\n",
       "11         1\n",
       "1          1\n",
       "29         1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas.Series(km.labels_).value_counts()\n",
    "#Checking CLuster Sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = kdd_data_10percent['label']\n",
    "label_names = map(\n",
    "    lambda x: pandas.Series([labels[i] for i in range(len(km.labels_)) if km.labels_[i]==x]), \n",
    "    range(k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 0 labels:\n",
      "smurf.          280766\n",
      "normal.           8700\n",
      "pod.               259\n",
      "warezclient.        31\n",
      "imap.                1\n",
      "satan.               1\n",
      "multihop.            1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 1 labels:\n",
      "portsweep.    1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 2 labels:\n",
      "warezclient.    59\n",
      "dtype: int64\n",
      "\n",
      "Cluster 3 labels:\n",
      "warezmaster.    15\n",
      "normal.          1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 4 labels:\n",
      "normal.    23\n",
      "dtype: int64\n",
      "\n",
      "Cluster 5 labels:\n",
      "normal.         3\n",
      "warezmaster.    1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 6 labels:\n",
      "normal.    3\n",
      "dtype: int64\n",
      "\n",
      "Cluster 7 labels:\n",
      "normal.    32\n",
      "dtype: int64\n",
      "\n",
      "Cluster 8 labels:\n",
      "back.      2189\n",
      "normal.     119\n",
      "dtype: int64\n",
      "\n",
      "Cluster 9 labels:\n",
      "normal.    2\n",
      "dtype: int64\n",
      "\n",
      "Cluster 10 labels:\n",
      "normal.      1972\n",
      "rootkit.        1\n",
      "multihop.       1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 11 labels:\n",
      "normal.    1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 12 labels:\n",
      "normal.    9\n",
      "imap.      1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 13 labels:\n",
      "normal.    9\n",
      "dtype: int64\n",
      "\n",
      "Cluster 14 labels:\n",
      "normal.             99\n",
      "buffer_overflow.     1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 15 labels:\n",
      "normal.    21\n",
      "dtype: int64\n",
      "\n",
      "Cluster 16 labels:\n",
      "normal.             6945\n",
      "phf.                   4\n",
      "multihop.              1\n",
      "warezmaster.           1\n",
      "rootkit.               1\n",
      "loadmodule.            1\n",
      "buffer_overflow.       1\n",
      "imap.                  1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 17 labels:\n",
      "normal.    12\n",
      "dtype: int64\n",
      "\n",
      "Cluster 18 labels:\n",
      "normal.    7\n",
      "dtype: int64\n",
      "\n",
      "Cluster 19 labels:\n",
      "normal.    3\n",
      "dtype: int64\n",
      "\n",
      "Cluster 20 labels:\n",
      "normal.       571\n",
      "ftp_write.      1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 21 labels:\n",
      "normal.         906\n",
      "portsweep.       75\n",
      "warezclient.     22\n",
      "dtype: int64\n",
      "\n",
      "Cluster 22 labels:\n",
      "normal.    2\n",
      "dtype: int64\n",
      "\n",
      "Cluster 23 labels:\n",
      "neptune.            107201\n",
      "normal.              57850\n",
      "satan.                1588\n",
      "ipsweep.              1247\n",
      "teardrop.              979\n",
      "portsweep.             964\n",
      "warezclient.           590\n",
      "nmap.                  231\n",
      "guess_passwd.           52\n",
      "smurf.                  24\n",
      "land.                   21\n",
      "imap.                    9\n",
      "rootkit.                 6\n",
      "ftp_write.               6\n",
      "pod.                     5\n",
      "loadmodule.              5\n",
      "buffer_overflow.         2\n",
      "spy.                     2\n",
      "warezmaster.             2\n",
      "multihop.                2\n",
      "dtype: int64\n",
      "\n",
      "Cluster 24 labels:\n",
      "normal.             18755\n",
      "warezclient.          285\n",
      "buffer_overflow.       26\n",
      "loadmodule.             3\n",
      "perl.                   3\n",
      "rootkit.                2\n",
      "warezmaster.            1\n",
      "ftp_write.              1\n",
      "guess_passwd.           1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 25 labels:\n",
      "normal.         1185\n",
      "warezclient.      32\n",
      "back.             14\n",
      "dtype: int64\n",
      "\n",
      "Cluster 26 labels:\n",
      "normal.      10\n",
      "multihop.     1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 27 labels:\n",
      "normal.      5\n",
      "multihop.    1\n",
      "dtype: int64\n",
      "\n",
      "Cluster 28 labels:\n",
      "normal.    33\n",
      "dtype: int64\n",
      "\n",
      "Cluster 29 labels:\n",
      "warezclient.    1\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(k):\n",
    "    print \"Cluster {} labels:\".format(i)\n",
    "    print label_names[i].value_counts()\n",
    "    print\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Assigned clusters in 140.836 seconds\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "pred = km.predict(kdd_data_corrected[num_features])\n",
    "tt = time() - t0\n",
    "print \"Assigned clusters in {} seconds\".format(round(tt,3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### To do \n",
    "to get majoirty labels for each clusters.\n",
    "\n",
    "check this labels with corrected test data to check accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}