HiCS_biclustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Try HiCS compare it with LOF and PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "import pandas as pd\n",
    "from scipy import stats\n",
    "import itertools\n",
    "import random\n",
    "import numpy as np\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.preprocessing import MinMaxScaler, label_binarize\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('PCA_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Item_Identifier</th>\n",
       "      <th>Item_Weight</th>\n",
       "      <th>Item_Fat_Content</th>\n",
       "      <th>Item_Visibility</th>\n",
       "      <th>Item_Type</th>\n",
       "      <th>Item_MRP</th>\n",
       "      <th>Outlet_Identifier</th>\n",
       "      <th>Outlet_Establishment_Year</th>\n",
       "      <th>Outlet_Size</th>\n",
       "      <th>Outlet_Location_Type</th>\n",
       "      <th>Outlet_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FDW58</td>\n",
       "      <td>20.750</td>\n",
       "      <td>Low Fat</td>\n",
       "      <td>0.007565</td>\n",
       "      <td>Snack Foods</td>\n",
       "      <td>107.8622</td>\n",
       "      <td>OUT049</td>\n",
       "      <td>1999</td>\n",
       "      <td>Medium</td>\n",
       "      <td>Tier 1</td>\n",
       "      <td>Supermarket Type1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FDW14</td>\n",
       "      <td>8.300</td>\n",
       "      <td>reg</td>\n",
       "      <td>0.038428</td>\n",
       "      <td>Dairy</td>\n",
       "      <td>87.3198</td>\n",
       "      <td>OUT017</td>\n",
       "      <td>2007</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tier 2</td>\n",
       "      <td>Supermarket Type1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NCN55</td>\n",
       "      <td>14.600</td>\n",
       "      <td>Low Fat</td>\n",
       "      <td>0.099575</td>\n",
       "      <td>Others</td>\n",
       "      <td>241.7538</td>\n",
       "      <td>OUT010</td>\n",
       "      <td>1998</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tier 3</td>\n",
       "      <td>Grocery Store</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FDQ58</td>\n",
       "      <td>7.315</td>\n",
       "      <td>Low Fat</td>\n",
       "      <td>0.015388</td>\n",
       "      <td>Snack Foods</td>\n",
       "      <td>155.0340</td>\n",
       "      <td>OUT017</td>\n",
       "      <td>2007</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tier 2</td>\n",
       "      <td>Supermarket Type1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FDY38</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Regular</td>\n",
       "      <td>0.118599</td>\n",
       "      <td>Dairy</td>\n",
       "      <td>234.2300</td>\n",
       "      <td>OUT027</td>\n",
       "      <td>1985</td>\n",
       "      <td>Medium</td>\n",
       "      <td>Tier 3</td>\n",
       "      <td>Supermarket Type3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Item_Identifier  Item_Weight Item_Fat_Content  Item_Visibility    Item_Type  \\\n",
       "0           FDW58       20.750          Low Fat         0.007565  Snack Foods   \n",
       "1           FDW14        8.300              reg         0.038428        Dairy   \n",
       "2           NCN55       14.600          Low Fat         0.099575       Others   \n",
       "3           FDQ58        7.315          Low Fat         0.015388  Snack Foods   \n",
       "4           FDY38          NaN          Regular         0.118599        Dairy   \n",
       "\n",
       "   Item_MRP Outlet_Identifier  Outlet_Establishment_Year Outlet_Size  \\\n",
       "0  107.8622            OUT049                       1999      Medium   \n",
       "1   87.3198            OUT017                       2007         NaN   \n",
       "2  241.7538            OUT010                       1998         NaN   \n",
       "3  155.0340            OUT017                       2007         NaN   \n",
       "4  234.2300            OUT027                       1985      Medium   \n",
       "\n",
       "  Outlet_Location_Type        Outlet_Type  \n",
       "0               Tier 1  Supermarket Type1  \n",
       "1               Tier 2  Supermarket Type1  \n",
       "2               Tier 3      Grocery Store  \n",
       "3               Tier 2  Supermarket Type1  \n",
       "4               Tier 3  Supermarket Type3  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Item_Identifier              False\n",
       "Item_Weight                   True\n",
       "Item_Fat_Content             False\n",
       "Item_Visibility              False\n",
       "Item_Type                    False\n",
       "Item_MRP                     False\n",
       "Outlet_Identifier            False\n",
       "Outlet_Establishment_Year    False\n",
       "Outlet_Size                   True\n",
       "Outlet_Location_Type         False\n",
       "Outlet_Type                  False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find columns with NaN\n",
    "\n",
    "pd.isnull(df).sum()>0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# replace NaN with specific values\n",
    "\n",
    "df['Item_Weight'].fillna(0, inplace=True)\n",
    "df['Outlet_Size'].fillna('MISSING', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Item_Identifier</th>\n",
       "      <th>Item_Weight</th>\n",
       "      <th>Item_Fat_Content</th>\n",
       "      <th>Item_Visibility</th>\n",
       "      <th>Item_Type</th>\n",
       "      <th>Item_MRP</th>\n",
       "      <th>Outlet_Identifier</th>\n",
       "      <th>Outlet_Establishment_Year</th>\n",
       "      <th>Outlet_Size</th>\n",
       "      <th>Outlet_Location_Type</th>\n",
       "      <th>Outlet_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FDW58</td>\n",
       "      <td>20.750</td>\n",
       "      <td>Low Fat</td>\n",
       "      <td>0.007565</td>\n",
       "      <td>Snack Foods</td>\n",
       "      <td>107.8622</td>\n",
       "      <td>OUT049</td>\n",
       "      <td>1999</td>\n",
       "      <td>Medium</td>\n",
       "      <td>Tier 1</td>\n",
       "      <td>Supermarket Type1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FDW14</td>\n",
       "      <td>8.300</td>\n",
       "      <td>reg</td>\n",
       "      <td>0.038428</td>\n",
       "      <td>Dairy</td>\n",
       "      <td>87.3198</td>\n",
       "      <td>OUT017</td>\n",
       "      <td>2007</td>\n",
       "      <td>MISSING</td>\n",
       "      <td>Tier 2</td>\n",
       "      <td>Supermarket Type1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NCN55</td>\n",
       "      <td>14.600</td>\n",
       "      <td>Low Fat</td>\n",
       "      <td>0.099575</td>\n",
       "      <td>Others</td>\n",
       "      <td>241.7538</td>\n",
       "      <td>OUT010</td>\n",
       "      <td>1998</td>\n",
       "      <td>MISSING</td>\n",
       "      <td>Tier 3</td>\n",
       "      <td>Grocery Store</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FDQ58</td>\n",
       "      <td>7.315</td>\n",
       "      <td>Low Fat</td>\n",
       "      <td>0.015388</td>\n",
       "      <td>Snack Foods</td>\n",
       "      <td>155.0340</td>\n",
       "      <td>OUT017</td>\n",
       "      <td>2007</td>\n",
       "      <td>MISSING</td>\n",
       "      <td>Tier 2</td>\n",
       "      <td>Supermarket Type1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FDY38</td>\n",
       "      <td>0.000</td>\n",
       "      <td>Regular</td>\n",
       "      <td>0.118599</td>\n",
       "      <td>Dairy</td>\n",
       "      <td>234.2300</td>\n",
       "      <td>OUT027</td>\n",
       "      <td>1985</td>\n",
       "      <td>Medium</td>\n",
       "      <td>Tier 3</td>\n",
       "      <td>Supermarket Type3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Item_Identifier  Item_Weight Item_Fat_Content  Item_Visibility    Item_Type  \\\n",
       "0           FDW58       20.750          Low Fat         0.007565  Snack Foods   \n",
       "1           FDW14        8.300              reg         0.038428        Dairy   \n",
       "2           NCN55       14.600          Low Fat         0.099575       Others   \n",
       "3           FDQ58        7.315          Low Fat         0.015388  Snack Foods   \n",
       "4           FDY38        0.000          Regular         0.118599        Dairy   \n",
       "\n",
       "   Item_MRP Outlet_Identifier  Outlet_Establishment_Year Outlet_Size  \\\n",
       "0  107.8622            OUT049                       1999      Medium   \n",
       "1   87.3198            OUT017                       2007     MISSING   \n",
       "2  241.7538            OUT010                       1998     MISSING   \n",
       "3  155.0340            OUT017                       2007     MISSING   \n",
       "4  234.2300            OUT027                       1985      Medium   \n",
       "\n",
       "  Outlet_Location_Type        Outlet_Type  \n",
       "0               Tier 1  Supermarket Type1  \n",
       "1               Tier 2  Supermarket Type1  \n",
       "2               Tier 3      Grocery Store  \n",
       "3               Tier 2  Supermarket Type1  \n",
       "4               Tier 3  Supermarket Type3  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Item_Identifier               object\n",
       "Item_Weight                  float64\n",
       "Item_Fat_Content              object\n",
       "Item_Visibility              float64\n",
       "Item_Type                     object\n",
       "Item_MRP                     float64\n",
       "Outlet_Identifier             object\n",
       "Outlet_Establishment_Year      int64\n",
       "Outlet_Size                   object\n",
       "Outlet_Location_Type          object\n",
       "Outlet_Type                   object\n",
       "dtype: object"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Remove identifier\n",
    "df = df.drop('Item_Identifier', axis=1)\n",
    "\n",
    "# Convert Categorical to Numerical\n",
    "df.Item_Fat_Content = pd.Categorical(df.Item_Fat_Content)\n",
    "df['Item_Fat_Content'] = df.Item_Fat_Content.cat.codes\n",
    "\n",
    "df.Item_Type = pd.Categorical(df.Item_Type)\n",
    "df['Item_Type'] = df.Item_Type.cat.codes\n",
    "\n",
    "df.Outlet_Identifier = pd.Categorical(df.Outlet_Identifier)\n",
    "df['Outlet_Identifier'] = df.Outlet_Identifier.cat.codes\n",
    "\n",
    "df.Outlet_Size = pd.Categorical(df.Outlet_Size)\n",
    "df['Outlet_Size'] = df.Outlet_Size.cat.codes\n",
    "\n",
    "df.Outlet_Location_Type = pd.Categorical(df.Outlet_Location_Type)\n",
    "df['Outlet_Location_Type'] = df.Outlet_Location_Type.cat.codes\n",
    "\n",
    "df.Outlet_Type = pd.Categorical(df.Outlet_Type)\n",
    "df['Outlet_Type'] = df.Outlet_Type.cat.codes\n",
    "\n",
    "df.Outlet_Establishment_Year = pd.Categorical(df.Outlet_Establishment_Year)\n",
    "df['Outlet_Establishment_Year'] = df.Outlet_Establishment_Year.cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Item_Weight</th>\n",
       "      <th>Item_Fat_Content</th>\n",
       "      <th>Item_Visibility</th>\n",
       "      <th>Item_Type</th>\n",
       "      <th>Item_MRP</th>\n",
       "      <th>Outlet_Identifier</th>\n",
       "      <th>Outlet_Establishment_Year</th>\n",
       "      <th>Outlet_Size</th>\n",
       "      <th>Outlet_Location_Type</th>\n",
       "      <th>Outlet_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20.750</td>\n",
       "      <td>1</td>\n",
       "      <td>0.007565</td>\n",
       "      <td>13</td>\n",
       "      <td>107.8622</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.300</td>\n",
       "      <td>4</td>\n",
       "      <td>0.038428</td>\n",
       "      <td>4</td>\n",
       "      <td>87.3198</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14.600</td>\n",
       "      <td>1</td>\n",
       "      <td>0.099575</td>\n",
       "      <td>11</td>\n",
       "      <td>241.7538</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.315</td>\n",
       "      <td>1</td>\n",
       "      <td>0.015388</td>\n",
       "      <td>13</td>\n",
       "      <td>155.0340</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000</td>\n",
       "      <td>2</td>\n",
       "      <td>0.118599</td>\n",
       "      <td>4</td>\n",
       "      <td>234.2300</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Item_Weight  Item_Fat_Content  Item_Visibility  Item_Type  Item_MRP  \\\n",
       "0       20.750                 1         0.007565         13  107.8622   \n",
       "1        8.300                 4         0.038428          4   87.3198   \n",
       "2       14.600                 1         0.099575         11  241.7538   \n",
       "3        7.315                 1         0.015388         13  155.0340   \n",
       "4        0.000                 2         0.118599          4  234.2300   \n",
       "\n",
       "   Outlet_Identifier  Outlet_Establishment_Year  Outlet_Size  \\\n",
       "0                  9                          4            2   \n",
       "1                  2                          7            1   \n",
       "2                  0                          3            1   \n",
       "3                  2                          7            1   \n",
       "4                  5                          0            2   \n",
       "\n",
       "   Outlet_Location_Type  Outlet_Type  \n",
       "0                     0            1  \n",
       "1                     1            1  \n",
       "2                     2            0  \n",
       "3                     1            1  \n",
       "4                     2            3  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Calculate LOF\n",
    "def knn(df,k):\n",
    "    nbrs = NearestNeighbors(n_neighbors=3)\n",
    "    nbrs.fit(df)\n",
    "    distances, indices = nbrs.kneighbors(df)\n",
    "    return distances, indices\n",
    "\n",
    "def reachDist(df,MinPts,knnDist):\n",
    "    nbrs = NearestNeighbors(n_neighbors=MinPts)\n",
    "    nbrs.fit(df)\n",
    "    distancesMinPts, indicesMinPts = nbrs.kneighbors(df)\n",
    "    distancesMinPts[:,0] = np.amax(distancesMinPts,axis=1)\n",
    "    distancesMinPts[:,1] = np.amax(distancesMinPts,axis=1)\n",
    "    distancesMinPts[:,2] = np.amax(distancesMinPts,axis=1)\n",
    "    return distancesMinPts, indicesMinPts\n",
    "\n",
    "def ird(MinPts,knnDistMinPts):\n",
    "    return (MinPts/np.sum(knnDistMinPts,axis=1))\n",
    "\n",
    "def lof(Ird,MinPts,dsts):\n",
    "    lof=[]\n",
    "    for item in dsts:\n",
    "        tempIrd = np.divide(Ird[item[1:]],Ird[item[0]])\n",
    "        lof.append(tempIrd.sum()/MinPts)\n",
    "    return lof\n",
    "\n",
    "\n",
    "# Create possible subspaces in each step\n",
    "def comboGenerator(startPoint,space,n):\n",
    "    combosFinal=[]\n",
    "    for item in itertools.combinations(list(set(space)-set(startPoint)),(n-len(startPoint))):\n",
    "        combosFinal.append(sorted(startPoint+list(item)))\n",
    "    return combosFinal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Item_Weight  Item_Fat_Content  Item_Visibility  Item_Type  Item_MRP  \\\n",
      "0     0.985908          0.337528         0.073755   0.869062  0.332160   \n",
      "1     0.361723          1.000000         0.364725   0.238659  0.210790   \n",
      "2     0.682667          0.337528         0.779792   0.784558  0.931526   \n",
      "3     0.296547          0.337528         0.130611   0.869062  0.580708   \n",
      "4     0.086049          0.809925         0.847914   0.238659  0.908819   \n",
      "\n",
      "   Outlet_Identifier  Outlet_Establishment_Year  Outlet_Size  \\\n",
      "0           1.000000                   0.539321     0.646505   \n",
      "1           0.242018                   0.885063     0.291577   \n",
      "2           0.034534                   0.447185     0.291577   \n",
      "3           0.242018                   0.885063     0.291577   \n",
      "4           0.538118                   0.090926     0.646505   \n",
      "\n",
      "   Outlet_Location_Type  \n",
      "0              0.174480  \n",
      "1              0.552136  \n",
      "2              1.000000  \n",
      "3              0.552136  \n",
      "4              1.000000  \n"
     ]
    }
   ],
   "source": [
    "# Calculate the index, we use this for selecting random sections in subspace\n",
    "## and scaling\n",
    "\n",
    "index_df = (df.rank()/df.rank().max()).iloc[:,:-1]\n",
    "print index_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df['Outlet_Type'] = df['Outlet_Type'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Item_Weight                  float64\n",
       "Item_Fat_Content                int8\n",
       "Item_Visibility              float64\n",
       "Item_Type                       int8\n",
       "Item_MRP                     float64\n",
       "Outlet_Identifier               int8\n",
       "Outlet_Establishment_Year       int8\n",
       "Outlet_Size                     int8\n",
       "Outlet_Location_Type            int8\n",
       "Outlet_Type                     int8\n",
       "dtype: object"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Start with 2-D subspaces\n",
    "listOfCombos = comboGenerator([],df.columns[:-1],2)\n",
    "testedCombos=[]\n",
    "selection=[]\n",
    "# Calculate the contrast score for each subspace\n",
    "# For each subspace that satisfies the cut_off point criteria, add additional dimensions\n",
    "while(len(listOfCombos)>0):\n",
    "    if listOfCombos[0] not in testedCombos:\n",
    "        alpha1 = pow(0.2,(float(1)/float(len(listOfCombos[0]))))\n",
    "        pvalue_Total =0\n",
    "        pvalue_cnt = 0\n",
    "        avg_pvalue=0\n",
    "        for i in range(0,50):\n",
    "            lband = random.random()\n",
    "            uband = lband+alpha1\n",
    "            v = random.randint(0,(len(listOfCombos[0])-1))\n",
    "            rest = list(set(listOfCombos[0])-set([listOfCombos[0][v]]))\n",
    "            k=stats.ks_2samp(df[listOfCombos[0][v]].values, df[((index_df[rest]<uband) & (index_df[rest]>lband)).all(axis=1)][listOfCombos[0][v]].values)\n",
    "            if not(np.isnan(k.pvalue)):\n",
    "                pvalue_Total = pvalue_Total+k.pvalue\n",
    "                pvalue_cnt = pvalue_cnt+1\n",
    "        if pvalue_cnt>0:\n",
    "            avg_pvalue = pvalue_Total/pvalue_cnt\n",
    "        if (1.0-avg_pvalue)>0.75:\n",
    "            selection.append(listOfCombos[0])\n",
    "            listOfCombos = listOfCombos + comboGenerator(listOfCombos[0],df.columns[:-1],(len(listOfCombos[0])+1))\n",
    "        testedCombos.append(listOfCombos[0])\n",
    "        listOfCombos.pop(0)\n",
    "        listOfCombos = [list(t) for t in set(map(tuple,listOfCombos))]\n",
    "    else:\n",
    "        listOfCombos.pop(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Item_MRP', 'Item_Visibility', 'Outlet_Establishment_Year', 'Outlet_Identifier', 'Outlet_Location_Type', 'Outlet_Size']\n"
     ]
    }
   ],
   "source": [
    "# selected feature combinations\n",
    "\n",
    "print selection[7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 3, 2])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check unique values in a column\n",
    "\n",
    "df.Outlet_Type.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HCiS AUC Score\n",
      "0.605753326865\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Python/2.7/site-packages/ipykernel-4.2.1-py2.7.egg/ipykernel/__main__.py:18: RuntimeWarning: divide by zero encountered in divide\n",
      "/Library/Python/2.7/site-packages/ipykernel-4.2.1-py2.7.egg/ipykernel/__main__.py:23: RuntimeWarning: invalid value encountered in divide\n"
     ]
    }
   ],
   "source": [
    "# Calculate the contrast score 50 times for each subspace, average the contrast scores from iterations\n",
    "scoresList=[]\n",
    "for item in selection:\n",
    "    m=50\n",
    "    knndist, knnindices = knn(df[item],3)\n",
    "    reachdist, reachindices = reachDist(df[item],m,knndist)\n",
    "    irdMatrix = ird(m,reachdist)\n",
    "    lofScores = lof(irdMatrix,m,reachindices)\n",
    "    scoresList.append(lofScores)\n",
    "\n",
    "# Calculate average LOF score for each data point from each subspace\n",
    "avgs = np.nanmean(np.ma.masked_invalid(np.array(scoresList)),axis=0)\n",
    "\n",
    "# Scale the results to 0,1 range\n",
    "scaled_avgs = MinMaxScaler().fit_transform(avgs.reshape(-1,1))\n",
    "\n",
    "# Here is the AUC score from HiCS\n",
    "print \"HCiS AUC Score\"\n",
    "print metrics.roc_auc_score(label_binarize(df['Outlet_Type'], classes=[0, 1, 2, 3]),scaled_avgs)  # CHANGE to your label name\n",
    "\n",
    "# Got \n",
    "# HCiS AUC Score: 0.605753326865"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOF AUC Score\n",
      "0.555114263833\n"
     ]
    }
   ],
   "source": [
    "# Calculate LOF AUC\n",
    "\n",
    "m=50\n",
    "knndist, knnindices = knn(df.iloc[:,:-1],3)\n",
    "reachdist, reachindices = reachDist(df.iloc[:,:-1],m,knndist)\n",
    "irdMatrix = ird(m,reachdist)\n",
    "lofScores = lof(irdMatrix,m,reachindices)\n",
    "ss=MinMaxScaler().fit_transform(np.array(lofScores).reshape(-1,1))\n",
    "print \"LOF AUC Score\"\n",
    "print metrics.roc_auc_score(label_binarize(df['Outlet_Type'], classes=[0, 1, 2, 3]),ss)   # CHANGE to your label name"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}