Lab 1) Segmenting and Clustering Neighborhoods in Toronto (Part 1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"text-align:center\">Part 1: Web Scrapping and Data Cleaning</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import selenium\n",
    "\n",
    "# With this code we get the page's source code\n",
    "from selenium import webdriver\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "\n",
    "driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "\n",
    "url = \"https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\"\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiPage = driver.page_source\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "soup = BeautifulSoup(wikiPage)\n",
    "\n",
    "table = soup.findChildren(\"tbody\")[0]\n",
    "rows = table.findChildren([\"tr\"])\n",
    "\n",
    "dictToronto = {\"Postal Code\":[], \"Borough\":[], \"Neighborhood\":[]}\n",
    "columnNames = [\"Postal Code\", \"Borough\", \"Neighborhood\"]\n",
    "\n",
    "# We iterate through the rows of the table and add them to our dictionary\n",
    "for i in range(len(rows)):\n",
    "    \n",
    "    cells = rows[i].findChildren([\"td\"])\n",
    "    \n",
    "    for j in range(len(cells)):\n",
    "        dictToronto[columnNames[j]].append(cells[j].next_element)\n",
    "    \n",
    "import pandas as pd\n",
    "\n",
    "dfToronto = pd.DataFrame.from_dict(dictToronto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Data Cleaning\"\"\"\n",
    "\n",
    "# First we erase trailing spaces\n",
    "dfToronto = dfToronto.apply(lambda x: x.str.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neighborhoods with no assigned value:  0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# We drop rows that have no Borough assigned\n",
    "dfToronto = dfToronto.replace(\"Not assigned\", np.nan).dropna(axis = 0, subset=[\"Borough\"])\n",
    "\n",
    "# Reset index\n",
    "dfToronto.reset_index(drop=True, inplace = True)\n",
    "\n",
    "# We check if there is any Neighborhood with value not assigned\n",
    "print(\"Neighborhoods with no assigned value: \", (dfToronto[\"Neighborhood\"].isnull()).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of the final DataFrame:  (103, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postal Code</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Regent Park, Harbourfront</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>M6A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Lawrence Manor, Lawrence Heights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>M7A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Queen's Park, Ontario Provincial Government</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>98</td>\n",
       "      <td>M8X</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>The Kingsway, Montgomery Road, Old Mill North</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>M4Y</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Church and Wellesley</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>M7Y</td>\n",
       "      <td>East Toronto</td>\n",
       "      <td>Business reply mail Processing Centre, South C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>101</td>\n",
       "      <td>M8Y</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Old Mill South, King's Mill Park, Sunnylea, Hu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>102</td>\n",
       "      <td>M8Z</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Mimico NW, The Queensway West, South of Bloor,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Postal Code           Borough  \\\n",
       "0           M3A        North York   \n",
       "1           M4A        North York   \n",
       "2           M5A  Downtown Toronto   \n",
       "3           M6A        North York   \n",
       "4           M7A  Downtown Toronto   \n",
       "..          ...               ...   \n",
       "98          M8X         Etobicoke   \n",
       "99          M4Y  Downtown Toronto   \n",
       "100         M7Y      East Toronto   \n",
       "101         M8Y         Etobicoke   \n",
       "102         M8Z         Etobicoke   \n",
       "\n",
       "                                          Neighborhood  \n",
       "0                                            Parkwoods  \n",
       "1                                     Victoria Village  \n",
       "2                            Regent Park, Harbourfront  \n",
       "3                     Lawrence Manor, Lawrence Heights  \n",
       "4          Queen's Park, Ontario Provincial Government  \n",
       "..                                                 ...  \n",
       "98       The Kingsway, Montgomery Road, Old Mill North  \n",
       "99                                Church and Wellesley  \n",
       "100  Business reply mail Processing Centre, South C...  \n",
       "101  Old Mill South, King's Mill Park, Sunnylea, Hu...  \n",
       "102  Mimico NW, The Queensway West, South of Bloor,...  \n",
       "\n",
       "[103 rows x 3 columns]"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We print the shape\n",
    "print(\"Shape of the final DataFrame: \", dfToronto.shape)\n",
    "dfToronto"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}