006_logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensure that we always get the same results\n",
    "np.random.seed(0)\n",
    "\n",
    "# Helper function to generate a random data sample\n",
    "def generate_random_data_sample(sample_size, feature_dim=2, num_classes=2):\n",
    "    # Create synthetic data using NumPy.\n",
    "    Y = np.random.randint(size=(sample_size, 1), low=0, high=num_classes)\n",
    "    # Make sure that the data is separable\n",
    "    X = (np.random.randn(sample_size, feature_dim)+5) * (Y+1)\n",
    "    return X, Y\n",
    "\n",
    "# Helper function to draw decision boundary\n",
    "def draw_result(x, x_values, y_values, cost=None):\n",
    "    plt.figure(figsize=(10,5))\n",
    "    ax = plt.gca()\n",
    "    ax.grid(color='#b7b7b7', linestyle='-', linewidth=0.5, alpha=0.5)\n",
    "    plt.scatter(x[:,0], x[:,1], c=colors)\n",
    "    plt.plot(x_values, y_values, label='Decision Boundary', color='#121212', linewidth=2, alpha=0.9)\n",
    "    if cost:\n",
    "        ax.text(0, 0, f'error = {cost:.2f}',fontsize=12,color='#000000')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate random separable data\n",
    "sample_size = 128\n",
    "x, labels = generate_random_data_sample(sample_size)\n",
    "y = np.array([labels.T[0]]).T\n",
    "colors = ['#333333' if label == 0 else '#b7b7b7' for label in labels]\n",
    "\n",
    "# generate random decision boundary\n",
    "theta = np.random.randn(3)\n",
    "x_values = [np.min(x[:, 0] - 5), np.max(x[:, 1] + 5)]\n",
    "y_values = -(theta[0] + np.dot(theta[1], x_values))/theta[2]\n",
    "\n",
    "draw_result(x, x_values, y_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic regression\n",
    "\n",
    "Optimize decision boundary with logistic regression.\n",
    "\n",
    "### 1. Feedforward\n",
    "\\begin{equation}\n",
    "\\mathbf{z} = XW\n",
    "\\end{equation}\n",
    "\n",
    "\\begin{equation}\n",
    "\\mathbf{\\hat{y}} = \\sigma(\\mathbf{z})\n",
    "\\end{equation}\n",
    "\n",
    "\\begin{equation}\n",
    "\\sigma(z) = \\frac{1}{1+e^{-z}}\n",
    "\\end{equation}\n",
    "\n",
    "### 2. Compute cost function\n",
    "\\begin{equation}\n",
    "L(\\mathbf{y}, \\mathbf{\\hat{y}}) = -\\frac{1}{m}\\big(\\mathbf{y}^T\\log(\\mathbf{\\hat{y}})+(1-\\mathbf{y})^T\\log(1-\\mathbf{\\hat{y}})\\big)\n",
    "\\end{equation}\n",
    "\n",
    "### 3. Backpropagation\n",
    "\\begin{equation}\n",
    "\\frac{\\partial L(\\mathbf{y}, \\mathbf{\\hat{y}})}{\\partial W} \n",
    "= \\frac{\\partial L(\\mathbf{y}, \\mathbf{\\hat{y}})}{\\partial \\mathbf{\\hat{y}}} \\cdot \\frac{\\partial \\mathbf{\\hat{y}}}{\\partial W} \n",
    "= \\frac{1}{m}X^T(\\mathbf{\\hat{y}} - \\mathbf{y})\n",
    "\\end{equation}\n",
    "\n",
    "### 4. Gradient descent\n",
    "\\begin{equation}\n",
    "W = W - \\alpha  \\frac{\\delta Loss(y, \\hat{y})}{\\delta W}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LogisticRegression:\n",
    "    \"\"\" Simple logistic regression \"\"\"\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def _init_params(self, x, y, iterations, learning_rate, reg_factor):\n",
    "        \"\"\" Initilize parameters. \n",
    "        \n",
    "        ----------\n",
    "        W : ndarray, shape (n_features+1,)\n",
    "            Coefficient vector\n",
    "        \"\"\"\n",
    "        self._X = np.hstack([np.ones((x.shape[0], 1)), x])\n",
    "        self._y = y\n",
    "        self._learning_rate = learning_rate\n",
    "        self._reg_factor = reg_factor\n",
    "        self.weights_ = np.random.rand(self._X.shape[1],1)\n",
    "        self.costs_ = np.zeros(iterations)\n",
    "        \n",
    "    def _sigmoid(self, x):\n",
    "        \"\"\" Computes sigmoid function. \"\"\"\n",
    "        return 1.0/(1 + np.exp(-x))\n",
    "    \n",
    "    def _feedforward(self):\n",
    "        \"\"\" Computes np.dot(X, W). \"\"\"\n",
    "        self._y_hat = self._sigmoid(self._X.dot(self.weights_))\n",
    "        \n",
    "    def _backprop(self):\n",
    "        \"\"\" Update weights. \"\"\"\n",
    "        m = len(self._y)\n",
    "        # update weights with L2 regularization term\n",
    "        _weights = self.weights_.copy()\n",
    "        # ignore bias term\n",
    "        _weights[0, 0] = 0 \n",
    "        self.weights_ -= self._learning_rate * self._X.T.dot(self._y_hat - self._y)/m + self._reg_factor/m*_weights\n",
    "    \n",
    "    def _get_cost(self):\n",
    "        \"\"\" Compute loss. \"\"\"\n",
    "        m = len(self._y)\n",
    "        # cost function with L2 regularization term\n",
    "        _weights = self.weights_.copy()\n",
    "        # ignore bias term\n",
    "        _weights[0, 0] = 0    \n",
    "        return -1/m * (self._y.T.dot(np.log(self._y_hat)) + (1-self._y).T.dot(np.log(1-self._y_hat))) + self._reg_factor/(2*m)*_weights.T.dot(_weights)\n",
    "\n",
    "    def fit(self, x, y, iterations=1000, learning_rate=0.2, reg_factor=0.5):\n",
    "        \"\"\" Fit model.\n",
    "        \n",
    "        ----------\n",
    "        x : ndarray, shape (n_samples, n_features)\n",
    "            Training data\n",
    "        y : ndarray, shape (n_samples,)\n",
    "            Target data\n",
    "        \"\"\"\n",
    "        self._init_params(x, y, iterations, learning_rate, reg_factor)\n",
    "        \n",
    "        # train model\n",
    "        for i in range(iterations):\n",
    "            self._feedforward()\n",
    "            self._backprop()\n",
    "            self.costs_[i] = self._get_cost()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression()\n",
    "lr.fit(x, y)\n",
    "\n",
    "new_y_values = -(lr.weights_[0] + lr.weights_[1]* x_values)/lr.weights_[2]\n",
    "\n",
    "draw_result(x, x_values, new_y_values, lr.costs_[-1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}