Lab_7_Principal_Componenet_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a4859540",
   "metadata": {},
   "source": [
    "# Principal Components Analysis Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6ce584d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3f7d6eda",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([\n",
    "    [1,0,-1],\n",
    "    [-1,1,0]\n",
    "]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f6104c35",
   "metadata": {},
   "outputs": [],
   "source": [
    "#X = np.array([\n",
    "#    [90,90,60,60,30],\n",
    "#    [60,90,60,60,30],\n",
    "#    [90,30,60,90,30]\n",
    "#]).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a929982",
   "metadata": {},
   "source": [
    "# Step_1: Normalize the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "44753e6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0.])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_Mean = np.mean(X,0)\n",
    "X_Mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5113e639",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5, -0.5],\n",
       "       [ 0. ,  0.5],\n",
       "       [-0.5,  0. ]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S = (X-X_Mean)/(len(X)-1)\n",
    "X = S\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "824bd326",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N = len(X)\n",
    "N"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "596ea155",
   "metadata": {},
   "source": [
    "# Step_2: Find or Calculate the Covariance Matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0caa5d8b",
   "metadata": {},
   "source": [
    "### It is important to create a square matrix. Factorization of a number into components needs a square matrix. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1f103b43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5  , -0.25 , -0.25 ],\n",
       "       [-0.25 ,  0.125,  0.125],\n",
       "       [-0.25 ,  0.125,  0.125]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_Cov = np.cov(X)\n",
    "X_Cov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "84d43db3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.25 , -0.125],\n",
       "       [-0.125,  0.25 ]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_Cov = np.cov(X.T)\n",
    "X_Cov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b889a135",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5 , -0.25],\n",
       "       [-0.25,  0.5 ]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#C_My = X.T*X\n",
    "C_My = np.matmul(X.T,X)\n",
    "C_My"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ca82c8c5",
   "metadata": {},
   "source": [
    "# Step_3: Find Eigen Values and Vectors on Covariance Matrix C"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "094bf2c5",
   "metadata": {},
   "source": [
    "### C*X = X*Lambda --> (C-Lambda)X=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "eebc6de7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import linalg as la"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ea90e29",
   "metadata": {},
   "source": [
    "### Calculate eigen Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cf890dcc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.75, 0.25])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u,v = la.eig(C_My)\n",
    "u"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98463d51",
   "metadata": {},
   "source": [
    "### Calculate eigen Vectors v --> Normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fff52dda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.70710678,  0.70710678],\n",
       "       [-0.70710678,  0.70710678]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b69d370",
   "metadata": {},
   "source": [
    "# Step_4: Sort the eigen values or arrange them form top to bottom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d4662f0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = u.argsort()[::-1]   \n",
    "u = u[idx]\n",
    "v = v[:,idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "745966d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.75, 0.25])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "939c64ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.70710678,  0.70710678],\n",
       "       [-0.70710678,  0.70710678]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f55f4d83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.70710678, -0.70710678]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PC1 = np.array([v[:,0]])\n",
    "PC1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "388b0c68",
   "metadata": {},
   "source": [
    "### The above eigen vector is the top principal component associated with the max eigen value"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96c13365",
   "metadata": {},
   "source": [
    "# Step_5: Reconstruct the dimensionally reduced X using the TOP PC1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8e6fe2c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_d = PC1*X.T\n",
    "#X_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b6340c3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.70710678, -0.35355339, -0.35355339]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_d = np.matmul(PC1,(X-X_Mean).T)\n",
    "X_d"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26b5410e",
   "metadata": {},
   "source": [
    "# Step_6: Compare the dimensions of X and X_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0bc41dbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 1)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_d.T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "13883c60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 2)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bdfec4c",
   "metadata": {},
   "source": [
    "### Calculate the mean square error between the original and reconstructed "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "69b9899c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mse = np.sum((X-X_d)**2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1501d9c",
   "metadata": {},
   "source": [
    "# Apply Inverse PCA and then compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "553ae398",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5 , -0.5 ],\n",
       "       [-0.25,  0.25],\n",
       "       [-0.25,  0.25]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_IPCA = np.matmul(PC1.T,X_d).T\n",
    "X_IPCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "503177d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.125"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse = np.sum((X-X_IPCA)**2)/len(X[0])\n",
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fa049416",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5, -0.5],\n",
       "       [ 0. ,  0.5],\n",
       "       [-0.5,  0. ]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2cbe909",
   "metadata": {},
   "source": [
    "# Apply PCA using sklearn API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4e7827ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "991ccd2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "edea39a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PCA(n_components=2)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9b664ea5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.75 0.25]\n"
     ]
    }
   ],
   "source": [
    "print(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "34631e33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.8660254 0.5      ]\n"
     ]
    }
   ],
   "source": [
    "print(pca.singular_values_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "28a10d50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 7.07106781e-01, -1.66533454e-16],\n",
       "       [-3.53553391e-01,  3.53553391e-01],\n",
       "       [-3.53553391e-01, -3.53553391e-01]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_D = pca.transform(X)\n",
    "X_D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "b1a054ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.70710678, -0.70710678],\n",
       "       [ 0.70710678,  0.70710678]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca.components_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "727c01a1",
   "metadata": {},
   "source": [
    "# Task: Apply PCA to "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "d5fa3149",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_Task = np.array([\n",
    "    [90,90,60,60,30],\n",
    "    [60,90,60,60,30],\n",
    "    [90,30,60,90,30]\n",
    "]).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "355aff60",
   "metadata": {},
   "source": [
    "# And reduce the dimensionality to 2. Reconstruct and calculate the mse. Report your findings and insignts. Use sklearn to verify your output."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}