01 Graph Convolution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install of pytorch geometric https://github.com/rusty1s/pytorch_geometric\n",
    "\n",
    "from torch_geometric.datasets import Planetoid\n",
    "from torch_geometric.utils import to_dense_adj, add_self_loops\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cora = Planetoid(root='./data', name='Cora')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cora()\n",
      "Number of nodes: 2708\n",
      "Number of edges: 10556\n",
      "Number of training nodes: 140\n",
      "Training node label rate: 0.05\n",
      "Is undirected: True\n"
     ]
    }
   ],
   "source": [
    "print(cora)\n",
    "print(f'Number of nodes: {cora.data.num_nodes}')\n",
    "print(f'Number of edges: {cora.data.num_edges}')\n",
    "print(f'Number of training nodes: {cora.data.train_mask.sum()}')\n",
    "print(f'Training node label rate: {int(cora.data.train_mask.sum()) / cora.data.num_nodes:.2f}')\n",
    "print(f'Is undirected: {cora.data.is_undirected()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(500)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cora.data.val_mask.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Node features\n",
    "X = cora.data.x # document bag of words\n",
    "\n",
    "# Adjacency matrix + add self-loop\n",
    "A = to_dense_adj(cora.data.edge_index).squeeze()\n",
    "A = A + torch.eye(A.shape[0])\n",
    "\n",
    "# Target\n",
    "y = cora.data.y # topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2708, 2708])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 4, 4,  ..., 3, 3, 3])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def masked_acc(logits, y, mask):\n",
    "    \n",
    "    a_max = torch.argmax(logits, dim=-1)\n",
    "    \n",
    "    eq = (a_max == y).float() * mask.float()\n",
    "            \n",
    "    return eq.sum() / mask.float().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def masked_cross_entropy(logits, y, mask):\n",
    "    \n",
    "    masked_y = y.masked_fill(mask == False, value=-1)\n",
    "    \n",
    "    loss = F.cross_entropy(logits, masked_y, ignore_index=-1)\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, A, n_epochs, opt):\n",
    "    \n",
    "    best_acc = 0.\n",
    "\n",
    "    for ep in range(n_epochs):\n",
    "\n",
    "        out = model(X, A) # [N, num_classes]\n",
    "\n",
    "        loss =  masked_cross_entropy(out, y, cora.data.train_mask)\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        opt.step()\n",
    "\n",
    "        acc = masked_acc(out, y, cora.data.test_mask)\n",
    "\n",
    "        if acc > best_acc:\n",
    "\n",
    "            best_acc = acc\n",
    "\n",
    "            print(f'epoch ={ep: .1f}\\t acc ={best_acc: .1%}')\n",
    "            \n",
    "    return best_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graph Convolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GCN(nn.Module):\n",
    "    \n",
    "    def __init__(self, in_feat=1433, hid_feat=64, num_classes=7):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.fc_1 = nn.Linear(in_feat, hid_feat)\n",
    "        \n",
    "        self.fc_2 = nn.Linear(hid_feat, num_classes)\n",
    "        \n",
    "    def forward(self, X, A):\n",
    "        \"\"\"\n",
    "        inputs:\n",
    "            X: Node features of shape [N, in_feat]\n",
    "            A: Adjacency matrix of shape [N, N]\n",
    "        returns:\n",
    "            O: Logits of shape [N, num_classes]\n",
    "        \"\"\"\n",
    "        \n",
    "        H = A @ self.fc_1(X) # [N, N] [N, hid_feat]\n",
    "        \n",
    "        H = torch.relu(H)\n",
    "        \n",
    "        O = A @ self.fc_2(H) # num_nodes, dim\n",
    "        \n",
    "        return O # [N, num_classes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# degree matrix\n",
    "\n",
    "D = A.sum(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4., 4., 6.,  ..., 2., 5., 5.])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# row normalization\n",
    "\n",
    "A_mean_pooling = torch.diag(1 / D) @ A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 1., 1.,  ..., 1., 1., 1.])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A_mean_pooling.sum(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# symmetric normalization (Kipf & Welling, 2017)\n",
    "\n",
    "norm_D = 1 / torch.sqrt(D)\n",
    "\n",
    "A_GCN = torch.diag(norm_D) @ A @ torch.diag(norm_D) # (kipf & Welling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = GCN(hid_feat=64)\n",
    "\n",
    "opt = torch.optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch = 0.0\t acc = 10.4%\n",
      "epoch = 3.0\t acc = 10.6%\n",
      "epoch = 4.0\t acc = 11.0%\n",
      "epoch = 5.0\t acc = 11.6%\n",
      "epoch = 6.0\t acc = 13.0%\n",
      "epoch = 7.0\t acc = 14.2%\n",
      "epoch = 8.0\t acc = 16.5%\n",
      "epoch = 9.0\t acc = 19.4%\n",
      "epoch = 10.0\t acc = 22.9%\n",
      "epoch = 11.0\t acc = 25.5%\n",
      "epoch = 12.0\t acc = 28.5%\n",
      "epoch = 13.0\t acc = 31.5%\n",
      "epoch = 14.0\t acc = 33.6%\n",
      "epoch = 15.0\t acc = 35.8%\n",
      "epoch = 16.0\t acc = 38.0%\n",
      "epoch = 17.0\t acc = 40.3%\n",
      "epoch = 18.0\t acc = 41.0%\n",
      "epoch = 19.0\t acc = 42.4%\n",
      "epoch = 20.0\t acc = 43.4%\n",
      "epoch = 21.0\t acc = 44.8%\n",
      "epoch = 22.0\t acc = 45.6%\n",
      "epoch = 23.0\t acc = 46.4%\n",
      "epoch = 24.0\t acc = 46.6%\n",
      "epoch = 25.0\t acc = 47.1%\n",
      "epoch = 27.0\t acc = 47.6%\n",
      "epoch = 30.0\t acc = 48.2%\n",
      "epoch = 31.0\t acc = 48.4%\n",
      "epoch = 33.0\t acc = 49.0%\n",
      "epoch = 37.0\t acc = 49.1%\n",
      "epoch = 40.0\t acc = 49.7%\n",
      "epoch = 41.0\t acc = 49.9%\n",
      "epoch = 45.0\t acc = 50.0%\n",
      "epoch = 47.0\t acc = 50.4%\n",
      "epoch = 48.0\t acc = 50.5%\n",
      "epoch = 49.0\t acc = 50.8%\n",
      "epoch = 50.0\t acc = 50.9%\n",
      "epoch = 51.0\t acc = 51.4%\n",
      "epoch = 54.0\t acc = 51.5%\n",
      "epoch = 55.0\t acc = 51.6%\n",
      "epoch = 56.0\t acc = 51.8%\n",
      "epoch = 57.0\t acc = 52.2%\n",
      "epoch = 58.0\t acc = 52.4%\n",
      "epoch = 59.0\t acc = 52.7%\n",
      "epoch = 60.0\t acc = 52.8%\n",
      "epoch = 61.0\t acc = 53.0%\n",
      "epoch = 62.0\t acc = 53.2%\n",
      "epoch = 64.0\t acc = 53.3%\n",
      "epoch = 65.0\t acc = 53.5%\n",
      "epoch = 66.0\t acc = 53.8%\n",
      "epoch = 71.0\t acc = 53.9%\n",
      "epoch = 73.0\t acc = 54.0%\n",
      "epoch = 74.0\t acc = 54.1%\n",
      "epoch = 75.0\t acc = 54.6%\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(0.5460)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_model(model, torch.eye(cora.data.x.shape[0]), 100, opt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GCN with Pytorch Geometric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch_geometric.nn import GCNConv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GCNpyG(nn.Module):\n",
    "    \n",
    "    def __init__(self, in_feat=1433, hid_feat=64, num_classes=7):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.gcn_1 = GCNConv(in_channels=in_feat, out_channels=hid_feat)\n",
    "        \n",
    "        self.gcn_2 = GCNConv(in_channels=hid_feat, out_channels=num_classes)\n",
    "        \n",
    "    def forward(self, data):\n",
    "        \n",
    "        X, edge_index = data.x, data.edge_index\n",
    "        # edge_index of shape [2, num_edges]\n",
    "        # X of shape [N, in_feat]\n",
    "        \n",
    "        # no need to add self-loops \n",
    "        # edge_index, _ = add_self_loops(edge_index, num_nodes=data.num_nodes)\n",
    "                \n",
    "        # the gcn layer takes the node features and the edge_index tensor\n",
    "        H = self.gcn_1(X, edge_index) # [N, hid_feat]\n",
    "        \n",
    "        H = torch.relu(H)\n",
    "        \n",
    "        O = self.gcn_2(H, edge_index) # [N, num_classes]\n",
    "        \n",
    "        return O"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model_pyg(model, n_epochs, opt):\n",
    "    \n",
    "    best_acc = 0.\n",
    "\n",
    "    for ep in range(n_epochs):\n",
    "\n",
    "        out = model(cora.data)\n",
    "\n",
    "        loss =  masked_cross_entropy(out, y, cora.data.train_mask)\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        opt.step()\n",
    "\n",
    "        acc = masked_acc(out, y, cora.data.test_mask)\n",
    "\n",
    "        if acc > best_acc:\n",
    "\n",
    "            best_acc = acc\n",
    "\n",
    "            print(f'epoch ={ep: .1f}\\t acc ={best_acc: .1%}')\n",
    "            \n",
    "    return best_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyg_gcn = GCNpyG()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_2 = torch.optim.Adam(pyg_gcn.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.Size([2, 13264])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch = 0.0\t acc = 16.2%\n",
      "epoch = 1.0\t acc = 23.3%\n",
      "epoch = 2.0\t acc = 30.1%\n",
      "epoch = 3.0\t acc = 39.9%\n",
      "epoch = 4.0\t acc = 47.8%\n",
      "epoch = 5.0\t acc = 54.5%\n",
      "epoch = 6.0\t acc = 59.4%\n",
      "epoch = 7.0\t acc = 63.7%\n",
      "epoch = 8.0\t acc = 66.1%\n",
      "epoch = 9.0\t acc = 68.3%\n",
      "epoch = 10.0\t acc = 70.3%\n",
      "epoch = 11.0\t acc = 72.1%\n",
      "epoch = 12.0\t acc = 73.2%\n",
      "epoch = 13.0\t acc = 74.3%\n",
      "epoch = 14.0\t acc = 75.1%\n",
      "epoch = 15.0\t acc = 75.5%\n",
      "epoch = 16.0\t acc = 75.7%\n",
      "epoch = 17.0\t acc = 75.8%\n",
      "epoch = 18.0\t acc = 76.3%\n",
      "epoch = 19.0\t acc = 76.6%\n",
      "epoch = 20.0\t acc = 77.0%\n",
      "epoch = 21.0\t acc = 77.2%\n",
      "epoch = 24.0\t acc = 77.6%\n",
      "epoch = 26.0\t acc = 78.0%\n",
      "epoch = 27.0\t acc = 78.3%\n",
      "epoch = 31.0\t acc = 78.6%\n",
      "epoch = 32.0\t acc = 78.7%\n",
      "epoch = 35.0\t acc = 79.2%\n",
      "epoch = 37.0\t acc = 79.7%\n",
      "epoch = 38.0\t acc = 79.8%\n",
      "epoch = 40.0\t acc = 79.9%\n",
      "epoch = 41.0\t acc = 80.2%\n",
      "epoch = 42.0\t acc = 80.4%\n",
      "epoch = 45.0\t acc = 80.6%\n",
      "epoch = 47.0\t acc = 80.7%\n",
      "epoch = 50.0\t acc = 81.2%\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(0.8120)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_model_pyg(pyg_gcn, 100, opt_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py37_UZA",
   "language": "python",
   "name": "py37_uza"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}