tea-gnn-pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0bdec4bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:98% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:98% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcb30a14",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9762ce43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rel_ids_1',\n",
       " 'sup_pairs',\n",
       " 'rel_ids_2',\n",
       " 'ref_pairs',\n",
       " 'ent_ids_2',\n",
       " 'triples_2',\n",
       " 'ent_ids_1',\n",
       " 'triples_1']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.sparse as sp\n",
    "import torch\n",
    "\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "from CSLS import eval_alignment_by_sim_mat\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "DATA_PATH = Path('TEA-GNN/data/ICEWS05-15')\n",
    "os.listdir(DATA_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8d23a47b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_triples(ent_file, rel_file, triple_file, times=False):\n",
    "    df_ent_1 = pd.read_csv(ent_file, delimiter='\\t', header=None, names=['id', 'entity'], index_col=0)\n",
    "    df_rel_1 = pd.read_csv(rel_file, delimiter='\\t', header=None, names=['relation', 'id'], index_col=1)\n",
    "    \n",
    "    if times:\n",
    "        df_triples_1 = pd.read_csv(triple_file, delimiter='\\t', header=None, names=['head', 'relation', 'tail', 'time_start', 'time_end'])\n",
    "    else:\n",
    "        df_triples_1 = pd.read_csv(triple_file, delimiter='\\t', header=None, names=['head', 'relation', 'tail', 'time'])\n",
    "    \n",
    "    df_trip_ent = df_triples_1.set_index('head').join(df_ent_1, rsuffix='_head').reset_index().rename(columns={'index': 'head', 'entity': 'entity_head'})\n",
    "    df_trip_ent = df_trip_ent.set_index('tail').join(df_ent_1).reset_index().rename(columns={'index': 'tail', 'entity': 'entity_tail'})\n",
    "    df_trip_ent = df_trip_ent.set_index('relation').join(df_rel_1).reset_index().rename(columns={'index': 'relation', 'relation': 'rel'})\n",
    "    \n",
    "    \n",
    "    entities = set(df_trip_ent['head'].unique().tolist() + df_trip_ent['tail'].unique().tolist())\n",
    "    relations = set([0, *(df_trip_ent['relation'].unique() + 1)])\n",
    "    times = set([0, *(df_trip_ent['time'].unique() + 1)])\n",
    "    triples = df_trip_ent[['head', 'relation', 'tail', 'time']].to_numpy().tolist()\n",
    "\n",
    "    return entities, relations, times, triples\n",
    "\n",
    "def load_alignments(align_file):\n",
    "    return pd.read_csv(align_file, delimiter='\\t', header=None, names=['entity_1', 'entity_2']).to_numpy().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "964e9fae",
   "metadata": {},
   "outputs": [],
   "source": [
    "### message passing matrix\n",
    "\n",
    "def normalize_adj(adj):\n",
    "    adj = sp.coo_matrix(adj)\n",
    "    ## creating the degree matrix = D\n",
    "    rowsum = np.array(adj.sum(1))  \n",
    "    d_inv_sqrt = np.power(rowsum, -0.5).flatten() \n",
    "    d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0\n",
    "    d_mat_inv_sqrt = sp.diags(d_inv_sqrt)\n",
    "    ##\n",
    "    return d_mat_inv_sqrt.dot(adj).transpose().dot(d_mat_inv_sqrt).T # creating the adj matrix for message passing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e85c932f",
   "metadata": {},
   "outputs": [],
   "source": [
    "entities_1, relations_1, times_1, triples_1 = create_triples(DATA_PATH/'ent_ids_1', DATA_PATH/'rel_ids_1', DATA_PATH/'triples_1')\n",
    "entities_2, relations_2, times_2, triples_2 = create_triples(DATA_PATH/'ent_ids_2', DATA_PATH/'rel_ids_2', DATA_PATH/'triples_2')\n",
    "train_alignments = load_alignments(DATA_PATH/'ref_pairs')\n",
    "dev_alignments = load_alignments(DATA_PATH/'sup_pairs')\n",
    "all_ent, all_rel, all_times, all_triples = entities_1.union(entities_2), relations_1.union(relations_2), times_1.union(times_2), triples_1 + triples_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c17c2256",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dev_alignments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "82b5ccaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_matrix(all_ent, all_rel, all_times, all_triples):\n",
    "    ent_size = max(all_ent) + 1\n",
    "    rel_size = max(all_rel) + 1\n",
    "    time_size = max(all_times) + 1\n",
    "\n",
    "    adj_matrix = sp.lil_matrix((ent_size, ent_size))\n",
    "    adj_features = sp.lil_matrix((ent_size, ent_size))\n",
    "    radj = []\n",
    "\n",
    "    rel_in = np.zeros((ent_size, rel_size))\n",
    "    rel_out = np.zeros((ent_size, rel_size))\n",
    "\n",
    "    for i in range(ent_size):\n",
    "        adj_features[i, i] = 1\n",
    "\n",
    "    time_link = np.zeros((ent_size, time_size))\n",
    "\n",
    "    for h, r, t, tau in all_triples:\n",
    "        adj_matrix[h, t] = 1; adj_matrix[t, h] = 1\n",
    "        adj_features[h, t] = 1; adj_features[t, h] = 1\n",
    "        radj.append([h, t, r, tau]); radj.append([t, h, r + rel_size, tau])\n",
    "        time_link[h, tau] += 1; time_link[t, tau] += 1\n",
    "        rel_out[h, r] += 1; rel_in[t, r] += 1\n",
    "\n",
    "    count = -1\n",
    "    s = set()\n",
    "    d = {}\n",
    "    r_index, t_index, r_val = [], [], []\n",
    "\n",
    "    for h, t, r, tau in sorted(radj, key = lambda x: x[0] * 10e10 + x[1]*10e5):\n",
    "        if ' '.join([str(h), str(t)]) in s:\n",
    "            r_index.append([count, r])\n",
    "            t_index.append([count, tau])\n",
    "            r_val.append(1)\n",
    "            d[count] += 1\n",
    "        else:\n",
    "            count += 1\n",
    "            d[count]  = 1\n",
    "            s.add(' '.join([str(h), str(t)]))\n",
    "            r_index.append([count, r])\n",
    "            t_index.append([count, tau])\n",
    "            r_val.append(1)\n",
    "    for i in range(len(r_index)):\n",
    "        r_val[i] /= d[r_index[i][0]]\n",
    "\n",
    "    time_features = time_link\n",
    "\n",
    "    time_features = normalize_adj(sp.lil_matrix(time_features)) #### using time as an averager for the node features\n",
    "\n",
    "    rel_features = np.concatenate([rel_in, rel_out], axis=1)\n",
    "    rel_features = normalize_adj(rel_features)  ### use relations as an averager for the node features\n",
    "    adj_features = normalize_adj(adj_features)\n",
    "    \n",
    "    return adj_matrix, np.array(r_index), np.array(r_val), np.array(t_index), adj_features, rel_features, time_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4d8894cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19054, 988, 4018, 198262, 19054)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## r_index => [unique head and tail, relation index numbers]\n",
    "## t_index => [unique head and tail, time index numbers]\n",
    "## r_val ==> equal weight for the unique head and tail over all relations/timestamps\n",
    "\n",
    "adj_matrix, r_index, r_val, t_index, adj_features, rel_features, time_features = get_matrix(all_ent, all_rel, all_times, all_triples)\n",
    "adj_matrix = np.stack(adj_matrix.nonzero(), axis=1) ## all indices where there is \"some kind of adjancent relation\"\n",
    "rel_matrix, rel_val = np.stack(rel_features.nonzero(), axis=1), rel_features.data ## Nodes x Relation array\n",
    "ent_matrix, ent_val = np.stack(adj_features.nonzero(), axis=1), adj_features.data\n",
    "time_matrix, time_val = np.stack(time_features.nonzero(), axis=1), time_features.data\n",
    "\n",
    "node_size = adj_features.shape[0]\n",
    "rel_size = rel_features.shape[1]\n",
    "time_size = time_features.shape[1]\n",
    "triple_size = len(adj_matrix)\n",
    "batch_size = node_size\n",
    "node_size, rel_size, time_size, triple_size, batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4ff74a8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = [np.expand_dims(i, axis=0) for i in [adj_matrix, r_index, r_val, t_index, adj_features, rel_features, time_features]]\n",
    "adj_matrix, r_index, r_val, t_index, adj_features, rel_features, time_features = outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7e9bd220",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1, 198262, 2), (1, 1230210, 2), (1, 1230210), (1,), (1,), (1,))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adj_matrix.shape, r_index.shape, r_val.shape, adj_features.shape, rel_features.shape, time_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "41276db4",
   "metadata": {},
   "outputs": [],
   "source": [
    "ent_dim, rel_dim, time_dim = 128, 128, 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b3b38d20",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda:1'\n",
    "class ContextualEmbedding(nn.Module):\n",
    "    def __init__(self, inp_dim, out_dim=128, other_dim=None):\n",
    "        super(ContextualEmbedding, self).__init__()\n",
    "        self.inp_dim = inp_dim\n",
    "        self.out_dim = out_dim\n",
    "        self.other_dim = other_dim if other_dim is not None else inp_dim\n",
    "        self.embedding = nn.Embedding(self.other_dim, self.out_dim)\n",
    "        \n",
    "    def forward(self, indices):\n",
    "        x = self.embedding(torch.arange(self.other_dim).long().to(device))\n",
    "        size = (self.inp_dim, self.other_dim)\n",
    "        indices = torch.sparse_coo_tensor(indices, values=torch.ones_like(indices[0, :], dtype=torch.float32), size=size)\n",
    "        indices = torch.sparse.softmax(indices, dim=len(indices.size())-1)\n",
    "        return torch.sparse.mm(indices, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3031f9d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "ent_sparse = torch.tensor(np.array([ent_matrix[:, 1], ent_matrix[:, 0]])).long().to(device)\n",
    "rel_sparse = torch.tensor(np.array([rel_matrix[:, 1], rel_matrix[:, 0]])).long().to(device)\n",
    "time_sparse = torch.tensor(np.array([time_matrix[:, 1], time_matrix[:, 0]])).long().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1608115b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class HeteroTimeGAT(nn.Module):\n",
    "    def __init__(self, adj, emb_size):\n",
    "        super(HeteroTimeGAT, self).__init__()\n",
    "        self.attend_relation = nn.Linear(emb_size * 3, 1)\n",
    "        self.attend_time = nn.Linear(emb_size * 3, 1)\n",
    "        self.adj = adj\n",
    "        \n",
    "    def forward(self, xr, xt, selfs, neighs):\n",
    "        indices = self.adj.coalesce().indices()\n",
    "        rel_x = self.attend_relation(xr).view(-1)\n",
    "        rel_x = torch.sparse_coo_tensor(indices, rel_x, self.adj.shape)\n",
    "        rel_x = torch.sparse.softmax(rel_x, 1)\n",
    "        att_rel_x = neighs * rel_x.coalesce().values().unsqueeze(-1)\n",
    "        \n",
    "        time_x = self.attend_time(xt).view(-1)\n",
    "        time_x = torch.sparse_coo_tensor(indices, time_x, self.adj.shape)\n",
    "        time_x = torch.sparse.softmax(time_x, 1)\n",
    "        att_time_x = neighs * time_x.coalesce().values().unsqueeze(-1)\n",
    "        \n",
    "        return att_rel_x, att_time_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "df8ae630",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 10/100 [00:06<01:15,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [53.4 65.8 71.6], mr = 50.431, mrr = 0.593, time = 1.380 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 20/100 [00:10<01:03,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [75.  82.5 85. ], mr = 26.447, mrr = 0.785, time = 1.263 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 30/100 [00:15<00:57,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [85.  90.5 92.1], mr = 12.570, mrr = 0.875, time = 1.371 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 40/100 [00:20<00:48,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [87.8 93.  94.3], mr = 6.505, mrr = 0.902, time = 1.342 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 50/100 [00:24<00:35,  1.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [89.4 94.  95.1], mr = 5.585, mrr = 0.916, time = 1.087 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 60/100 [00:28<00:29,  1.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [89.3 93.6 95.7], mr = 5.605, mrr = 0.915, time = 1.141 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 70/100 [00:33<00:23,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [90.2 94.1 95.5], mr = 4.648, mrr = 0.921, time = 1.244 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 80/100 [00:37<00:14,  1.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [89.6 93.8 95.2], mr = 4.255, mrr = 0.916, time = 1.140 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 90/100 [00:42<00:07,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [89.3 94.  95.2], mr = 4.715, mrr = 0.915, time = 1.317 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:46<00:00,  2.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accurate results: hits@[1, 5, 10] = [89.3 93.8 95.3], mr = 5.869, mrr = 0.913, time = 1.141 s \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def get_train_set(batch_size = batch_size):\n",
    "    negative_ratio =  batch_size // len(train_alignments) + 1\n",
    "    train_set = np.reshape(np.repeat(np.expand_dims(train_alignments,axis=0),axis=0,repeats=negative_ratio),newshape=(-1,2))\n",
    "    np.random.shuffle(train_set); train_set = train_set[:batch_size]\n",
    "    train_set = np.concatenate([train_set,np.random.randint(0,node_size,train_set.shape)],axis = -1)\n",
    "    return train_set\n",
    "    \n",
    "class FullModel(nn.Module):\n",
    "    def __init__(self, adj, node_size, rel_size, time_size, triple_size, ent_sparse, rel_sparse, time_sparse, emb_size=128, depth=1, device='cuda'):\n",
    "        super(FullModel, self).__init__()\n",
    "\n",
    "        self.device = device\n",
    "        self.node_size = node_size\n",
    "        self.rel_size = rel_size\n",
    "        self.time_size = time_size\n",
    "        self.triple_size = triple_size\n",
    "        self.emb_size = emb_size\n",
    "        self.depth = depth\n",
    "        self.adj = adj\n",
    "        self.indices = self.adj.coalesce().indices()\n",
    "        self.indices_size = self.indices.size(1)\n",
    "        self.activation = nn.ReLU()\n",
    "        \n",
    "        self.ent_embedding = ContextualEmbedding(self.node_size, self.emb_size).to(self.device)\n",
    "        self.rel_embedding = ContextualEmbedding(self.rel_size, self.emb_size, self.node_size).to(self.device)\n",
    "        self.time_embedding = ContextualEmbedding(self.time_size, self.emb_size, self.node_size).to(self.device)\n",
    "        \n",
    "        self.attend_relation = nn.Linear(emb_size * 3, 1)\n",
    "        self.attend_time = nn.Linear(emb_size * 3, 1)\n",
    "        \n",
    "        self.dropout = nn.Dropout(0.3)\n",
    "#         self.graph_net = TNR_GraphAttention(adj, node_size, rel_size, time_size, triple_size, emb_size, depth, device)\n",
    "        self.ent_emb = self.ent_embedding(ent_sparse).to(self.device)\n",
    "        self.rel_emb = self.rel_embedding(rel_sparse).to(self.device)\n",
    "        self.time_emb = self.time_embedding(time_sparse).to(self.device)\n",
    "    \n",
    "    def forward(self, inputs):\n",
    "        r_index_sq, r_val_sq, t_index_sq = inputs\n",
    "        self.neighs, self.selfs = self.ent_emb[self.indices.T[:, 1]], self.ent_emb[self.indices.T[:, 0]]\n",
    "        \n",
    "        outputs = []\n",
    "        features = self.activation(self.ent_emb)\n",
    "        node_size = self.node_size\n",
    "        \n",
    "        outputs.append(features)\n",
    "        \n",
    "        for i in range(self.depth):\n",
    "            \n",
    "            rels_sum = torch.sparse_coo_tensor(r_index_sq.T, r_val_sq, (self.triple_size, self.rel_size)).to(self.device)\n",
    "            rels_sum = torch.sparse.mm(rels_sum.float(), self.rel_emb)\n",
    "            rels_sum_n = torch.nn.functional.normalize(rels_sum, 2)\n",
    "            r_neighs = self.neighs - 2 * torch.sum(self.neighs * rels_sum_n, 1, keepdims=True) * rels_sum_n\n",
    "            r_concat = torch.cat([self.selfs, self.neighs, rels_sum_n], axis=1)\n",
    "            \n",
    "            times_sum = torch.sparse_coo_tensor(t_index_sq.T, r_val_sq, (triple_size, time_size)).to(self.device)\n",
    "            times_sum = torch.sparse.mm(times_sum.float(), self.time_emb)\n",
    "            times_sum_n = torch.nn.functional.normalize(times_sum, 2)\n",
    "            t_neighs = self.neighs - 2 * torch.sum(self.neighs * times_sum_n, 1, keepdims=True) * times_sum_n\n",
    "            t_concat = torch.cat([self.selfs, self.neighs, times_sum_n], axis=1)\n",
    "            \n",
    "            indices = self.adj.coalesce().indices()\n",
    "            rel_x = self.attend_relation(r_concat).view(-1)\n",
    "            rel_x = torch.sparse_coo_tensor(indices, rel_x, self.adj.shape)\n",
    "            rel_x = torch.sparse.softmax(rel_x, 1)\n",
    "            att_rel_x = self.neighs * rel_x.coalesce().values().unsqueeze(-1)\n",
    "\n",
    "            time_x = self.attend_time(t_concat).view(-1)\n",
    "            time_x = torch.sparse_coo_tensor(indices, time_x, self.adj.shape)\n",
    "            time_x = torch.sparse.softmax(time_x, 1)\n",
    "            att_time_x = self.neighs * time_x.coalesce().values().unsqueeze(-1)\n",
    "\n",
    "            indices = torch.stack([self.indices[0, :], torch.arange(self.indices[0, :].size(0)).to(self.device)])\n",
    "            \n",
    "            th = torch.sparse_coo_tensor(indices, torch.ones_like(indices[0]), (node_size, self.indices[0, :].size(0)))\n",
    "            new_r = torch.sparse.mm(th.float(), att_rel_x)\n",
    "            new_t = torch.sparse.mm(th.float(), att_time_x)\n",
    "            \n",
    "            new_features = new_r + new_t\n",
    "            \n",
    "            features = self.activation(new_features)\n",
    "            outputs.append(features)\n",
    "            \n",
    "        ent_encoding = torch.cat(outputs, 1)\n",
    "        \n",
    "        times_sum = torch.sparse_coo_tensor(t_index_sq.T, r_val_sq, (self.triple_size, self.time_size)).to(device)\n",
    "        times_sum_n = torch.sparse.mm(times_sum.float(), self.time_emb)\n",
    "        new_indices = torch.vstack([self.adj.coalesce().indices()[0, :], torch.arange(self.adj.coalesce().indices()[0, :].size(0)).to(device)])\n",
    "        time_indices = torch.sparse_coo_tensor(new_indices, torch.ones_like(new_indices[0]), (self.node_size, new_indices[0, :].size(0)))\n",
    "        time_encoding = self.activation(torch.sparse.mm(time_indices.float(), times_sum_n))\n",
    "        \n",
    "        \n",
    "        return self.dropout(torch.cat([ent_encoding, time_encoding], 1))\n",
    "\n",
    "adj_sparse_pre = torch.tensor(adj_matrix).squeeze(0).long()\n",
    "adj_sparse = torch.sparse_coo_tensor(adj_sparse_pre.T, torch.ones_like(adj_sparse_pre[:, 0]), (node_size, node_size)).to(device)\n",
    "ent_sparse = torch.tensor(np.array([ent_matrix[:, 1], ent_matrix[:, 0]])).long().to(device)\n",
    "rel_sparse = torch.tensor(np.array([rel_matrix[:, 1], rel_matrix[:, 0]])).long().to(device)\n",
    "time_sparse = torch.tensor(np.array([time_matrix[:, 1], time_matrix[:, 0]])).long().to(device)\n",
    "r_index_sq, r_val_sq, t_index_sq = r_index.squeeze(0), r_val.squeeze(0), t_index.squeeze(0)\n",
    "alignment_pairs = get_train_set()\n",
    "\n",
    "model = FullModel(adj_sparse, node_size, rel_size, time_size, triple_size, ent_sparse, rel_sparse, time_sparse, depth=2, device=device).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1, weight_decay=0.01)\n",
    "\n",
    "\n",
    "def l1(ll, rr):\n",
    "        return torch.sum(torch.abs(ll - rr), axis=-1)\n",
    "gamma = 1.0\n",
    "losses = []\n",
    "t_prec = []\n",
    "accs = []\n",
    "t_mrrs = []\n",
    "for i in tqdm(range(100)):\n",
    "    model.train()\n",
    "    optimizer.zero_grad()\n",
    "    features = model((r_index_sq, r_val_sq, t_index_sq))\n",
    "    comp = features[alignment_pairs]\n",
    "    l, r, fl, fr = comp[:, 0, :], comp[:, 1, :], comp[:, 2, :], comp[:, 3, :]\n",
    "    loss = torch.sum(nn.ReLU()(gamma + l1(l, r) - l1(l, fr)) + nn.ReLU()(gamma + l1(l, r) - l1(fl, r))) / batch_size\n",
    "    losses.append(loss.item())\n",
    "    loss.backward(retain_graph=True)\n",
    "    optimizer.step()\n",
    "    if ((i + 1) % 10) == 0:\n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            feat = model((r_index_sq, r_val_sq, t_index_sq))[dev_alignments]\n",
    "            Lvec = np.array(feat[:, 0, :].cpu()) #np.array([vec[e1] for e1, e2 in feat])\n",
    "            Rvec = np.array(feat[:, 1, :].cpu()) #np.array([vec[e2] for e1, e2 in dev_alignments])\n",
    "            Lvec = Lvec / np.linalg.norm(Lvec,axis=-1,keepdims=True)\n",
    "            Rvec = Rvec / np.linalg.norm(Rvec,axis=-1,keepdims=True)\n",
    "            t_prec_set, acc, t_mrr = eval_alignment_by_sim_mat(Lvec, Rvec, [1, 5, 10], 1, csls=10, accurate = True)\n",
    "            accs.append(acc)\n",
    "            t_mrrs.append(t_mrr)\n",
    "            t_prec.append(t_prec_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "4de8cb4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fea15ea9fa0>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEjCAYAAAA1ymrVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA4E0lEQVR4nO3dd3xb1fn48c8jeY94Jk5iO2QPsgdJ2EkICSGsQoFAJ4Qwy2rLKHxLoe2XL6X8IIWWFUJLacqexYwwEkaBkG1nkh3bsR3vPaXz++PKju3IjmNLli0979dLL11dHV09EuS5x4/OPUeMMSillAocNl8HoJRSqntp4ldKqQCjiV8ppQKMJn6llAowmviVUirAaOJXSqkAo4lfKaUCjCZ+pZQKMJr4lV8Tkf0iMrfVvp+LyFfNnxeRe0SkwnWrERFHs8dbm71WRORyEflURA6LSK6IrBSRi928txGR4c0e/1pEckRkrDc/s1LHoolfKcAY86AxJsoYEwVcD3zT+NgYMxZAROzAv4ElwIPAUCAVuB+4VkSWiYi4O76I/A9wG3CmMWaruzZKdRdN/Ep13L1ALXC2MeZTY0yFMabeGPM1cC7QB/hJ6xeJyB+Ba4AzjDHfd2vESrkR5OsAlOoNRCQSK3mPAYJF5GngPGAbsA/4O/BL4B3gn81e+hAwESvpH+zWoJVqgyZ+FQjeFpGGZo9DgA3HeYyTgVXGmEoRuQFIAYYDicBXwAvGmGwRSWj1unmu5zTpqx5DSz0qEFxkjIltvAE3duIY/YBs1/Z44G1jTJkxZi9W4kdEooHKVq9bBPxQRB7oXOhKeZ4mfqU6pgAY4NrOAC4SkWgRGQKcBsQBTwLPt3rd98Bc4EYRubu7glWqPZr4leqYb4DZIhIOLAeygN2u7ZXAfa42S1u/0DWKZy5wh4jc1k3xKtUmrfEr1QHGmHIR+Tew1BhzHXB18+dFJMgY0+D+1WCM2Swi84GPRaTGGPO0l0NWqk2iK3Ap1TEiEgS8hvWX8h+BTUAEcCFwJzDTGFPhswCV6iBN/EodBxGxAT/H6vGPAeqAVcCDxpgtPgxNqQ7TxK+UUgFGf9xVSqkAo4lfKaUCjCZ+pZQKMJr4lVIqwGjiV0qpAKOJXymlAowmfqWUCjCa+JVSKsBo4ldKqQCjiV8ppQKMJn6llAowmviVUirAaOJXSqkAo4lfKaUCTK9YgSsxMdEMHjzY12EopVSvsn79+gJjTN/W+3tF4h88eDDr1q3zdRhKKdWriMgBd/u11KOUUgFGE79SSgUYTfxKKRVgNPErpVSA0cSvlFIBRhO/UkoFGE38SikVYHrFOH6lejtjDNUN1RTVFFFcU0xxbTGF1YUU1xZT01BDiD2EYFswIfYQQmwh1mN7cNN2032r7abXuPaLiK8/quoFNPEr1QnGGKoaqpoSubv7olpXknfdahw1Xo8ryBZEiC2EUHtoyxOH68TQel+wLZhQeyghdus1YUFhhAeFt7iF2cMIDz6yHREUQXiwa39QOHab3eufS3mWJn6lsBJ5ZX2llbCb9cpbPK5p+bjOWef2WGH2MOLC4ogLiyM+LJ7hscOJCz3yOD4svsXz4UHh1DvrqXXUUueoo95RT52zjjpHHXVO12PXdut9tY5a6p2tnnfUHbWv+TEr6yspri1uatt4vBpHDTUNNRjMcX13wbbgo04W4UHhTSeR5ieTMHsYEcERTSeN1s+HB4VjF9+fSAwGp3HSYBpwOp04jMO6OR1N207jpMHZ0OK5xn1O4/41brfdHLP563819Vf0jThq1oUu0cSv/JoxhrK6MnIrc8mryrNulXnkVuaSX53fIpHXO+vdHiM8KNxK1qFxJIYnMiJuBAlhCS2Sd1xoHPHh1n1EcMRxxxlqDyXUHtrVj9tlxhhqHbVUN1RT01BDdUN1i1uNw7Wv3tquaqhq0a75dkVdBYerDh91nOM9sfgru9ixiY0gW1CLbZvYsIvdutns1DR4/i9FTfyq1zLGUFpb2pTQcytzj0rweVV5VDdUt3idTWwkhieSFJFEUkQSo+NHWwk8NL4peTfvlYcHhfvoE3Y/ESEsKIywoDCvHN8YQ52zrulk4O7E4TAOr7z38QoSVxK2HUnCjQm5rYQdJEHYbC0Tt7ttm9h8+nuMJn7VIxljKKktaUrojUm86bErsbeum9vFTt+IviRFJDEybiSnp5xO/4j+JEVaSb5/ZH8SwxMJsun/+r4gIk1/3cSExvg6nICl//erbmeMobi2uEVCd9dTr3XUtnidXez0i+jX1EuflTKrRUJPikgiITxBk7pSx6D/QlS3KKgu4PPMz1mduZo1uWuOKr8ESZCV1COTGJswljmD5rRI6EmRSSSEJegIEm9xusor+v0GBE38yiuMMewq2cXqzNWszlxNRkEGAMlRyVww7AKGxAxpUYJJCE/AJl6+nrC2Aor3QdE+KNoLlfkQEgWh0c1ufaz7sD5H9oVE9Z6EWF8N1cVQXeK6L4aaZtttPVdTBhiwh0JIBAS7bs23g8MhJLLVc+EQHNlOu8ZtVzu7l1KO0wENteCohYa6Vve1bp6rBUddq/tmzzvqrf/mQWHWdxIU0uo+FOwhLe+DQtt+zh7qvc/eCT0nEtXr1TvqWZe3js+zrJ59dkU2ABMSJ3DL5Fs4M/VMRsSO8N6PWsZAVZErue+1Enzz7crDLdsHhUFHR0y0OEH0cb8d1sb+5o87cgJxOqCmtFXCLjly73a/K4m393nEDuGxEB5n3SL7QuJI1+NYEBvUV0FdlXUCqa+07usqrWOX51jb9dWudpVwvCN07CFHTgLB4a1OMq59xtlGYq6zPp+75O3JH4TtroTtbHB9nx4ahSS2ViePMDcnFDcnlrN+B30GeCYGF038qktKa0v5KvsrVmeu5qvsr6ioryDUHsrJA05myfglnJFyhmfHIDudVgIq2usmwe+D2rKW7fskQ9wQGDkf4odA/FDrcfwQCIuxkmxdBdSWH7nVlFnHab6vtrzZPtd9eU7L5zuSIIIjW54QwvoA0iyZl1hJv71jBUceSdbhcZA4HMJijyT05sm9+f7QaPDkSdcYK+nWVzU7YTSeLKqO7G96rp3tugrrL7C6SitBtu41R0S10ZNu1qNuM3mGtd9Tb/4ae3DL78gY1wmgjb8MOvtXRUNN28/Vlrc8vg7nVD1BZlkmqzJXsTprNRvyNuAwDhLCEpg/eD6zUmcxY8CMrg2BdNRDycEjJZnm5Zni/dY/iEa2IIgdZCX0lOnWffwQK7nHnWD1INtjs1sngLAujjBxOq2kV1Pm5kTRxomj8eZ0QEQiJAx3n7BbJ/KgkK7F6ikiEBxm3Yj3dTTeIWKdDOzBvo7EozTxq2NyOB1kFGQ01ev3lO4BYHjscK4edzWzUmcxLnHc8dXo6yqtJO6uJFOaaf253yg4wkrkiSNg5LwjPfb4odAnpWfUTm22I714pXq4HvAvRvVEVfVVfJPzDaszV/NF1hcU1RQRJEFM7T+VS0ddypkpZ5ISndLxAzqdcPBr2PIGfP8RlGW3fD48zkroKSfBhMtcyd3Ve49K8myJQqkAp4lfNcmrzGv6YXZNzhrqnHVEh0RzWvJpzE6dzanJp9InpE/HD2gMZG+wkv3WN62aeHAEjJgH/a9qWW8Pj/Pa51JKtaSJP4AZY9hZvJNVmav4PPNzthZuBawhl5eNuozZqbOZnDSZYNtx1jfztsGW162EX7zf+hFt+Nkw7mIYtcAavaGU8hlN/AGmzlHHutx1TT/O5lbmIggT+k7g1im3MitlFsNihx3/kMvCPVavPuMNyN9ujcwYciaccQeMPs/6gVIp1SNo4g8QFXUVPLjmQT49+ClVDVWE2cM4eeDJ3DjxRk5POZ3E8MTjP2hpNmx9y+rZH9pg7Rt0Mpz7CJx4EUR5dipZpZRnaOIPAOV15Vz/yfVsK9jGhcMvZHbqbGYMmNG5GRgrC2Db27DlTTjwNWBgwCQ4+w9WKSfmOH7wVUr5hCZ+P1daW8p1H1/HzuKdPDLrEc4adNbxH6SmFHakQcbrsHe1dZVk4iiYfQ+Mvdi6gEgp1Wto4vdjJTUlXPvxtewu2c3SWUs5M/XMjr+4rgq+/9Aq4+z62LpoKnYQnHorjLsEksbqEEuleilN/H6qqKaIJSuXsL90P4/PeZzTkk879osa6mDPp1ay3/G+dSVqVH+YdjWM/yEkT9Vkr5Qf0MTvhwqqC1iycgmZ5Zk8cdYTnDLwlLYbOx2w/0urjLP9XausEx4HEy61evYnnNp7ZqZUSnWIJn4/k1+Vz+KVi8mtzOVvZ/2NGQNmHN3I6YSstdZY+61vW7NWhkRZwy7HXQJDZ/Wc+WCUUh6nid+P5FXmcc3Ka8iryuPJs55kWv9prRpsg/SXrRE5pZnWbIQj51tlnBHzjj2hmVLKL3g18YvIrcASQIBlxpilIhIPvAIMBvYDlxljir0ZRyDIrczl6o+upqimiGfOfobJ/Sa3bLAjDV75sXVh1bA5MOd/YNS5rmmBlVKBxGuJX0TGYSX96UAd8KGIvAdcC3xqjHlIRO4G7gbu8lYcgSC7IpvFHy2mtLaUZ85+hol9J7ZscOBreP1qGDgZrnwVIjtxsZbyC/UOJzklNWQVV5FVXG3dl1STVVyN02lIiAohPjKUhMgQ4iNDSIgKISEytGk7LiKEkCAvr5SmvM6bPf4xwBpjTBWAiHwOXAxcCMxytXkBWI0m/k7LLM9k8UeLqaiv4Ll5zzE2cWzLBnnb4KVFEJMKV74GkQm+CVR1i7oGJzml1UeSevGR7ezianLLanA2W+PFJjAgJpzk2HCC7Tb2F1Sx/kAxRZV1Ldo11ycsiIQo62QQHxlCguuk0PyEER8ZQqKrjZ4oeh5vJv4twP+KSAJQDZwLrAOSjDE5rja5QJK7F4vItVh/HTBo0CAvhtl7HSg7wOKPFlPjqGH5vOWMSRjTskHJQfjXxdaMmD95U5O+H6htcHCopIbsFon9SILPK6/BuEnsKXHhnDwskZS4cJLjrMepcRH0jwkj2H50YnY6DaXV9RRW1lFYUUtRZR2FlXXWfUVt03ZmURUbD5ZQXFWHo40zRXRoEPFRIa6TguvkENX2CSMsWEeReZsY04Hl4jp7cJHFwI1AJbAVqAV+boyJbdam2BjT7py806ZNM+vWrfNanL3RvtJ9LP5oMQ3OBpbNW8ao+FEtG1QVwfJ5UHEYrv7AuuBKNTHGUF7bgABBNht2m2C3CTbBe2sCd0BNvYNDJdVNiTy7pGWv/XB5bYvEbrcJA2LCSIkLJyUuosV9cmx4m4nd05xOQ1lN44mijqJK18mhwjphWCeKWtdz1q2hjRNFsF0ID7YTERJEeIjdtW1v2g4PcT0ODiI8xGa1a7Y/zNX+yHZQi/3d8X30FCKy3hgzrfV+r/64a4xZDix3BfAgkAXkicgAY0yOiAwADrd3DHW0PSV7WPzRYgyG5fOXMyJuRMsGdZWw4lKrx/+TtzTpA9V1DtKzSthwsISNB4vZcLCEgopat20bTwJ2EYJsgs1m3dvd3eTItvs2NuyCdW9reZJpfH1VvYNsV6/9cHnLmIJswsBYq4d+5si+JMc2JvdwUuIjSIoOJagHJDKbTYiNCCE2IoRhHZibzxhDWXUDhZXWXxMFTSeEWipqHVTXNVBd76CqzkGN6768poH88lqq6prvb2izJNWWIJs0O3nYCXedGBpPHuHB1s1m6xkXK95+9gj6RXdiXq12eHtUTz9jzGERGYRV358JDAF+Bjzkun/HmzH4m++Lv2fJyiXYxMbz855naOzQlg0c9fDaz63ZMi97EQaf6pM4fckYw8GiKjYcLGbjwRI2HCxme055UyliSGIkZ4xMZHT/aATBYQwOp3VrcBqcrnuH04nDiXXfZptmt1ZtGuqdNDgdbbZpvIUG20iJC2f2qH6uhB7elOCT+oRh7yEJyJNEhJiIYGIighnahUlcjTHUOZxUu04G1fUOql33VXUtTyDVrltV/ZHtpufqG6iqa6Cgopaaemv/8Z5QvOXaM4aCh1f09PY4/jdcNf564CZjTImIPAS86ioDHQAu83IMfmNH0Q6WrFxCiD2E5fOWMzhmcMsGxsC7t8CulXDeUhhzni/C7HaVtQ1szipho6s3v/FgCYWVdQBEhtiZNCiWG84cxpQTYpmUGkd8pF6c5i9EhNAgO6FBdmIjfB1N7+HtUs/pbvYVAp2YIjKwbS3cyrUrryUiOILn5z1Pap/Uoxt9cj9s/jfMugemXdXtMXYHYwz7CiqbevIbD5awI7esqXc2tG8ks0f3Y8qgOCYPimVkUrRf9piV6gq9crcXyMjP4LqPryM6JJrl85e7X+T8myfhv0utCdXOvLPbY/SW8pp60rNK2XCg2Er0mSWUVNUD1miRSYNi+cWcEUweFMvk1FhiI7Q3r9SxaOLv4TYd3sQNn9xAbGgsy+cvZ2DUwKMbZbwOH/0GxpxvrX7VS2fQdDoNewsqXT15qze/M6+8aRTLiH5RzDsxiSmD4phyQhzD+kZpb16pTtDE34Otz1vPjZ/cSGJ4IsvnL6d/ZP+jG+35DN66Hk44DS5+rlfNpFlWU8+mZiWbTZkllFZbvfk+YUFMGhTHOeP6M2VQHBNTY4kJP85F35VSbmni76HW5q7lpk9vIikiieXzl9Mvot/RjbI3wCs/gb6j4Ip/Q7Bnh3x5mtNp+GZvIe+lH2Ld/mJ251dgjPUHysh+0Zw7vj+TU+OYckIsQxOjesxwOqX8jSb+HuibQ99wy2e3kByVzHPzn3O/EHrhHmusfkQ8/Oh1CIvp/kA76FBJNa+vz+K19ZlkFlUTHRrEtMFxnD9xoKs3H0N0mPbmleoumvh7mP9m/5dbV93KoD6DWHb2MhLC3UyzUJ4HL/4AMPDjt6DPgG6P81jqGpx8sj2PV9Zm8sWufIyBU4Yl8Ot5o5g/tr9elq+UD2ni70G+yPqC21bdxrDYYTx79rPEhbmZyaKmFP51CVQWwM/+0+MWOv8+r5xX1mby1sZsiirr6N8njF/MHs6lU1MZlKADrZXqCTTx9xCrDq7il5//kpFxI3n27GeJCXVTummohZd/BPnb4cpXIGVq9wfqRkVtA+9tPsTLazPZlFlCsF2YOyaJy05K5YwRfXXkjVI9jCb+HuCTA59wx+d3MCZhDE+f/TR9QtwsjuJ0wJtLrPVxL14Gw+d2f6DNGGNYf6CYV9ZmkpaRQ1WdgxH9ovifhWP4weRkEqJCfRqfUqptmvh97MP9H3L3F3czLnEcT819iugQN5NyGAMf3AXb3oF5/wsTfDfLRX55LW9uyOKVdZnsza8kMsTO+RMGcvn0VCanxvp0ZkulVMdo4vehtL1p3PPVPUzqO4kn5z5JZHCk+4ZfPgJrl8EpN8Mpv+jeIIEGh5PPv8/nlbWZfLbjMA1Ow9QT4nj4kmEsnDCAyFD930ip3kT/xfrIu3ve5bf//S1Tk6by1zl/JSK4jR8+178An/0RJiyCub/v1hj3F1Ty6rpM3tiQRV5ZLYlRIVx92hAum5bC8H4eni5QKdVtNPH7wFu73uJ3X/+O6QOm88ScJwgPCnffcEcavHebVc+/8K9g8/6869V1Dj7YksMrazNZs68Im8CsUf144IJUzhrTL6AWsVDKX2ni72av7nyVP3z7B04deCpLZy8lLKiNq20PfHNkgfRLXwC79y5wMsaQkV3KK2szeXfzIcprGjghIYI75o/ikikp9I/p2VcEK6WOjyb+bvTSjpd4cM2DnJFyBo/OepRQexsjX/K2wUuXH1kgPTTKK/GUVNXx9sZsXl6byY7cckKDbJw7fgCXTUtlxpB4nTJBKT+lib+bvLjtRR5e+zCzU2fzyJmPEGJvY/rgkkzrAq2gcK8skO50Gr7eU8gr6zL5aGsudQ1OxiX34Q8XjuWCSck6EZpSAUATfzf459Z/8ud1f2buoLk8fMbDBLdVtqkqgn9dbK2Ze9X7EDvIYzEYY3j2i728+O0BsoqriQkP5oqTUrnspFTGDuy58/wopTxPE7+X7S/dz6PrH+WsQWfx8JkPE2xrI+nXVcK/L4PiA9YC6f3HeTSO5/+7n//7YAczh8Zzx3ydL0epQKaJ38se3/g4ofZQfjvzt20nfUc9vHYVZK+3fsj18ALpW7JL+dMHO5g7ph/LfjpNL7JSKsDp2DwvSs9P5+MDH/PzsT93P8smNFsg/SNY+P/gxAs8GkNVXQO3vLyR2IhgHv7hRE36Sint8XuLMYbH1j9GfFg8Px3707YbfvqAa4H031jr5XrY7/+zjX0FlaxYPIP4SF2PVimlPX6v+TL7S9blreP6ide3PRXDt0/BV4+5Fki/y+MxpKXn8PLaTK4/cxinDHezmItSKiBp4vcCh9PB0g1LGRQ9iB+O/KH7Rhmvw4d3e22B9KziKu5+M52JqbH88uyRHj22Uqp308TvBWn70thVvIubp9zs/gfdpgXST/XKAukNDie3vbwJY+DxRZN0mgWlVAta4/ewWkctf934V8YmjGXeCfOObnBoo7VAeuJIWOSdBdKf+Gw36w4Us/TySZyQ0EaZSSkVsLQr6GEv73iZnMocbp96OzZp9fUW7oF//RDC4+HHb0B4rMff/7t9RTzx2S4unpzMRZOTPX58pVTvp4nfg8rqyng2/VlOTT6VGQNmtHyy+QLpP/HOAumlVfXc9vJGBsVH8PuLPHsBmFLKf2ipx4OWZyynvK6c26fc3vKJmjJYcQlU5sPP3vPKAunGGO5+M53D5bW8ccMpROniKEqpNmiP30NyK3NZsX0FC4cuZFT8qJZPvrEYDm+Hy1702gLpL6/N5IMtufx6/igmpsZ65T2UUv5BE7+HPLX5KZzGyS8mt1oaMWs97FoJZ90HI7yzQPquvHIe+M9WThueyLWnD/XKeyil/Icmfg/YU7KHt3e/zaLRi0iOavWD6pqnICQapl7llfeuqXdw80sbiQgJ4tHLJuoc+kqpY9LE7wFLNywlIiiCJeOXtHyiLAe2vgWTfwxhfbzy3g99sIMdueU8cukE+vXRlbKUUsemib+LNuRtYHXmaq4edzVxYXEtn1z3PDgdMONar7z3Zzvy+MfX+/n5KYOZMzrJK++hlPI/mvi7oHEitr7hffnRmB+1fLK+xkr8I8+BeM/X3Q+X1fDr19IZM6APdy8Y7fHjK6X8lyb+LliVuYpN+Zu4cdKNRARHtHxyyxtQVQAzrvP4+zqdhl++upmqugaeuGKSLqiilDoumvg7qcHZwNINSxncZzAXDb+o5ZPGWD/q9h0DQ2d5/L2f/XIvX+0u4Hfnj2V4v2iPH18p5d808XfSO7vfYV/pPm6bchtBtlYXSx34GnIzrN6+h2fd3JxZwiMf7WTBuP4sOinVo8dWSgUGryZ+EbldRLaKyBYReUlEwkRkiIisEZHdIvKKiPS61UGqG6p5ctOTTOw7kTmD5hzdYM1TEB4HEy736PtW1FqrafWLDuWhiyfoalpKqU7xWuIXkWTgFmCaMWYcYAcWAX8CHjPGDAeKgcXeisFbVmxfweHqw9w+9fajk2/xAdiRBlN+BiER7g/QSfe9vYXMoiqWLppMTEQb6/cqpdQxeLvUEwSEi0gQEAHkAHOA113PvwBc5OUYPKqkpoTnM55nVsospia5mX5h7TJAYPqSo5/rgrc3ZvPmxmxunjOC6UPiPXpspVRg8VriN8ZkA48AB7ESfimwHigxxjS4mmUBvWru4GUZy6hsqOSWKbcc/WRdJWz4p7WqVkyKx97zYGEV//P2FqadEMfNczw/wZtSKrB4s9QTB1wIDAEGApHAOcfx+mtFZJ2IrMvPz/dSlMfnUMUhXtrxEhcMu4ARcSOObrD5JagphZk3eOw96x1Obn55IzaBpYsmEaSraSmlusibWWQusM8Yk2+MqQfeBE4FYl2lH4AUINvdi40xzxpjphljpvXt29eLYXbc3zb9DZvYuGnSTUc/6XTCmmdgwCRInXH085306MffszmzhIcumUBKnGd/M1BKBSZvJv6DwEwRiRDrF9CzgG3AKqBxBfKfAe94MQaP2Vm0k//s+Q9XjrmS/pH9j26w9zMo+N7q7XtotM3Xuwt4+vM9LDoplXPHe37hFqVUYPJmjX8N1o+4G4AM13s9C9wF/FJEdgMJwHJvxeBJSzcsJSokisXj2hiEtOYZiOwHY3/gkfcrqqzjtlc2MTQxkvvOP9Ejx1RKKfDyClzGmN8Bv2u1ey8w3Zvv62nf5XzHV9lf8cupvyQmNOboBgW7rTn3Z/0GgkK7/H7GGO58fTMlVfX8/aqTiAjR1bSUUp6jvxQeQ+NEbP0j+3PlmCvdN/ruGbCHwLSrPfKe//zmAJ9sP8zdC0YzdqCbE41SSnWBJv5jWHlgJVsKt3DTpJsItbvpzdeUwqZ/w7hLIKpfl99ve04Z//v+dmaP6stVpw7u8vGUUqo1TfztqHfW8/iGxxkeO5zzh57vvtHGf0FdhUdm4ayuc3DLSxvpExbMny+dqFMyKKW8QhN/O978/k0Olh/ktim3Ybe5mfrY6bB+1E2dCQMnd/n9/pi2jV2HK3js8okkRnX9twKllHJHE38bquqreGrzU0xNmsoZKWe4b/T9h1ByAGZe3+X3+3BLDivWHOS6M4Zy+oiecd2CUso/aeJvwwvbXqCwptD9RGyNvn0K+qTA6DbKQB10qKSau97IYEJKDL+aN6pLx1JKqWPRxO9GYXUh/9jyD+YOmsvEvhPdN8rdAvu/hOnXgL3zwy0dTsPtr2yi3uHkL4smExKk/0mUUt6lA8TdeDb9WWodte4nYmu05mkICremX+6CJ1ftZs2+Iv7fpRMZkhjZpWMppVRHHFf3UkSGich4bwXTE2SWZfLq969y8YiLGRIzxH2jykLIeA0mXg4RnZ8ief2BIpZ+uosLJw3k4im9apJSpVQv1uEev4jcAwwHnCISaoz5iffC8p0nNj5BsC2YGya2M8Pm+r9DQw3M6PyPuqXV9dzy0iYGxobxx4vG6dBNpVS3abPHLyK3iEjzMYwTjTFXG2OuAdoofPduWwu38sH+D/jxmB/TN6KNkTWOeli73FpEvd+YTr2PMYZ738ogr6yGxxdNJjpMV9NSSnWf9ko9hcCHInKB6/FKEflQRFYCH3k/tO63dP1SYkNjuXpcO1MvbH8Xyg/BjM7Puf/a+izeS8/h9rNHMnlQXKePo5RSndFm4jfGrADOByaIyLtYq2ddDFxqjLmjm+LrNl9nf823Od9y3YTriAqJarvht09D/FAYMa9T77Mnv4L7393KyUMTuP7MYZ2MVimlOu9YP+4OA14FrgVuAv4ChHs7qO7mNE4e2/AYyVHJXDbqsrYbZq+HrO9g+nVgO/5hl7UN1pQMoUE2Hrt8Enab1vWVUt2vzR93ReQfQD3WIunZxpglIjIZWCYia40xv++mGL3ug30fsKNoB/93+v8RYg9pu+G3T0NINExqY5bOY/jzhzvZeqiMZT+dRv+YsE5Gq5RSXdPeqJ7JxpiJACKyEcAYsxE4X0Qu7I7gukOdo44nNj7B6PjRnDvk3LYblufC1rfgpGsgrM9xv8/qnYd57qt9/PTkEzj7xKQuRKyUUl3TXuL/QEQ+AoKBfzd/whjTK5ZL7IhXd75KdkU2z8x9Bpu0U75ZuxycDTB9yXG/R355Lb9+bTOjkqK559zOjQRSSilPaTPxG2PuFpE+gNMYU9GNMXWb8rpynkl/hhkDZnDywJPbblhfA+ueh5HzIeH4f5D9zZvplNc08O8lMwkLdjPLp1JKdaN2L+AyxpR1VyC+8I+t/6CktqT9idgAtrwBVQWdumAru6SaT7Yf5tazRjAyKboL0SqllGcE7Ixg+VX5vLjtRRYMXsDYhLFtNzQG1jwFfcdYF20dpw8ycgC4aLJOyaCU6hmOmfhF5KgVQdzt622e2vwU9Y56bp58c/sND3wNuRnWCludmFYhLSOHsQP76ARsSqkeoyM9/m86uK/X2Fe6jzd3vcmloy4ltU9q+43XPAXhcTDh8uN+n6ziKjYeLGHhhAGdjFQppTyvvXH8/YFkINw1fr+xu9sHa2x/r/XExicItYdy3YRjrJNbfAB2pMEpt0DI8X/kDzJyAVg4XhO/UqrnaO/H3fnAz4EU4NFm+8uBe7wYk1dtzt/Mxwc+5sZJN5IQntB+47XPAdKpIZwA72XkMC65DyckaJlHKdVztDec8wXgBRG5xBjzRjfG5DXGGB5d9yjxYfH87MRjLKBSVwkbXoAx50NMynG/V2ZRFZszS7jrnNGdjFYppbyjvVLPj40x/wIGi8gvWz9vjHnUzct6tC+zv2TD4Q3cO+NeIoKPUbrZ/DLUlMLMzs3C+cEWazSPlnmUUj1Ne6WexvpEO1NV9h4Op4PH1j/GoOhBXDLykvYbGwNrnoEBkyB1RqfeLy09hwkpMQxK6NU/hyil/FB7pZ5nXPcPdF843vOfvf9hd8luHjnzEYJtx1j4ZM9nULATfvBMp4ZwZhZVsTmrlLsXaJlHKdXztFfqeby9Fxpj2lmJvGepaajhrxv/yriEccw7oQPz6K95GiL7wdgfdOr90jK0zKOU6rnaK/Wsb7b9APA7L8fiNS/veJm8qjwePO3BY69tW7Abdq2EWb+BoM5dp/Z+Rg4TU2JIjdcyj1Kq5znWqB4AROS25o97k9LaUpZlLOPU5FOZPmD6sV/w3TNgC4Zp7Sy/2I6DhVWkZ5Vyz7la5lFK9UwdnavHeDUKL1q+ZTnldeXcPuX2YzeuKYVN/4Zxl0BUv069X2OZZ8E4LfMopXomv56kLbcylxXbVnDe0PMYFT/q2C/Y+C+oq4CZxz8LZ6O0jENMTI3VMo9Sqsdq78fdco709CNEpHGKZgGMMeb4l6HqZk9uehKD4abJNx27sdNhDeFMnQkDJ3fq/Q4UVrIlu4x7dbEVpVQP1l6Nv9dPHn9iwokkRyWTHNWBKZG//xBKDsDZnR+92lTmGd+/08dQSilva3chlt5u0ehFHW/87VPQJwVGn9/p90tLz2FSaiwpcVrmUUr1XH5d4++wvK2w/0uYfg3YO3cu3F9QydZDZZynUzArpXo4TfxgXbAVFA5TjjFxWzuOlHk08SulejavJX4RGSUim5rdykTkNhGJF5GPRWSX6z7OWzF0SGUhpL8KEy+HiPhOHyYtPYcpg2JJjg33YHBKKeV5Xkv8xpidxphJxphJwFSgCngLuBv41BgzAvjU9dh3NvwDGmo6tZB6o735FWzLKWPhhIGei0sppbyku0o9ZwF7jDEHgAuBxquAXwAu6qYYjuaoh++esxZR79f5IZjvu8o85+poHqVUL9BdiX8R8JJrO8kYk+PazgWS3L1ARK4VkXUisi4/P987UW1/F8oPwYzOzbnfKC0jl6knxDEgRss8Sqmez+uJX0RCgAuA11o/Z4wxtDEdhDHmWWPMNGPMtL59+3onuG+fhvihMKIDM3a2YU9+BdtzynQmTqVUr9EdPf4FwAZjTJ7rcZ6IDABw3R/uhhiOlr0esr6D6deBrfNfw/vpjWUeTfxKqd6hOxL/FRwp8wC8CzSOm/wZ8E43xHC0b5+GkGiYdGWXDpOWkcO0E+LoHxPmocCUUsq7vJr4RSQSOBt4s9nuh4CzRWQXMNf1uHuV58LWt2DyjyCs81MO7T5cwY7cchbqRVtKqV7Eq1M2GGMqgYRW+wqxRvn4ztrl4GyA6dd26TDvZ+QgolMwK6V6l8C7cre+BtY9DyPnQ8KwLh0qLT2Hk06I1zKPUqpXCbzEv+UNqCro0gVbALvyytmZV65j95VSvU5gJX5jrHl5+o6xLtrqgrTGMo+O5lFK9TKBlfgPfgO56TDjOjjWouvH8H5GDicNjiepj5Z5lFK9S2Al/m+fgvA4mHB5lw7zfV453+dV6BTMSqleKXASf8lB2PGeNfVySNcWSklLt8o854zT+r5SqvcJnMT/3TJAYPqSLh8qLSOH6YPj6RetZR6lVO8TGIm/rhI2vABjzoeYlC4d6vu8cnYf1jKPUqr3CozEv/llqCmFmV2bhRPgvfQcbALztcyjlOql/D/xGwNrnoEBkyB1RhcPZUhLP8SMIQla5lFK9Vr+n/j3fAYFO63efheHcO7MK2dPfiXnaplHKdWL+X/iX/M0RPaDsT/o8qHed5V5zhmrZR6lVO/l34m/YDfsWgnTroag0C4dyhjDexk5zByaQN/orh1LKaV8yb8T/3fPgC3YSvxdtCO3nL35lToFs1Kq1/PvxB8eB9Ougmi3y/oel7TG0Txa5lFK9XJenY/f52bf45HDGGNIy8jh5GEJJEZpmUcp1bv5d4/fQ7bnlLOvoJKF4wf6OhSllOoyTfwdkJZxCLtNmD+26yUjpZTyNU38x2BdtJXDyUMTSNAyj1LKD2jiP4ath8rYX1ilo3mUUn5DE/8xvJ+R4yrz6GgepZR/0MTfjsbRPKcMSyA+MsTX4SillEdo4m/H1kNlHCis0imYlVJ+RRN/O95Lt8o8807UMo9Syn9o4m+DMYb3M3I4dXgicVrmUUr5EU38bdiSXcbBoirOG69lHqWUf9HE34b3Mg4RZBPm6UVbSik/o4nfjcaLtk4dnkhshJZ5lFL+RRO/GxnZpWQVV+tFW0opv+Tfs3N2Ulp6DsF2Yb6O5lGqx6ivrycrK4uamhpfh9LjhIWFkZKSQnBwcIfaa+JvxRjDe+k5nDY8kZiIjn2JSinvy8rKIjo6msGDByNdXD/bnxhjKCwsJCsriyFDhnToNVrqaWVzVinZJdWcq6N5lOpRampqSEhI0KTfioiQkJBwXH8JaeJv5f0Mq8yjF20p1fNo0nfveL8XTfzNNI7mOX1EXy3zKKX8lib+ZjZllpBdUs1CLfMopdzYv38/48aNO2r/fffdxyeffALA0qVLqaqqavF8WVkZv/3tb5k8eTKTJ09m0aJFbN26tUWbe++9l9TUVKKiorz3AVw08TeTlp5DiN3G3BP1oi2lVMf9/ve/Z+7cucDRib+oqIi5c+eSnJzM119/zcaNG7njjju45ppr+Pbbb5vanX/++Xz33XfdEq+O6nFxOq25eU4fkUhMuJZ5lOrJHvjPVrYdKvPoMU8c2IffnT/2mO0cDgdLlizh66+/Jjk5mXfeeYcbbriB8847j0OHDnHo0CFmz55NYmIiq1at4le/+hUPPPAACxYsaDrG1KlTeffdd7nkkkv44osvAJg5c6ZHP097tMfvsimrhEOlNXrRllKqXbt27eKmm25i69atxMbG8sYbbzQ9d8sttzBw4EBWrVrFqlWrqKioYN++fSxYsIA1a9Zw0kknsWDBAq6++mpqamqYMmUKGzZs6PbP4NUev4jEAs8B4wADXA3sBF4BBgP7gcuMMcXejKMjtMyjVO/RkZ65twwZMoRJkyYBVs99//79bbbdvn07U6dOBeDOO+/kjTfeICoqiilTpnDfffcxatQo9uzZw5QpU7oh8iO83eP/C/ChMWY0MBHYDtwNfGqMGQF86nrsU41lnjNG9qVPmJZ5lFJtCw0Nbdq22+00NDS0295utwNgs9kYNGgQ8fHxzJgxA4DDhw/Tr18/7wXbBq8lfhGJAc4AlgMYY+qMMSXAhcALrmYvABd5K4aO2phZTE5pDQsn6Nh9pVTXREdHU15eDsDo0aObSjkOh4OsrCxKSkpYs2YNWVlZrF69mpNPPrnbY/Rmj38IkA/8XUQ2ishzIhIJJBljclxtcgG3tRURuVZE1onIuvz8fC+GCWnpuYQE2Zg7Rss8SqmuufbaaznnnHOYPXs20dHR9OvXj08//ZQ//elP/OAHP+CKK65gwYIFPPbYYyxbtoyQEGsG4DvvvJOUlBSqqqpISUnh/vvv91qMYozxzoFFpgHfAqcaY9aIyF+AMuBmY0xss3bFxpi49o41bdo0s27dOq/E6XQaTnnoM8anxLDsp9O88h5Kqa7bvn07Y8aM8XUYxy0vL4+FCxdy5513cvHFFxMUFMSOHTvYuHEjV1xxhcfex933IyLrjTFHJTZv9vizgCxjzBrX49eBKUCeiAxwBTUAOOzFGI5pw8FicstqdEF1pZRXJCUlsXLlStauXcuMGTMYP348999/v9sLwbqL10b1GGNyRSRTREYZY3YCZwHbXLefAQ+57t/xVgwd8V56DiFBNs7SMo9Sykvi4+P585//7Oswmnj7Aq6bgRUiEgLsBa7C+ivjVRFZDBwALvNyDG1yOg0fbMlh1si+RIXqtWxKqcDg1WxnjNkEuCucn+XN9+2o9QeLySur1Yu2lFIBJaCv3E1LzyFUyzxKqQATsIm/8aKtWaO0zKOUCiwBm/jXHSjmcHktCycM9HUoSqlewpvTMs+aNYtRo0YxadIkJk2axOHD3hvwGLCJPy39kFXmGd39l0srpfyLJ6ZlBlixYgWbNm1i06ZNXp3KISBrHA6n4f0tucwZ3Y9ILfMo1ft8cDfkZnj2mP3Hw4KHjtnMW9Myd6eA7PGv3V9EfrmO5lFKHT9vTst81VVXMWnSJP7whz/grVkVIEB7/GnpOYQF25ijZR6leqcO9My9xVvTMq9YsYLk5GTKy8u55JJLePHFF/npT3/qlc8QcD1+h9PwgavMExESkOc9pVQXeGta5uTkZMCa3fPKK6/06jKMAZf4v9tXREFFLQvH62gepZTndWZa5oaGBgoKCgCor6/nvffe8+pcPgHX5U3LOER4sJ3Zo/v6OhSllB9qnJa5sdbfelrmxMTEo6ZlrqysZP78+dTX1+NwOJg7dy5LlizxWoxem5bZkzw1LbPDaZjx4CfMGJLA337UvUudKaW6Rqdlbl9PmZa5x1mzr5CCijodzaOU6jYBNS1zT5SWnmOVeUbpaB6lVPfpadMyB0yPv8Hh5MMtuZw1ph/hIXZfh6OUUj4TMIl/zb4iCivrWDheyzxKqcAWMIk/LSOHiBA7s7TMo5QKcAGR+I+UeZK0zKOUCngBkfi/3VtEkZZ5lFJd5M1pme+9915SU1OJiopqsb+2tpbLL7+c4cOHM2PGjHaniOiogEj8aRmHiAyxM2uUXrSllPI8T0zLfP7557udpmH58uXExcWxe/dubr/9du66664ux+v3wzmbl3nCgrXMo5Q/+NN3f2JH0Q6PHnN0/Gjumn7spOqtaZlnzpzp9v3eeecd7r//fgB++MMf8otf/AJjDCLS6c/q9z3+b/YWUlxVrxdtKaU8wpvTMruTnZ1NamoqAEFBQcTExFBYWNilz+D3Pf609BwiQ+ycOVLLPEr5i470zL3FW9Mydye/7vHXO5x8uDWXs0/UMo9SyjO8NS1zW5KTk8nMzASgoaGB0tJSEhISuvIR/Dvxf7OnkJKqes7V0TxKqW7SmWmZ23PBBRfwwgsvAPD6668zZ86cLtX3wc8Tf1p6DlGhQZyhZR6lVDdpnJZ59uzZREdHHzUt8xVXXHHUtMxglYJSUlKoqqoiJSWl6QfdxYsXU1hYyPDhw3n00Ud56KGurz7m19MyP7l6N+U1Ddx1zmgvRKWU6k46LXP7dFpmlxtnDdekr5TyKZ2WWSmlApBOy6yUUp3UG0rTvnC834smfqVUrxAWFkZhYaEm/1aMMRQWFhIWFtbh12ipRynVK6SkpJCVlUV+fr6vQ+lxwsLCSElJ6XB7TfxKqV4hODiYIUOG+DoMv6ClHqWUCjCa+JVSKsBo4ldKqQDTK67cFZF84EAnX54IFHgwnN5Ov48j9LtoSb+Plvzh+zjBGHPUnDW9IvF3hYisc3fJcqDS7+MI/S5a0u+jJX/+PrTUo5RSAUYTv1JKBZhASPzP+jqAHka/jyP0u2hJv4+W/Pb78Psav1JKqZYCocevlFKqGb9O/CJyjojsFJHdInK3r+PxFRFJFZFVIrJNRLaKyK2+jqknEBG7iGwUkfd8HYuviUisiLwuIjtEZLuItL8eoB8Tkdtd/062iMhLItLx2c96Cb9N/CJiB/4GLABOBK4QkRN9G5XPNAC/MsacCMwEbgrg76K5W4Htvg6ih/gL8KExZjQwkQD9XkQkGbgFmGaMGQfYgUW+jcrz/DbxA9OB3caYvcaYOuBl4EIfx+QTxpgcY8wG13Y51j/qZN9G5VsikgIsBJ7zdSy+JiIxwBnAcgBjTJ0xpsSnQflWEBAuIkFABHDIx/F4nD8n/mQgs9njLAI82QGIyGBgMrDGx6H42lLgTsDp4zh6giFAPvB3V+nrORGJ9HVQvmCMyQYeAQ4COUCpMWalb6PyPH9O/KoVEYkC3gBuM8aU+ToeXxGR84DDxpj1vo6lhwgCpgBPGWMmA5VAQP4mJiJxWJWBIcBAIFJEfuzbqDzPnxN/NpDa7HGKa19AEpFgrKS/whjzpq/j8bFTgQtEZD9WCXCOiPzLtyH5VBaQZYxp/CvwdawTQSCaC+wzxuQbY+qBN4FTfByTx/lz4l8LjBCRISISgvUDzbs+jsknRESw6rfbjTGP+joeXzPG/MYYk2KMGYz1/8Vnxhi/69V1lDEmF8gUkVGuXWcB23wYki8dBGaKSITr381Z+OEP3X67ApcxpkFEfgF8hPXL/PPGmK0+DstXTgV+AmSIyCbXvnuMMe/7LiTVw9wMrHB1kvYCV/k4Hp8wxqwRkdeBDVij4Tbih1fw6pW7SikVYPy51KOUUsoNTfxKKRVgNPErpVSA0cSvlFIBRhO/UkoFGE38KmCJiENENjW7eexqVREZLCJbPHU8pTzJb8fxK9UB1caYSb4OQqnupj1+pVoRkf0i8rCIZIjIdyIy3LV/sIh8JiLpIvKpiAxy7U8SkbdEZLPr1niJv11Elrnmdl8pIuGu9re41kZIF5GXffQxVQDTxK8CWXirUs/lzZ4rNcaMB/6KNZMnwBPAC8aYCcAK4HHX/seBz40xE7HmuGm8QnwE8DdjzFigBLjEtf9uYLLrONd756Mp1Ta9clcFLBGpMMZEudm/H5hjjNnrmtwu1xiTICIFwABjTL1rf44xJlFE8oEUY0xts2MMBj42xoxwPb4LCDbG/FFEPgQqgLeBt40xFV7+qEq1oD1+pdwzbWwfj9pm2w6O/Ka2EGt1uCnAWteCH0p1G038Srl3ebP7b1zbX3NkGb4fAV+6tj8FboCmdXxj2jqoiNiAVGPMKuAuIAY46q8OpbxJexoqkIU3m60UrDVnG4d0xolIOlav/QrXvpuxVqm6A2vFqsYZLG8FnhWRxVg9+xuwVm9yxw78y3VyEODxAF/mUPmA1viVasVV459mjCnwdSxKeYOWepRSKsBoj18ppQKM9viVUirAaOJXSqkAo4lfKaUCjCZ+pZQKMJr4lVIqwGjiV0qpAPP/ATLgVcRF4YvIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "plt.plot(np.array(accs)[:, 0], label='hit@1')\n",
    "plt.plot(np.array(accs)[:, 1], label='hit@5')\n",
    "plt.plot(np.array(accs)[:, 2], label='hit@10')\n",
    "fig.suptitle('HIT@K')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Hit %')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "9c968086",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Hit %')"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEjCAYAAAAomJYLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlzUlEQVR4nO3de3xUd53/8dcnkxsJBAgJLYUAoUBv1kJNoUq3N3vhp664dVV069p1ldW13te1Vn+21uvPx67rpfVS3brdXWt/3lr5abcXm9BWWyvBXig0gRBoSWiZkAAhQK7z+f0xBxjSAQLk5Mzl/Xw85jHnfM85M58MZN453/M955i7IyIiMlxB1AWIiEhmUkCIiEhaCggREUlLASEiImkpIEREJC0FhIiIpKWAEBGRtBQQIiKSlgJCBDCzLWZ2xbC268zs96nLzexGM+sJHr1mNpQyvy5lWzOzd5jZw2YWN7OXzexBM7smzXu7mc1Nmf8nM3vJzM4J82cWORYFhMhxcPevuPt4dx8PfAB44sC8u58DYGYx4C7g/cBXgDlADXAzsMLMfmhmlu71zexzwMeAS9x9Xbp1RMaKAkJk9H0W6AOudPeH3b3H3Qfc/XHgDUAF8O7hG5nZl4D3ARe7+4YxrVgkjcKoCxDJJWZWTvJL/iygyMy+D7wJWA9sBn4MfAL4NfCfKZt+DTiPZDi8OKZFixyBAkLkkHvNbDBlvhj483G+xmuBBnffa2YfBGYAc4Eq4PfAne7ebmZThm13VbBM4SAZQ11MIoe8xd0nHXgA/3gCrzEVaA+mzwXudfdud28lGRCY2QRg77DtlgN/bWZfOLHSRUafAkJkdO0ApgXTa4G3mNkEM6sFLgImA98F7hi23QbgCuAfzeyGsSpW5GgUECKj6wngMjMbB/w70Aa0BNMPAp8P1vnm8A2DUUtXAJ8ys4+NUb0iR6RjECKjyN33mNldwDfd/R+A96YuN7NCdx9MvzW4+zNmdjXwkJn1uvv3Qy5Z5IhMd5QTGV1mVgj8nOQe+peAp4EyYBnwz8CF7t4TWYEiI6SAEAmBmRUA15HcgzgL6AcagK+4+3MRliYyYgoIERFJSwepRUQkLQWEiIikpYAQEZG0FBAiIpKWAkJERNJSQIiISFoKCBERSUsBISIiaSkgREQkLQWEiIikpYAQEZG0FBAiIpKWAkJERNJSQIiISFo5c0e5qqoqnz17dtRliIhklTVr1uxw9+p0y3ImIGbPnk1jY2PUZYiIZBUze+FIy9TFJCIiaSkgREQkLQWEiIikpYAQEZG0FBAiIpKWAkJERNJSQIiISFo5cx6EiIyuoYTT2dPH9u4+tnf3sn1PL509/ZSXFDKlvJjK4DFlfPK5pDAWdckyyhQQInnG3dm1b4Dte3p5eXcv8ZQA2N7dR7y7l5e7e+nY00fCR/6640sKD4XGgQAZf2C65BWhUlasr59Mp38hkRzS0zcYfOkf+sLf3t0bPJLT8e4++ocSr9h2UlkRp1aUMrWilPmnTOCUilJOqSgJnpOPyvJi9vUP0rm3n669/XT2JJ+79vYdbOva289Lu3tZt62brr39ad8LoLSogCnlJUcNlYPt44uZUFKImYX9EUoKBYRIGt29AzRu6eLJzV109fRTGDNiBUZhQQGFBUYsZhQVFBArMIpiRixoL4xZcnlBQcq0URQrCLY3CmMFKe2v3LYweN3kfHJZQYGxO/irP/XL/sDjwF7A3v6hV/ws5cUxTplYyikTSqmbNZlTghA4NSUAqieUUFo0si6i4sJiJpUVc3raq/cczt3Z0zdIV09/SoAEYRKEy4H2lngPnXv76B1IHyjFsQImlxdRWV7C5LIiYsFnGDM7OF2QMl9gRqyAlOlDz6nbHtqGNNsfvjxWUJBczw79fxhXHGNcUfJRWlRAaVHsYFtpUYxYQfaGmgJCBNi1r58/bU4GwpObO1m/rZuEQ1HMqBpfwmDCGUo4A0MJhhLO4JAzmEgcVxfMaCsuLEh+wU8o5axpFVxyRnXwpV/K1JS//MeXRPdrbmZUlBZRUVrE7KryEW2zr38wZc/kQIAcHiq79w8c/DcZSjgJD6bdSRx85hVtQ2m2CfvfsLiwgNLCgsNC47Dpg20FB4OmJHgevs3RXieMIFJASF7q2tvPnzZ38sfWZCg0vdyNe/KXeWHNJD58+TwWz6nk/JmTj/qXdSLhB7+oBhOJIDiGhckrlh2aPjA/MOSH1h1KBM/BusHrVYwrOtTtM6GUSWVFOdnlUlZcSFllITWVZWPyfu7JkEgbNClhM5hIJEPHDwXMwFCC3oEEvQND7O8fYv9A8tEbPPb3Jw7OH1jem7JOd+8A+/uH6B1IHLbMjzO0zpsxkV9ff9GofzYKCMkLHXv6eHJzJ0+2JvcQNmzvAZL94K+ZNZmPXzGfxbWVnFczacRdLZDskig++JebRvFkIzMjZmRMV5C70zeYOBgWrwiQlCDqC54nlxWHUosCQnLS9u5e/tjamewyau1kU8deAMqKY7xm1mSWLZjOhXMqOXf6JIoLdTqQZA4zozToNpoUcS0KCMkJ23btT9lD6GLzjmQgjC8p5ILZk3lbXQ2Layt51fSJFMUUCCIjoYCQrLS1a9+hPYTNnWzt2g9ARWkhi2oredeimSyeU8nZ0yooVCCInJBQA8LMlgLfItk5+yN3/9qw5bOAO4BqoAu41t3bgmXvAT4XrPold78zzFolc7k7L3TuO2wPoX1XMhAmlRWxaHYlf/e6WhbPqeTMUysypi9ZJNuFFhBmFgNuA64E2oDVZrbS3denrPYvwH+6+51mdjnwVeDdZlYJ3ATUAQ6sCbbdGVa9klnie3r53fr4wVB4ubsXgCnlxSyeU8mKi+eweE4l86dOoECBIBKKMPcgFgEt7t4KYGZ3A8uA1IA4G/hEMN0A3BtMXw085O5dwbYPAUuBn4ZYr2SAbbv28/1HNnH36q30DyaonlDC4tpKFs+ZwoW1lcydOj4nh3aKZKIwA2I6sDVlvg1YPGydZ4BrSHZD/RUwwcymHGHb6eGVKlHb2rWP765q4Rdr2nCHv37NDN57US3zFAgikYn6IPU/Abea2XXAo0A78MprBRyBma0AVgDMnDkzjPokZK0dPdzWsIl7n24nZsbyC2bygUtPZ/qkcVGXJpL3wgyIdqAmZX5G0HaQu28juQeBmY0H3uruu8ysHbh02Larhr+Bu98O3A5QV1cX4UUP5Hht2L6HW+tb+M2z2yiKFfC3r53FP1x8OqdOLI26NBEJhBkQq4F5ZlZLMhiWA+9KXcHMqoAud08AnyE5ogngAeArZjY5mL8qWC5Zbv22bm5t2Mj/PPcy44pivP/iObzvojlUTyiJujQRGSa0gHD3QTO7nuSXfQy4w93XmdktQKO7ryS5l/BVM3OSXUwfCrbtMrMvkgwZgFsOHLCW7PTM1l18p76F3z2/nQklhVx/2Vzeu6SWyeXhXCJARE6e+fFeFSpD1dXVeWNjY9RlyDCNW7r4dn0Lj27oYOK4It67pJbrlsxm4riiqEsTEcDM1rh7XbplUR+klhzk7vyxtYtvP7yRJ1o7mVJezKeXnsm1F85kQqmCQSRbKCBk1Lg7j27cwa31G1m9ZSfVE0r43BvP4l2LZ+r2kiJZSL+1ctLcnYefj/Odhhae2bqLaRNLuWXZOby9rua4Lp0tIplFASEnLJFwHlj3Mt+pb2H9S93UVI7jq9ecyzXnT6ekUMEgku0UEHLchhLOb57dxm0NLWzY3kNtVTn/8rbzWLbgNF1KWySHKCBkxAaHEtz79Da+29BC6469zJs6nm8tX8CbXn2arqAqkoMUEHJM/YMJfvnnNr67qoWtXfs5a1oF3/ub87n6nFN1JVWRHKaAkCPqHRji541b+d6qTWzb3ct5MyZy05vO4fVnTdUF9ETygAJCXmF//xA/efIFbn+0lfiePupmTearb301F8+rUjCI5BEFhBzmydZOPnTXn9nR089r50zhm8sX8No5UxQMInlIASEH7esf5JM/f4bxJYV8/9rXUDe7MuqSRCRCCgg56FsPb6Rt537+74oLFQ4iggatC5C8DPePHtvMO+pqWDxnStTliEgGUEAIQwnnxnvWMmlcEZ95w5lRlyMiGUIBIfzkyRd4eusu/vebzmZSme7PICJJCog8t727l6/f38xfzKti2YLToi5HRDKIAiLP3bxyHQNDCb70lldpKKuIHEYBkcd+t347//Pcy3zk9fOYNaU86nJEJMMoIPLU3r5Bblq5jjNOmcCKi+dEXY6IZCCdB5Gn/u2hDbTv2s8vP/haXaJbRNLSN0Meeq59N3f8YTN/s3gmr5mlE+JEJL1QA8LMlppZs5m1mNkNaZbPNLMGM3vKzJ41szcE7bPNbL+ZPR08vh9mnflkKOF85ldrmTK+hH9eqnMeROTIQutiMrMYcBtwJdAGrDazle6+PmW1zwE/c/fvmdnZwH3A7GDZJndfEFZ9+erOx7ewtn03t75rIRPHFUVdjohksDD3IBYBLe7e6u79wN3AsmHrOFARTE8EtoVYT97btms///pgM5eeUc0bz50WdTkikuHCDIjpwNaU+bagLdXNwLVm1kZy7+HDKctqg66nR8zsL0KsM2/ctHIdQ+58cZnOeRCRY4v6IPU7gf9w9xnAG4D/MrMC4CVgprsvBD4B3GVmFcM3NrMVZtZoZo0dHR1jWni2uf+5l3lo/XY+fsV8airLoi5HRLJAmAHRDtSkzM8I2lL9PfAzAHd/AigFqty9z907g/Y1wCZg/vA3cPfb3b3O3euqq6tD+BFyw57eAW5euY6zplXw3otqoy5HRLJEmAGxGphnZrVmVgwsB1YOW+dF4PUAZnYWyYDoMLPq4CA3ZjYHmAe0hlhrTvvXBzewfU8vX73mXJ3zICIjFtooJncfNLPrgQeAGHCHu68zs1uARndfCXwS+KGZfZzkAevr3N3N7GLgFjMbABLAB9y9K6xac9kzW3dx5xNb+NsLZ7GgZlLU5YhIFjF3j7qGUVFXV+eNjY1Rl5FRBocSvPnWP9C5t4/ffeISJpRqWKuIHM7M1rh7XbplutRGDvvxH7aw/qVuvn/t+QoHETlu6pDOUVu79vGNhzZwxVmncPU5p0ZdjohkIQVEDnJ3Pv/r5zCDLyw7R+c8iMgJUUDkoPvWvkxDcwefvOoMpk8aF3U5IpKlFBA5prt3gJv/3zpeNb2C97x2VtTliEgW00HqHPP1+5vo7OnjjvdcQKHOeRCRk6BvkByy5oWd/OTJF7nudbWcO2Ni1OWISJZTQOSIgaEEN/5qLdMqSvnkVa+4KomIyHFTF1OO+NFjm2nevocf/m0d5SX6ZxWRk6c9iBzwYuc+vvXwBpaecypXnn1K1OWISI5QQGQ5d+ez966lsKCAm998TtTliEgOUUBkuZXPbOOxjTv41NVncOrE0qjLEZEcooDIYrv29fPF36znvJpJXHuhznkQkdGlo5lZ7P/c38TOfQP853vPJVagy2mIyOjSHkSW+tPmLn76p62876Jazj7tFXdjFRE5aQqILNQ/mODGe9YyfdI4PnrFvKjLEZEcpS6mLPSDRzbREu/hx393AWXF+icUkXBoDyLLbN6xl+80tPDGV0/jsjOmRl2OiOQwBUQWcXc+e89aSgoLuOlNZ0ddjojkOAVEFrnnqXYe39TJp5eeydQKnfMgIuFSQGSJrr39fOm3z3P+zEm8a9HMqMsRkTyggMgSX73vebr3D/DVa15Ngc55EJExEGpAmNlSM2s2sxYzuyHN8plm1mBmT5nZs2b2hpRlnwm2azazq8OsM9M9samTn69pY8XFczjj1AlRlyMieSK0MZJmFgNuA64E2oDVZrbS3denrPY54Gfu/j0zOxu4D5gdTC8HzgFOA35nZvPdfSisejNV78AQn71nLTMry/jI63XOg4iMnTD3IBYBLe7e6u79wN3AsmHrOHDgNOCJwLZgehlwt7v3uftmoCV4vbzzvVWbaN2xly//1asoLYpFXY6I5JEwA2I6sDVlvi1oS3UzcK2ZtZHce/jwcWyLma0ws0Yza+zo6BitujNGS7yH763axFsWnMZfzKuOuhwRyTNRH6R+J/Af7j4DeAPwX2Y24prc/XZ3r3P3uurq3PoCdXduvGct44pjfE7nPIhIBMIMiHagJmV+RtCW6u+BnwG4+xNAKVA1wm1z2s8b2/jT5i5ufMOZVI0vibocEclDYQbEamCemdWaWTHJg84rh63zIvB6ADM7i2RAdATrLTezEjOrBeYBfwqx1oyyo6ePL9/3PItmV/K219QcewMRkRCENorJ3QfN7HrgASAG3OHu68zsFqDR3VcCnwR+aGYfJ3nA+jp3d2Cdmf0MWA8MAh/KpxFMX/7t8+zrH+Qr17xK5zyISGRCvRSou99H8uBzatvnU6bXA0uOsO2XgS+HWV8m+v3GHdzzVDsfuXwuc6fqnAcRiU7UB6klRe/AEJ+9dy21VeX842Vzoy5HRPKcbiaQQW6tb+GFzn3c9b7FOudBRCKnPYgMsWXHXn7w6Cbeev4MXje3KupyREQUEJnit2tfYmDI+aer50ddiogIoIDIGKua47xqegXTJo6LuhQREUABkRF27etnzQs7dQtREckoCogM8OjGHSQcLjtTASEimUMBkQEamuJUlhdz3oxJUZciInLQcQWEmZ1uZueGVUw+Gko4q5rjXDK/mpjOmhaRDDLi8yDM7EZgLpAwsxJ3f3d4ZeWPZ9p2sXPfgLqXRCTjHDEgzOwjwG0p10A6z93fESx7diyKywcNTXEKDC7R/R5EJMMcrYupE7jfzN4czD9oZveb2YMkL8Ano6C+Kc5rZk1mYllR1KWIiBzmiAHh7j8B/hJ4tZmtBNYA1wBvc/dPjVF9OW17dy/rtnWre0lEMtKxDlKfTvKGPiuADwHfAnQm1yhZ1RwH0PkPIpKRjnYM4j+AAaAMaHf395vZQpL3b1jt7reMUY05q6Gpg2kTSznzVF3WW0Qyz9FGMS109/MAzOwpAHd/CvhLM1s2FsXlsv7BBL9v2cGbF5yGmYa3ikjmOVpA/I+ZPQAUAXelLnD3X4daVR5YvaWLnr5BdS+JSMY6YkC4+w1mVgEk3L1nDGvKCw1NcYpjBSyZOyXqUkRE0jrqiXLu3j1WheSb+uY4i+dUUlasezaJSGbStZgi8ELnXlo79nK5hreKSAY7ZkCYWclI2mTkGpo0vFVEMt9I9iCeGGHbK5jZUjNrNrMWM7shzfJ/M7Ong8cGM9uVsmwoZdnKkbxftqhv7mBOVTmzq8qjLkVE5IiOdh7EqcB0YFxw/sOBsZgVJM+NOCoziwG3AVcCbcBqM1vp7usPrOPuH09Z/8PAwpSX2O/uC0b+o2SHff2D/LG1k3dfOCvqUkREjupoR0ivBq4DZgDfSGnfA9w4gtdeBLS4eyuAmd0NLAPWH2H9dwI3jeB1s9rjLZ30DyZ0/EFEMt7RhrneCdxpZm9191+ewGtPB7amzLcBi9OtaGazgFqgPqW51MwagUHga+5+b5rtVpC8DAgzZ848gRLHXn1znPLiGBfMroy6FBGRozpaF9O17v7fwGwz+8Tw5e7+jTSbnajlwC9SLi0OMMvd281sDlBvZmvdfdOwGm4Hbgeoq6vzUawnFO7OqqY4F82rorhQA8hEJLMd7VvqwBHU8cCENI9jaQdqUuZnBG3pLAd+mtrg7u3BcyuwisOPT2Sl5u172La7V6OXRCQrHK2L6QfB8xdO8LVXA/PMrJZkMCwH3jV8JTM7E5hMysgoM5sM7HP3PjOrApYAXz/BOjJG/YHhrTr+ICJZ4GhdTN8+2obu/pFjLB80s+tJ3lwoBtzh7uvM7Bag0d0PDF1dDtzt7qldRGcBPzCzBMm9nK+ljn7KVquaOjjntApOqSiNuhQRkWM62iimNSnTX+AERhi5+33AfcPaPj9s/uY02z0OnHu875fJdu8bYM2LO/ngJadHXYqIyIgcaxQTAGb2sdR5OX6PbOxgKOHqXhKRrDHSoTQZP0Io061qijO5rIgFNZOiLkVEZEQ01nIMDCWcVRs6uGR+NbEC3RxIRLLD0Q5S7+HQnkOZmR249LcB7u4VYReXK55t20XX3n51L4lIVjnaMQjdKHmUNDTFKTC4ZH511KWIiIyYupjGQH1znPNnTmZSWXHUpYiIjJgCImTx7l6ea+9W95KIZB0FRMhWNXcAujmQiGQfBUTI6pviTJtYylnTdEhHRLKLAiJE/YMJft+yg0vPmIqZhreKSHZRQISocUsXPX2DXHaGRi+JSPZRQISooTlOcayAJXOroi5FROS4KSBCVN8UZ/GcSspLjnZNRBGRzKSACMmLnfvY1LFXo5dEJGspIELS0Jy8OdDlOv9BRLKUAiIk9U1xaqvKmV1VfuyVRUQykAIiBPv6B3mitVPdSyKS1RQQIXhiUyf9gwl1L4lIVlNAhKC+KU5ZcYwLaidHXYqIyAlTQIwyd2dVcwcXza2ipDAWdTkiIidMATHKNmzvoX3Xfl29VUSyXqgBYWZLzazZzFrM7IY0y//NzJ4OHhvMbFfKsveY2cbg8Z4w6xxN9U3J4a06QC0i2S60U3zNLAbcBlwJtAGrzWylu68/sI67fzxl/Q8DC4PpSuAmoI7kbU/XBNvuDKve0dLQHOfsaRWcOrE06lJERE5KmHsQi4AWd291937gbmDZUdZ/J/DTYPpq4CF37wpC4SFgaYi1jord+wZY88JOLjtTF+cTkewXZkBMB7amzLcFba9gZrOAWqD+eLY1sxVm1mhmjR0dHaNS9Ml4dGMHQwnX8FYRyQmZcpB6OfALdx86no3c/XZ3r3P3uurq6P9qb2iOM6msiAU1Gt4qItkvzIBoB2pS5mcEbeks51D30vFumxESCeeR5g4umV9NrEA3BxKR7BdmQKwG5plZrZkVkwyBlcNXMrMzgcnAEynNDwBXmdlkM5sMXBW0Zaxn23fTubdf3UsikjNCG8Xk7oNmdj3JL/YYcIe7rzOzW4BGdz8QFsuBu93dU7btMrMvkgwZgFvcvSusWkdDfVOcAoOL50Xf1SUiMhpCvZONu98H3Des7fPD5m8+wrZ3AHeEVtwoa2iKs3DmZCaXF0ddiojIqMiUg9RZLb6nl7Xtu9W9JCI5RQExClY1J4fYXnqGupdEJHcoIEZBQ1OcUypKOHtaRdSliIiMGgXESRoYSvDYxh1cdsZUzDS8VURyhwLiJK3e0kVP36Cu3ioiOUcBcZJWNXdQFDMumlsVdSkiIqNKAXGS6pviLK6dQnlJqCOGRUTGnALiJGzt2kdLvEfdSyKSkxQQJ6GhOXlzIJ3/ICK5SAFxEuqb4syeUkZtVXnUpYiIjDoFxAna3z/EE5s61b0kIjlLAXGCnmjdQd9gQveeFpGcpYA4QfVNccqKYyyeUxl1KSIioVBAnAB3p6GpgyVzqygpjEVdjohIKBQQJ2BjvIf2XfvVvSQiOU0BcQLqm5LDWy87U1dvFZHcpYA4AQ1Ncc6aVsG0ieOiLkVEJDQKiOO0e/8AjS/s5DLd+0FEcpwC4jg9trGDoYTr7GkRyXkKiOPU0NTBxHFFLKiZFHUpIiKhUkAch0TCeWRDnEvmV1MY00cnIrkt1G85M1tqZs1m1mJmNxxhnbeb2XozW2dmd6W0D5nZ08FjZZh1jtSz7bvZ0dOv7iURyQuh3cTAzGLAbcCVQBuw2sxWuvv6lHXmAZ8Blrj7TjNL/ebd7+4LwqrvRDQ0xTGDi+frALWI5L4w9yAWAS3u3uru/cDdwLJh67wfuM3ddwK4ezzEek5aQ3OchTWTqCwvjroUEZHQhRkQ04GtKfNtQVuq+cB8M/uDmf3RzJamLCs1s8ag/S0h1jkiHXv6eLZtt7qXRCRvRH2fzEJgHnApMAN41MzOdfddwCx3bzezOUC9ma11902pG5vZCmAFwMyZM0MtdFVwc6BLdXkNEckTYe5BtAM1KfMzgrZUbcBKdx9w983ABpKBgbu3B8+twCpg4fA3cPfb3b3O3euqq8M9LtDQHGfqhBLOOa0i1PcREckUYQbEamCemdWaWTGwHBg+GuleknsPmFkVyS6nVjObbGYlKe1LgPVEZGAowWMbdnDZGVMxs6jKEBEZU6F1Mbn7oJldDzwAxIA73H2dmd0CNLr7ymDZVWa2HhgCPuXunWb2OuAHZpYgGWJfSx39NNYat+xkT9+g7h4nInkl1GMQ7n4fcN+wts+nTDvwieCRus7jwLlh1nY8GprjFMWMi+ZVRV2KiMiY0enAI9DQFGdRbSXjS6I+pi8iMnYUEMewtWsfG+M9ujmQiOQdBcQxHBjequMPIpJvFBDHUN8UZ9aUMuZUlUddiojImFJAHMX+/iEe39Sp4a0ikpcUEEfxx9ZO+gYT6l4SkbykgDiK+qY444piLK6tjLoUEZExp4A4AnenvinOkrlVlBbFoi5HRGTMKSCOoCXeQ/uu/Vx2pu79ICL5SQFxBPVNwfBWnf8gInlKAXEEDc1xzjx1AqdNGhd1KSIikVBApNHdO0Djlp0avSQieU0BkcZjG3YwmHDdPU5E8poCIo2G5jgTxxWxsGZS1KWIiERGATFMIuGsao5z8fxqCmP6eEQkf+kbcJi17bvZ0dPP5RreKiJ5TgExTENzHDO4eJ4CQkTymwJimIamOAtqJjFlfEnUpYiIREoBkaJjTx/PtO3WyXEiIiggDvPIhg4ADW8VEUEBcZiGpjhTJ5RwzmkVUZciIhK5UAPCzJaaWbOZtZjZDUdY5+1mtt7M1pnZXSnt7zGzjcHjPWHWCTAwlODRjR1ceka1bg4kIgIUhvXCZhYDbgOuBNqA1Wa20t3Xp6wzD/gMsMTdd5rZ1KC9ErgJqAMcWBNsuzOsete8sJM9vYPqXhIRCYS5B7EIaHH3VnfvB+4Glg1b5/3AbQe++N09HrRfDTzk7l3BsoeApSHWSkNTnKKYsWRuVZhvIyKSNcIMiOnA1pT5tqAt1Xxgvpn9wcz+aGZLj2PbUdXQHOeC2ZVMKC0K821ERLJGaF1Mx/H+84BLgRnAo2Z27kg3NrMVwAqAmTNnnnARbTv3sWF7D2+vqznh1xARyTVh7kG0A6nfuDOCtlRtwEp3H3D3zcAGkoExkm1x99vdvc7d66qrT/zM54bm5PDWS3X+g4jIQWEGxGpgnpnVmlkxsBxYOWyde0nuPWBmVSS7nFqBB4CrzGyymU0GrgraQtHQFGdmZRmnV5eH9RYiIlkntC4mdx80s+tJfrHHgDvcfZ2Z3QI0uvtKDgXBemAI+JS7dwKY2RdJhgzALe7eFUadvQNDPL5pB8svmKnhrSIiKczdo65hVNTV1XljY+Nxbxfv7uVLv32e5YtqeN3pGsEkIvnFzNa4e126ZVEfpI7c1IpSvv3OhVGXISKScXSpDRERSUsBISIiaSkgREQkLQWEiIikpYAQEZG0FBAiIpKWAkJERNJSQIiISFo5cya1mXUAL5zES1QBO0apnGynz+Jw+jwOp8/jkFz4LGa5e9qrneZMQJwsM2s80unm+UafxeH0eRxOn8chuf5ZqItJRETSUkCIiEhaCohDbo+6gAyiz+Jw+jwOp8/jkJz+LHQMQkRE0tIehIiIpJX3AWFmS82s2cxazOyGqOuJkpnVmFmDma03s3Vm9tGoa4qamcXM7Ckz+03UtUTNzCaZ2S/MrMnMnjez10ZdU5TM7OPB78lzZvZTMyuNuqbRltcBYWYx4DbgfwFnA+80s7OjrSpSg8An3f1s4ELgQ3n+eQB8FHg+6iIyxLeA+939TOA88vhzMbPpwEeAOnd/FcnbKi+PtqrRl9cBASwCWty91d37gbuBZRHXFBl3f8nd/xxM7yH5BTA92qqiY2YzgDcCP4q6lqiZ2UTgYuDfAdy93913RVpU9AqBcWZWCJQB2yKuZ9Tle0BMB7amzLeRx1+IqcxsNrAQeDLiUqL0TeCfgUTEdWSCWqAD+HHQ5fYjMyuPuqiouHs78C/Ai8BLwG53fzDaqkZfvgeEpGFm44FfAh9z9+6o64mCmb0JiLv7mqhryRCFwPnA99x9IbAXyNtjdmY2mWRvQy1wGlBuZtdGW9Xoy/eAaAdqUuZnBG15y8yKSIbDT9z9V1HXE6ElwJvNbAvJrsfLzey/oy0pUm1Am7sf2KP8BcnAyFdXAJvdvcPdB4BfAa+LuKZRl+8BsRqYZ2a1ZlZM8iDTyohrioyZGck+5ufd/RtR1xMld/+Mu89w99kk/1/Uu3vO/YU4Uu7+MrDVzM4Iml4PrI+wpKi9CFxoZmXB783rycGD9oVRFxAldx80s+uBB0iOQrjD3ddFXFaUlgDvBtaa2dNB243ufl90JUkG+TDwk+CPqVbg7yKuJzLu/qSZ/QL4M8nRf0+Rg2dV60xqERFJK9+7mERE5AgUECIikpYCQkRE0lJAiIhIWgoIERFJSwEhcgxmNmRmT6c8Ru0MYjObbWbPjdbriYymvD4PQmSE9rv7gqiLEBlr2oMQOUFmtsXMvm5ma83sT2Y2N2ifbWb1ZvasmT1sZjOD9lPM7B4zeyZ4HLg0Q8zMfhjcW+BBMxsXrP+R4N4cz5rZ3RH9mJLHFBAixzZuWBfTO1KW7Xb3c4FbSV79FeA7wJ3u/mrgJ8C3g/ZvA4+4+3kkr2N04Kz9ecBt7n4OsAt4a9B+A7AweJ0PhPOjiRyZzqQWOQYz63H38WnatwCXu3trcJHDl919ipntAKa5+0DQ/pK7V5lZBzDD3ftSXmM28JC7zwvmPw0UufuXzOx+oAe4F7jX3XtC/lFFDqM9CJGT40eYPh59KdNDHDo2+EaSdzw8H1gd3JhGZMwoIEROzjtSnp8Iph/n0O0n/wZ4LJh+GPggHLzX9cQjvaiZFQA17t4AfBqYCLxiL0YkTPqLROTYxqVc3RaS92U+MNR1spk9S3Iv4J1B24dJ3nntUyTvwnbgqqcfBW43s78nuafwQZJ3I0snBvx3ECIGfFu3+JSxpmMQIicoOAZR5+47oq5FJAzqYhIRkbS0ByEiImlpD0JERNJSQIiISFoKCBERSUsBISIiaSkgREQkLQWEiIik9f8BkOwTro0sTGwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "plt.plot(t_mrrs, label='t_mmr')\n",
    "fig.suptitle('HIT@K')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Hit %')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c6c99fe3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Loss')"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEjCAYAAAA1ymrVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuTklEQVR4nO3deXhU5d3/8fc3C1khCSRsCUlkVVD2RVQQFVtr61KtrYpL1dZH6+NWny78uqhdnrYuj0tbrai4VbFVsVr3pSioLCL7vi9hDRBIICSZ5f79MTMxQCIhZDJhzud1XXM5c2Zyzn08+pl7vuc+9zHnHCIi4h0JsW6AiIi0LAW/iIjHKPhFRDxGwS8i4jEKfhERj1Hwi4h4jIJfRMRjFPziWWa2zszGxrodIi1NwS8i4jEKfpE6zCzFzB40s83hx4NmlhJ+L9fM3jCz3Wa2y8ymmVlC+L2fmdkmM6sws+VmdlZs90SkYUmxboBIK/ML4GRgIOCA14BfAr8C7gBKgLzwZ08GnJn1Af4bGOac22xmxUBiyzZbpPHU4xc50DjgN8657c65UuBu4Mrwez6gC1DknPM556a50GRXASAF6Gtmyc65dc651TFpvUgjKPhFDtQVWF/n9frwMoB7gVXAe2a2xsx+DuCcWwXcBtwFbDezF82sKyKtlIJf5ECbgaI6rwvDy3DOVTjn7nDOdQfOB34cqeU7515wzp0W/lsH/Kllmy3SeAp+8bpkM0uNPIBJwC/NLM/McoFfA38HMLNvmVlPMzNgD6EST9DM+pjZmeGTwFXAfiAYm90ROTwFv3jdW4SCOvJIBWYDC4CFwBzgd+HP9gI+APYC04FHnHNTCNX3/wjsALYCHYHxLbcLIkfGdCMWERFvUY9fRMRjFPwiIh6j4BcR8RgFv4iIxyj4RUQ8RsEvIuIxCn4REY9R8IuIeIyCX0TEYxT8IiIeo+AXEfEYBb+IiMco+EVEPEbBLyLiMQp+ERGPUfCLiHiMgl9ExGOSYt2AxsjNzXXFxcWxboaIyDHliy++2OGcyzt4+TER/MXFxcyePTvWzRAROaaY2fr6lqvUIyLiMQp+ERGPUfCLiHiMgl9ExGMU/CIiHqPgFxHxGAW/iIjHeCb4t+6p4v0l22LdDBGRmPNM8E+atYHrn5vN/ppArJsiIhJTngn+an8Q52DDrspYN0VEJKY8E/yBYBCA9Tv3xbglIiKx5Zng9wUcAOt3qscvIt7mmeAPBMPBv0s9fhHxNs8Evz+oHr+ICHgp+AORGr+CX0S8zTPBHyn1bNq9H1/4S0BExIs8E/yRUk8g6Ni8e3+MWyMiEjseCv4ve/nrVO4REQ/zTvAHHO0z2gCwQWP5RcTDPBP8gaCjS1YqqckJOsErIp7mmeD3BR1JiQkUtc9QqUdEPC0p1g1oKYFgkOQEo2OHdE3bICKe5pkevz/gSEwwitqns2FXJcHwKB8REa/xTPAHgo6kRKMoN4MqX5DtFdWxbpKISExELfjNbKKZbTezRXWWDTSzGWY2z8xmm9nwaG3/YL6gIzEhgaL26YBm6RQR74pmj/9p4JyDlt0D3O2cGwj8Ovy6RURq/MUdMgBYr3n5RcSjohb8zrmpwK6DFwPtws+zgM3R2v7BIjX+rtmpJCWYevwi4lktParnNuBdM7uP0JfOKQ190MyuB64HKCwsPOoN+8M1/qTEBPJz0jSWX0Q8q6VP7t4I3O6c6wbcDjzZ0AedcxOcc0Odc0Pz8vKOesOBoCMpIbS7RR0ydAtGEfGslg7+q4HJ4ecvAS12ctcfDJKUYAAUtU9n3Q6VekTEm1o6+DcDp4efnwmsbKkNR2r8AEUd0imv8rO7sqalNi8i0mpErcZvZpOAMUCumZUAdwI/BB4ysySginANvyX4w1M2ABTkhIZ0lpTtJzu9TUs1QUSkVYha8DvnLmvgrSHR2uZXCdX4Qz3+gpw0AErKKjkxPysWzRERiRnPXLnrCwRrSz3d2n/Z4xcR8RrPBH8g6EhODAV/VloybVOT2KiRPSLiQZ4Jfn94yoaIgpx09fhFxJO8E/yBL4dzQqjOr+AXES/yRPAHg46gg6TEL4O/W046JWWVOKfpmUXEWzwR/IFwuB/c499XE6Cs0herZomIxIQ3gj9805UDa/xfDukUEfESTwS/LxAEDu7xa0iniHiTJ4I/0uOvW+MvaB/q8WtIp4h4jSeC3x88tMbfLjWZrLRk9fhFxHO8EfyBQ2v8EBnSqR6/iHiLN4I/GK7x1yn1gMbyi4g3eSL4A/WUeiAyln+/xvKLiKd4Ivh9taWeQ3v8+30Bdu7TvPwi4h2eCP5Ijz858eAav4Z0ioj3eCL4IzX+Q3r8GtIpIh7kjeAP1F/jV49fRLzIG8FfewHXgbubmZJETnqyhnSKiKd4IvgbGtUDmpdfRLzHE8HfUI0fQiN7NqrHLyIe4o3gb6DGD6H7727SWH4R8RBPBH+ggRo/hHr81f4gpRXVLd0sEZGY8ETw1zdJW0RkXv6NqvOLiEd4I/gDDdf4u9UO6VSdX0S8wRvBX3vl7qHBn197Jy71+EXEG6IW/GY20cy2m9miOsv+YWbzwo91ZjYvWtuvq75bL0akt0kiN7ONevwi4hlJUVz308BfgGcjC5xz34s8N7P7gT1R3H6t+m69WFd+Tjobd6nHLyLeELUev3NuKrCrvvfMzIDvApOitf266rv1Yl3ddEMWEfGQWNX4RwHbnHMrW2Jj/mD90zJHFOSks2n3/tovCBGReBar4L+Mw/T2zex6M5ttZrNLS0uPamP+2lJP/bvbrX0avoBje0XVUW1HRORY0OLBb2ZJwEXAP77qc865Cc65oc65oXl5eUe1Tf9hSj2RWTpV5xcRL4hFj38ssMw5V9JSG/yqSdogVOMHzcsvIt4QzeGck4DpQB8zKzGz68JvXUoLndSNOFyNv2u2xvKLiHdEbTinc+6yBpZ/P1rbbEhkkrbkBmr8qcmJdGqXolk6RcQTPHHlbiAYxAwSGujxQ2jqBg3pFBEv8ETw+4Ouwfp+REFOmk7uiogneCb4G6rvR3Rrn87W8qraoZ8iIvHKG8EfcA3W9yMKctIIBB1b9mgsv4jEN08EfyAYJLGBMfwRkemZdYJXROKdJ4Lf16gaf3heftX5RSTOeSL4AwHX4HQNEV2yU0kw3ZBFROKfJ4K/MSd3kxMT6JKVplswikjc80jwBxucp6euAk3PLCIe4JHgP3yNH0J1fo3lF5F454ngb0yNH0LTM2+rqKLaH2iBVomIxIYngt8fDB62xg+hHr9zsHm3xvKLSPzySPA7khtR4+/dKROABSW7o9wiEZHY8UTwBxoxqgegX9csstOTmbZyRwu0SkQkNjwR/L5AsFE1/sQE49SeuUxbWYpzuv+uiMQnTwR/Y3v8AKN75bKtvJqV2/dGuVUiIrHhieD3B12jxvEDnNYrdH/fqSuO7gbvIiKtlSeCP9DIcfwA+dlp9MjLUJ1fROKWJ4LfF3AkNqLGHzGqVx4z1+6kyqfx/CISfzwR/IFgsFHDOSNG986lyhfki/VlUWyViEhseCL4GzNJW10jjutAcqIxdaXq/CISf7wR/IHG1/gBMlKSGFKUw7QVqvOLSPzxRPAHgo6kxCPb1VG98liypZzSiuootUpEJDY8Efz+YPCIevwAY/qEhnW+s2hLNJokIhIz3gj+wJHV+AH6dmlHv67teH7mBl3FKyJxxRvBH3QkH2Gpx8wYN6KIZVsrmLNhd3QaJiISA1ELfjObaGbbzWzRQctvNrNlZrbYzO6J1vbrOpIpG+q6YGBXMlOSeH7m+ii0SkQkNqLZ438aOKfuAjM7A7gAGOCc6wfcF8Xt1wpN0nbkwZ+RksS3B+XzxoIt7K6siULLRERaXtSC3zk3Fdh10OIbgT8656rDn9kere3XFTiCuXoOdvmIQmr8QV7+oqSZWyUiEhstXePvDYwys5lm9rGZDWvog2Z2vZnNNrPZpaVNv5DKORe+gKtpu3pCl3YMKcrRSV4RiRstHfxJQHvgZOAnwD/NrN6uuHNugnNuqHNuaF5eXpM3GAxndVNKPRFXnFzI2h37mLHm4B8wIiLHnpYO/hJgsguZBQSB3Ghu0BcIAjTp5G7E1/p2JsFg+pqdzdUsEZGYaVTwm1mGmSWEn/c2s/PNLLkJ2/sXcEZkPUAbIKrzIgTCXf4jmaTtYBkpSfTq2Fb34hWRuNDYHv9UINXM8oH3gCsJjdppkJlNAqYDfcysxMyuAyYC3cNDPF8ErnZRLpz7w8Hf1Bp/RP+CLBaU7FGdX0SOeUmN/Jw55yrD4f2Ic+4eM5v3VX/gnLusgbeuOJIGHi1/uNRzNDV+gP7dsnnpixJKyvbTrX16czRNRCQmGtsNNjMbCYwD3gwvS4xOk5pXpNTT1OGcEQMKsgBYuGnPUbdJRCSWGhv8twHjgVedc4vNrDswJWqtakaRUs/R9vj7dG5LcqIxX3V+ETnGNarU45z7GPgYIHySd4dz7pZoNqy5+APNU+NPSUrkhC7tWLBRPX4RObY1dlTPC2bWzswygEXAEjP7SXSb1jz8wVCN/2hG9UT0L8hi0aY9BIM6wSsix67GdoP7OufKgQuBt4HjCI3safUCtaN6miP4s6mo9rNmx76jXpeISKw0NviTw+P2LwRed875gGOi2+sLNE+NH2BAQTaAxvOLyDGtscH/GLAOyACmmlkRUB6tRjWn2lE9R1njB+jZMZP0NoksKFGdX0SOXY09ufsw8HCdRevDUyy3epEaf2Iz1PgTE4wTu2ZpZI+IHNMae3I3y8z+LzJbppndT6j33+o113DOiP4FWSzZXF47B5CIyLGmsfWPiUAF8N3woxx4KlqNak5fDudsnuA/qSCLan+Q5VsrmmV9IiItrbHB38M5d6dzbk34cTfQPZoNay5fTtLWPBORDuyWDcA7i7YesNw5xytflLBIV/aKSCvX2DTcb2anRV6Y2anA/ug0qXnV1vibqcdf2D6d8wZ05S9TVjF5TuiuXM45fv/mUu54aT6XPT6DZVuPifPeIuJRjZ2k7QbgWTPLCr8uA66OTpOal78Zh3MCmBn3XdKfnXur+enLC8jJaMN/lm7nuRnruWRIAVNXlvL9iZ8z+Uen0DU7rVm2KSLSnBrV43fOzXfODQD6A/2dc4OAM6Pasmbib8bhnBEpSYk8duUQendqy7VPf85zM9bzX6O7c893+vPMtcPZV+PnqomzdIN2EWmVjigNnXPl4St4AX4chfY0u+aanfNgbVOTefraYQzqls0dZ/fm5984HjPj+M7tmHDlUDbsrORnryxo1m2KiDSHxpZ66tO8SRolzV3jr6tj21Qm/+jUQ5aP7NGBm8/syf3vr2DuhjIGFeY0+7ZFRJrqaOofx8SUDZEaf3Izlnoa49rTjqNDRhvufXd5i25XRORwvjINzazCzMrreVQAXVuojUeldpK2Zi71HE5GShI3ndGTz1bv5NNVUb2tsIjIEfnK4HfOtXXOtavn0dY5dzRlohbjCzbPrReb4vIRhXTNSuWed5frXr0i0mq0bP0jBgLNPGXDkUhNTuS2sb2Zv3E37y7e1uLbFxGpT9wH/5fj+GOzqxcNzue43AwmfrI2JtsXETlY/Ad/M87O2RRJiQmcc2Jn5mwoY1+1PyZtEBGpywPBH7tST8RpPXPxBx2z1u6KWRtERCLiPvgDzTxlQ1MMKcqhTVICn2h0j4i0AnEf/P5mvOduU6UmJzKsOEfDOkWkVYha8JvZRDPbbmaL6iy7y8w2mdm88OPcaG0/wh8MkphgmMX2QuNTeuSybGsFpRXVMW2HiEg0e/xPA+fUs/wB59zA8OOtKG4fCPX4Y1nmiTitZy4An61Wr19EYitqwe+cmwrE/GxmINA6gv/E/CzapSbx2aqdsW6KiHhcLGr8/21mC8KloAZnLzOz6yP3+C0tLW3yxvxBF9P6fkRigjGyRwc+WbVDV/GKSEy1dPA/CvQABgJbgPsb+qBzboJzbqhzbmheXl6TN+gPBpvttotH67SeuWzavZ8Nuypj3RQR8bAWTUTn3DbnXMA5FwQeB4ZHe5uBVtLjBzglXOfXsE4RiaUWDX4z61Ln5beBRQ19trn4WkmNH6B7bgZdslI1rFNEYipqM2ya2SRgDJBrZiXAncAYMxtIaC7/dcB/RWv7EYGgI6mVlHrMjDF9OvKvuZsor/LRLjU51k0SEQ+KWvA75y6rZ/GT0dpeQ1rLcM6IS4d1Y9KsDbw2dxNXjiyOdXNExINaR1c4ivyBYKup8QP0L8iiX9d2PD9zg0b3iEhMxH/wt6JSD4TKPeNGFLFsawVzNuyOdXNExINaTyJGSaCVlXoAzh/YlYw2ibwwc0OsmyIiHhT3we9rZaUegMyUJC4YlM8bCzazp9IX6+aIiMfEffC3xh4/wOXDC6n2B3llTkmsmyIiHhP3wR+q8be+4D8xP4sB3bJ5YZZO8opIy4r74A/1+Fvnbo4bXsiq7XuZvb7sgOWlFdW8s2hLjFolIvGudSZiM2ptwznr+taALmSmJDHpoJO8d/17MTf8fY6mcBaRqIj/4A86klthqQcgvU0SFw7qyhsLt7C7sgaAldsqeGthqLf/p7eXqQwkIs0u7oO/NU3SVp/LhhdS4w/y6txNADz8n1WkJycy/hvHM79kD28v2hrjFopIvIn74PcFgq22xg/Qr2sWAwqymDRrAyu3VfDGgs1cdUoxPxjVnd6dMrnv3eX4AsFYN1NE4kjrTcRmEmilo3rqumx4ISu27eXmSXNJS07kh6O6k5hg/PTrx7Nmxz7+OXtjrJsoInEk7oO/tdyB66ucN6ArmSlJLNtawVUji2mf0QaAs07oyLDiHB54fyXLt1bEuJUiEi/iP/hb0Xz8DclISeLiwflkpiTxw1HH1S43M+46vx/OOc778yf87ePVBII62SsiRyf+g7+VTdLWkPHnnsCHd5xOh8yUA5b365rFe7eP5szjO/LHt5dx6YTp7K8JxKiVIhIPWn8iHqVAMNjqe/wAqcmJdGqXWu97HTJTePSKwfz2whP5fF0ZU5Zvb+HWiUg8ifvg9wdaf42/McyMS4d1o21KEtNWlsa6OSJyDIv/4A86ko+BUk9jJCcmMLJHB6au2KELu0SkyeIjEb9Ca7+A60iN7p3Hpt37WV26L9ZNEZFjVNwHv+8YqfE31um98wCYukLlHhFpmrgO/mDQ4Rxx1ePv1j6d43IzVOcXkSaL6+D3h8e8x0uNP2J0r1xmrNlFtV/DOkXkyMVXIh4kcrFTPPX4IVTn3+8LMHtd2eE/LCJykLgOfl8wNLlZPNX4AU7u3oHkRFOdX0SaJK6DPxAI9fjjLfgzUpIYWtSeqStDN2pxzrG32h/jVonIsSIpWis2s4nAt4DtzrkTD3rvDuA+IM85F7XbTEVq/IlxVuMHGNU7l3veWc6lE6azfGsFZZU+BhRkcenwwtpJ30RE6hPNRHwaOOfghWbWDfgasOHg95qbP05LPQDnntiFrlmp7PcF+Xq/ztx6Vi+qfEHGT17IiN9/wIw1O2PdRBFppaLWLXTOTTWz4nreegD4KfBatLYd4Y/TUg9AcW4Gn40/64Blt43txdyNu/nR3+fw8IcrObl7hxi1TkRasxatgZjZBcAm59z8ltheZFRPa78RS3MxMwYX5nDlyCI+W72Tlds0h7+IHKrFgt/M0oH/B/y6kZ+/3sxmm9ns0tKmjV6JlHoSW/GtF6Ph0mHdaJOUwDPT18W6KSLSCrVkIvYAjgPmm9k6oACYY2ad6/uwc26Cc26oc25oXl5ekzZYewFXHJZ6vkqHzBTO69+VyXM2UV7lA0JXMT/16Vo+XRW1c+kicoxoseB3zi10znV0zhU754qBEmCwc25rtLYZqfHH2wVcjXH1KUVU1gR45YsSAkHHz15ZwN3/XsK4J2Zy1+uLdTMXEQ+L5nDOScAYINfMSoA7nXNPRmt79fF7rMZfV/+CbAYVZvPs9PXM3bCb1+dv5uYze7K32s9Tn65j6spS/nr5YE7o0i7WTRWRFhbNUT2XHeb94mhtOyLg0Rp/xNUji7ntH/NYu2MfPzvneG4c0wOAs47vxI//OY8fPDObt28bRbvU5Bi3VERaUlwnYqTU47Uaf8S5J3XhrOM78tsLT6wNfYDTeuXytyuHsLW8irteXxzDFopILMR18MfrJG2N1SYpgSe/P4wrTy465L3BhTncdEZPJs/ZxFsLt8SgdSISK3Ed/D4P1/gb4+YzezKgIIv/9+pCtpVXxbo5ItJC4jr4A7VTNsT1bjZZcmICD3xvINW+IFc+OZMFJbtj3SQRaQFxnYheHs7ZWN3zMnnsyiHs2e/jwr9+yh/fXkaVT0M9ReJZfAe/Sj2NMrp3Hu/dfjqXDOnG3z5ezQ+emY1zLtbNEpEo8Ubwq9RzWFlpyfzpO/35xbkn8MmqHUzX7J4icSuuEzEQx9MyR8uVI4vo1C6FBz9YGeumiEiUxHXw+1TjP2KpyYnccHoPZq3dxfTV6vWLxKO4Dv7IOP7kOLwDVzRdNryQvLYpPPThilg3RUSiIK4T0e/xC7iaKtLrn7FmFzNV6xeJO/Ed/AHV+Jtq3IhCcjNT+NVri3QbR5E4E9fB77U7cDWn1ORE/njRSZRV+rh0wgy+99h0fQGIxIm4Dn4N5zw6Y/t2YtpPz+DO8/qybuc+Lp0wg1smzWV7haZ3EDmWxXUiRko9qvE3XWpyItecehwf/+QMbj2rF+8s2spZ93/Mc9PXEQzqIi+RY1F8B39tj1/Bf7RSkxO5/ezevHPbKAYUZPOr1xZz6YQZrN2xD4Bqf4DX5m3iztcWMW1lqb4URFqxqN2IpTUIBB0JBgkK/mbTPS+T564bzstflPCbN5ZwzoNT+eZJXfhoRSm79tWQmGA8M309RR3SuWRIAf3ysyhsn05+dhqpyYmxbr6IEOfB7w861fejwMy4ZGg3RvfO4xevLuT1+ZsZe0InLh9RyLDi9ry7eCsvzNzAfe99eR1AanICj181lFG98mLYchGBeA/+QFD1/Sjq1C6VJ64eRjDoDvhVdeGgfC4clE9pRTXrd+5jY1klf/5wFeMnL+S920eT3iau/7MTafXiujvsDzoN5WwBDZXS8tqmMLS4Pd8eVMAfLjqJkrL9PFTPHEDBoGPH3mqWb63AFz4hLyLRE9ddr0DQ6cRuKzGiewe+N7QbT3yylvMHdqVvl3a8u3gbD36wgjWl+6gJB/7gwmye+v5wstJ1A3iRaInr4B/TJ4+u2WmxboaEjT/3eD5cto2fvryArLRkPlu9k14dM7n2tOPo3C4Ff9BxzzvL+e5j03nuuuF0bJca6yaLxCU7Fm64MXToUDd79uxYN0OawWvzNnHri/PITk/mx2f35vLhhSTVmUTvk5U7uP652XTIbMP1o7pTWRNgb7WfrLRkju/cjj6d25LXNqVZ2xQMOuZuLGNQtxyNAJO4YmZfOOeGHrJcwS8tyTnHJ6t2cFJ+Ftnpber9zLyNu7nmqVmUVfoAMIO6/5nmZ6cxuncep/fO5bReeWSmHN0P1yemreF3by7ll988gR+M6n5AW99ZtJWRPTo02FaR1kzBL8eU/TUBKqp8ZKYmkZacyK59NSzfWsHSrRXMWruTT1ftZG+1n87tUpn4/WH07drusOt0zrGvJnDAF8XGXZV87YGpVPsDZKQk8fFPzqB9RijkJ83awPjJCzmjTx4Tvz8Ms6/+NRAIOm74+xdkpyVzz3f6H/bzItHWUPDH9ageOXaltUmkY7tU0tskYWZ0yEzhlJ65XHfacTx25VDm/vps/n7dCMzgkr99xscrSr9yfftrAtz64jwG3P0ez89cD4S+CH712iLM4KlrhlNZE+CB90PXHmzYWcnv3lhCbmYbpiwv5V/zNh22zX/7eDXvL9nGS1+UMHnO4T/fWJ+s3MGYe6ewdEt5s62ztZq/cTcbd1XGuhmtgj8QZNKsDeyt9jf7uqMW/GY20cy2m9miOst+a2YLzGyemb1nZl2jtX2Jb8mJCZzWK5dXf3QqRR0yuPbpz3n4w5XM2VDG/prAAZ/dvHs/lzz2Gf9esJnendryi1cX8Ye3lvL6/M18tLyUO77Wh9N753HFiEKen7mepVvK+Z+X5pOQYLz6o1MZXJjN3f9eQmlFdYPtWViyhwfeX8G5J3VmeHF77np9MZt37z/q/dxWXsWtL85l3c5KHv7w6G+HuWNvNXv2+456PdGwZ7+PcU/M5OeTFzTp73furWbqYToAx5L3lmxj/OSFfLpqR7OvO2qlHjMbDewFnnXOnRhe1s45Vx5+fgvQ1zl3w+HWpVKPfJW91X5umTSX/yzbDkCCQUFOOh0y29Ahow3zNu6myhfkwe8NZEyfPH7zxhKenb6eBIMT87N49UenkphglO2r4fR7p5CUmMCufTXcf8kALh5SwKrtFZz70CeM7duRR8YNOWT7+2sCfPPP06isDvDObaMo3+/nnIemMrgwh2evHd7kE8aBoGPcEzOYv3EPZ/ftxOvzN/Pe7aPp3altk9a3vybAmfd/hC8Q5OHLBnFKj9zD/s2ufTWs37mPQYU5jd7O3mo/y7eWs2xrBdvLqxk3orBRI7Qe/Wg1f3pnGYkJxuxfjCUno/HnVYJBx6UTZjBr3S5eumEkw4rbN/pvj5Zzjo9WlLK2dB9dslLpnJVKZkoSlTUB9tX4yc9Oo6hDRoN/X+0PsGFnJb0OOq6X/O0ztpZX8dH/nNHkC1EbKvVEbTinc26qmRUftKzub9UMoPWfYJBWLzMliSevHsrmPVUs3rSHRZvLWbtjH2X7ati0u4riDhn84aKTav/Huvv8fhR3yODxaWv4w0Un1f5PlZPRhtvG9uY3byzha307cdHgfAB6dmzLrWN7ce+7y/nWn6cB4A840tokkpWWzJ79PtaU7uP5H4wgO70N2elt+MU3T+AXry7iztcXM6BbNjnpyeTnpNGrY9va7ZWUVTJ5ziY+X7eLGn8Qf9CRkpTAqT1zGXtCJ95auIUZa3Zx3yUDOPP4jnywdBuPTFnFg5cOatK/p8enrWHLnioKctK44omZ3PG1Ptx4eo8Gv5jKq3xcOmE6q7bv5V83nUr/guza90rKKnn5ixJ+OKo7GXXOmby/ZBs3vTCHGv+XF+J9tHw7//ivkV85V1O1P8DET9dS2D6dDbsq+WDpNi4Z2q3R+/bCrA3MWreLNkkJ/O9bS5l84ylHdI6losrHIx+t5rtDu3FcbsYBy697ZjYbdlbSNjWJdmnJ9OvajnP6dWb4ce1ZtrWC3725hBlrdjW47jaJCTx6xWDOOqHTIe9NW1nKna8tZs2OfTx9zTDG9OkIwKJNe/h8XRm//OYJUZl9IKond8PB/0akxx9e9nvgKmAPcIZzrt7fZmZ2PXA9QGFh4ZD169dHrZ0iEb5AkFfnbOLr/TofcBGZLxDkt28sYf3OSpISjIQEo8oXYHelj/IqH5cNL+SG03vUft45x00vzOGthVsPWH9mShIDumXhHExfsxPnoF/XdmSkJJGcaJTt87GkTi3/4sEF3P/dAQD871tLeWLaGv5zxxiKw+FU4w8SdI7EBCMpwRoMu+0VVYy59yNO753HvZcM4OevLOCNBVsYWpTDNacex9f6dTrg3tS+QJBrn/6c6at30jY1iYKcdP51U+iXUbU/wMWPfsaiTeUML27PU9cMIyMliTkbyrj88Rn07tSWW87sRZ/ObVmypZz/eu4LLh3WjT9e3L/Bf+///HwjP31lAc9eO5zxkxdyQpe2PHH1sNr3V26rYNHmPeRnp1OQk0andqm1gbhlz37O/r+pDOiWxfkDuvKzVxbyyLjBnHtSFyD0y2ntjn307JhZ77Z3V9Zw9cRZzC/ZQ/e8DF676VTapoaO/W0vzuX1+Zu5cFA++2sClFXW1P6CzEpLprzKR056G24f24tvnNSFbeVVbNldRaUvQEabxNANjd5exrKt5fzl8sF8vV9nAFZsq+ChD1by5sItFHVIJ+gcCWa8e9toUpMTueOf83l70Ramjz+LrLSmX8wYk1E99QV/nffGA6nOuTsPtx6VeuRY5JyjvMrP7soayip9rN2xlznrd/PF+jKq/AHOH9CViwcX0K19+gF/t3VPFVOWb2f19r3cfnbv2h719ooqRv1pChcOzOeHo7vzxLQ1TJ676YDedc+OmQwpzGFIUQ5j+3aqHaE0fvJCXpq9kQ9+fDrFuRk455g0ayOPfLSKkrL95LVN4duD8jmtZy7DitvzmzcWM2nWRu75Tn/SkhO5edJc7j6/H1efUsxv31jCk5+s5aqRRTw/cwODC7O587x+XDVxFm1Tk3jlxlPIzfzyWov73l3OX6as4g8XncSoXrk8P3MDby7Ywqk9c/n1t/qSkpTA2Q98TEpSIm/echq/fWMpf5+5njm/OpvMlCSqfAHOuv9jNtU5Z5KbmcIVJxcybkQR4ycv4JNVO3jvttPJz0njGw9NpcYf5L3bT2dvtZ+bJ83h01U7a9tf18691Vzx5CxWb9/LDWN68Ncpq/ha3048Mm4wr87dxI//OZ/bx/bm1rG9av+mssbP1BWlvL9kO52zUrh+dI+vDOc9+31cPXEWizbt4ZpTi5m+ZieLNpWTkpTATWf05PrR3Zm9rowrnpzJbWN7MW5EEaf+8T9cOrwbv7ngkOg8Iq0x+AuBt+p772AKfpGQu15fzLPT1xF0kJKUwEWD8+nWPp1AwFHlD7B0SwVzNpSxu9JHRptErj3tOE7vncd3H5vO1acUc+d5/Q5YXyDo+HjFdv4+YwPTVpbiC4SmOfEHHf99Rk/+5+t9cM5x1cRZzN2wm19+8wR+PnkhV40s4jcXnMibC7Zwy4tzCTpHTnobJt94Su2vkbrbuObpz/ls1Q6C4bwZUpTD7PVl9MzL5DtDCvjD28t46NKBXDAwn1lrd/Hdx6bz58sGcd6ArkyYupr/fWsZD35vIDkZbSgpq+SDJduYsry0tq11r8GYsmw71zz9OVeNLOLDpdsp3VvNCV3asaBkN3+7Ykhtr3vZ1nL++4W5lJRVMuHKoYzuncfjU9fw+7eW8v1Tinlp9kb65Wcx6YcnH3W5paLKxzVPfc7s9WX069qOiwcXcP7Argd8Qd48aS7vLt7Kef278sqcEj6843R65NX/K6WxWkXwm1kv59zK8PObgdOdc9853HoU/CIhW/dUccukuYzs0YGrRhbRIfPQq5iDQceSLeU8+tFq3ly4BYB2qaFrFL7qhGlljZ9Za3fx6aodoRvvjO1dW/9fv3MfZz8Q6kkf37kt/7rp1Nqa/dsLt/DAByv408X9GzwJvLuyhjv+OZ8+ndsy7uQi8rPT+GTlDm77xzx27K0mPzuNj34yhuTEBAJBx4j//YAR3TvwuwtOZPS9UxhSlMPT1ww/YJ1rSvfyzGfrqKj2c+93BtSGs3OOyx+fyfQ1O8nPTuNvVwyhZ8dMLnt8Bku3lPPUNcOYtnIHj09dQ7u0ZP56+WBG9uhQ+7eREl1WWjJv3zqq2aZ9qfYH2LqnqsETvdvKqzjr/o/ZW+1nTJ+8Q/a3KVo8+M1sEjAGyAW2AXcC5wJ9gCCwHrjBOXfYAc8KfpGmWbx5DxOmruHM4ztywcD8o1rXhKmreeSj1bx8wykN1suP1PaKKv749jK+3q9zbU8cQqWp1+Zt4tuD8pk0awNv3zqaPp0bP5pp7Y59vDBzPTeO6Vlb7tq5t5qLHv2M9TtD1wl8d2gB479xwiFfhnur/YyfvJBLhhQwunfL3j/iqU/Xcve/l/DcdcOb5d4VunJXRI6aLxA84CRwtExdUcpVE2cBHPbE8JFYt2Mf//f+Ci4fUcjJ3Ts0yzqbk3OOVdv3HjK0s6lafDiniMSflgh9gJO7d6BdahK+gOPHZ/dutvUW52bw8GVNGw7bEsys2UL/qyj4RaTVaZOUwF3n9yM5MUHTc0eBgl9EWqWLBhfEuglxS5O0iYh4jIJfRMRjFPwiIh6j4BcR8RgFv4iIxyj4RUQ8RsEvIuIxCn4REY85JubqMbNSQpO6NUUu0Pw3rWz9vLjfXtxn8OZ+e3Gf4cj3u8g5d8hsb8dE8B8NM5td3yRF8c6L++3FfQZv7rcX9xmab79V6hER8RgFv4iIx3gh+CfEugEx4sX99uI+gzf324v7DM2033Ff4xcRkQN5occvIiJ1xHXwm9k5ZrbczFaZ2c9j3Z5oMLNuZjbFzJaY2WIzuzW8vL2ZvW9mK8P/rP8u2McwM0s0s7lm9kb49XFmNjN8vP9hZg3fWfwYZWbZZvaymS0zs6VmNjLej7WZ3R7+b3uRmU0ys9R4PNZmNtHMtpvZojrL6j22FvJweP8XmNngI9lW3Aa/mSUCfwW+AfQFLjOzvrFtVVT4gTucc32Bk4Gbwvv5c+BD51wv4MPw63hzK7C0zus/AQ8453oCZcB1MWlVdD0EvOOcOx4YQGj/4/ZYm1k+cAsw1Dl3IpAIXEp8HuungXMOWtbQsf0G0Cv8uB549Eg2FLfBDwwHVjnn1jjnaoAXgQti3KZm55zb4pybE35eQSgI8gnt6zPhjz0DXBiTBkaJmRUA3wSeCL824Ezg5fBH4nGfs4DRwJMAzrka59xu4vxYE7pTYJqZJQHpwBbi8Fg756YCuw5a3NCxvQB41oXMALLNrEtjtxXPwZ8PbKzzuiS8LG6ZWTEwCJgJdHLObQm/tRXoFKt2RcmDwE+BYPh1B2C3c84ffh2Px/s4oBR4KlziesLMMojjY+2c2wTcB2wgFPh7gC+I/2Md0dCxPap8i+fg9xQzywReAW5zzpXXfc+Fhm7FzfAtM/sWsN0590Ws29LCkoDBwKPOuUHAPg4q68Thsc4h1Ls9DugKZHBoOcQTmvPYxnPwbwK61XldEF4Wd8wsmVDoP++cmxxevC3y0y/8z+2xal8UnAqcb2brCJXwziRU+84OlwMgPo93CVDinJsZfv0yoS+CeD7WY4G1zrlS55wPmEzo+Mf7sY5o6NgeVb7Fc/B/DvQKn/1vQ+iE0OsxblOzC9e2nwSWOuf+r85brwNXh59fDbzW0m2LFufceOdcgXOumNBx/Y9zbhwwBfhO+GNxtc8AzrmtwEYz6xNedBawhDg+1oRKPCebWXr4v/XIPsf1sa6joWP7OnBVeHTPycCeOiWhw3POxe0DOBdYAawGfhHr9kRpH08j9PNvATAv/DiXUM37Q2Al8AHQPtZtjdL+jwHeCD/vDswCVgEvASmxbl8U9ncgMDt8vP8F5MT7sQbuBpYBi4DngJR4PNbAJELnMXyEft1d19CxBYzQqMXVwEJCo54avS1duSsi4jHxXOoREZF6KPhFRDxGwS8i4jEKfhERj1Hwi4h4jIJfPM3MAmY2r86j2SY4M7PiujMtirQWSYf/iEhc2++cGxjrRoi0JPX4RephZuvM7B4zW2hms8ysZ3h5sZn9JzwH+odmVhhe3snMXjWz+eHHKeFVJZrZ4+H55N8zs7Tw528J30NhgZm9GKPdFI9S8IvXpR1U6vlenff2OOdOAv5CaDZQgD8Dzzjn+gPPAw+Hlz8MfOycG0Bo/pzF4eW9gL865/oBu4GLw8t/DgwKr+eG6OyaSP105a54mpntdc5l1rN8HXCmc25NeBK8rc65Dma2A+jinPOFl29xzuWaWSlQ4JyrrrOOYuB9F7qJBmb2MyDZOfc7M3sH2Eto2oV/Oef2RnlXRWqpxy/SMNfA8yNRXed5gC/Pq32T0Fwrg4HP68w0KRJ1Cn6Rhn2vzj+nh59/RmhGUIBxwLTw8w+BG6H2XsBZDa3UzBKAbs65KcDPgCzgkF8dItGiXoZ4XZqZzavz+h3nXGRIZ46ZLSDUa78svOxmQnfA+gmhu2FdE15+KzDBzK4j1LO/kdBMi/VJBP4e/nIw4GEXuoWiSItQjV+kHuEa/1Dn3I5Yt0WkuanUIyLiMerxi4h4jHr8IiIeo+AXEfEYBb+IiMco+EVEPEbBLyLiMQp+ERGP+f8IG3vhfFRtaAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# losses.append(loss.item())\n",
    "fig = plt.figure()\n",
    "plt.plot(losses)\n",
    "fig.suptitle('Loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c426d92",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}