ULMFiT_Tokenize.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ULMFiT + Siamese Network for Sentence Vectors\n",
    "## Part One: Tokenizing\n",
    "This notebook will tokenize the sentences from the SNLI dataset for use in the next notebook\n",
    "\n",
    "### You must have the fastai library installed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "import json\n",
    "import html\n",
    "import re\n",
    "import pickle\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import sklearn\n",
    "from sklearn import model_selection\n",
    "from functools import partial\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils \n",
    "import torch.optim as optim\n",
    "import torch.optim.lr_scheduler as lr_scheduler\n",
    "from torch.utils.data import dataset, dataloader\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import time\n",
    "import math\n",
    "import sys\n",
    "import data\n",
    "\n",
    "token_files = './data/tokens/'\n",
    "snli_root = './data/snli_1.0/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! wget https://github.com/briandw/SiameseULMFiT/releases/download/1/data.zip\n",
    "#! unzip ./data.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1140304\n"
     ]
    }
   ],
   "source": [
    "# load and process the all the sentences, just to get the LM trained\n",
    "raw_text = []\n",
    "for file in [f\"{snli_root}snli_1.0_train.jsonl\", f\"{snli_root}snli_1.0_dev.jsonl\", f\"{snli_root}snli_1.0_test.jsonl\"]:\n",
    "    with open(file) as fp:\n",
    "        while True:\n",
    "            line = fp.readline()\n",
    "            if line != None and len(line) > 0:\n",
    "                item = json.loads(line)\n",
    "                raw_text.append(item['sentence1'])\n",
    "                raw_text.append(item['sentence2'])\n",
    "            else:\n",
    "                break\n",
    "print(len(raw_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split the language model data into train and validation sets\n",
    "lm_train, lm_valid = sklearn.model_selection.train_test_split(raw_text, test_size=0.1)\n",
    "df_trn = pd.DataFrame(lm_train)\n",
    "df_val = pd.DataFrame(lm_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "BOS = 'x_bos'  # beginning-of-sentence tag\n",
    "\n",
    "re1 = re.compile(r'  +')\n",
    "\n",
    "def fixup(x):\n",
    "    x = x.replace('#39;', \"'\").replace('amp;', '&').replace('#146;', \"'\").replace(\n",
    "        'nbsp;', ' ').replace('#36;', '$').replace('\\\\n', \"\\n\").replace('quot;', \"'\").replace(\n",
    "        '<br />', \"\\n\").replace('\\\\\"', '\"').replace('<unk>','u_n').replace(' @.@ ','.').replace(\n",
    "        ' @-@ ','-').replace('\\\\', ' \\\\ ')\n",
    "    return re1.sub(' ', html.unescape(x))\n",
    "\n",
    "def get_texts(df):\n",
    "    texts = df[0].astype(str)\n",
    "    texts = list(texts.apply(fixup).values)\n",
    "    texts = f'{BOS} ' + df[0].astype(str)\n",
    "    tok = Tokenizer().proc_all_mp(partition_by_cores(texts))\n",
    "    return tok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_trn = np.concatenate(get_texts(df_trn))\n",
    "tok_val = np.concatenate(get_texts(df_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['x_bos', 'people', 'are', 'in', 'a', 'room', 'discussing', 'around', 'a', 'computer', 'printer', '.',\n",
       "       'x_bos', 'the', 'girl', 'is', 'playing', 'on', 'a', 'swing', 'set', '.', 'x_bos', 'a', 'cleaning',\n",
       "       'woman', 'in', 'a', 'bright', 'uniform', 'is', 'pushing', 'a', 'cart', '.', 'x_bos', 'animals',\n",
       "       'playing', 'in', 'a', 'field', 'x_bos', 'a', 'person', 'is', 'taking', 'a', 'picture', 'of', 'some',\n",
       "       'kids', '.', 'x_bos', 'protesters', 'joining', 'on', 'a', 'city', 'street', '.', 'x_bos', 'bicyclist',\n",
       "       'riding', 'their', 'bikes', 'across', 'a', 'metal', 'bridge', '.', 'x_bos', 'a', 'man', 'is',\n",
       "       'holding', 'a', 'flashlight', '.', 'x_bos', 'the', 'team', 'swiftly', 'moves', 'their', 'traditional',\n",
       "       'boat', 'down', 'the', 'river', '.', 'x_bos', 'a', 'man', 'with', 'long', 'hair', 'and', 'a', 'pink',\n",
       "       'shirt'], dtype='<U17')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tok_val[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save our work\n",
    "np.save(f'{token_files}tok_trn.npy', tok_trn)\n",
    "np.save(f'{token_files}tok_val.npy', tok_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_trn = np.load(f'{token_files}tok_trn.npy')\n",
    "tok_val = np.load(f'{token_files}tok_val.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('a', 1496301),\n",
       " ('x_bos', 1140304),\n",
       " ('.', 999604),\n",
       " ('the', 555295),\n",
       " ('in', 423992),\n",
       " ('is', 387917),\n",
       " ('man', 276785),\n",
       " ('on', 245180),\n",
       " ('and', 215231),\n",
       " ('are', 206834),\n",
       " ('of', 200547),\n",
       " ('with', 176178),\n",
       " ('woman', 143101),\n",
       " ('two', 126950),\n",
       " ('people', 125650),\n",
       " (',', 119923),\n",
       " ('to', 118745),\n",
       " ('at', 102452),\n",
       " ('wearing', 84424),\n",
       " ('an', 83451),\n",
       " ('his', 75557),\n",
       " ('shirt', 65479),\n",
       " ('young', 64126),\n",
       " ('men', 63408),\n",
       " ('playing', 61568)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freq = Counter(np.concatenate([tok_trn, tok_val]))\n",
    "freq.most_common(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34158"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_vocab = 60000\n",
    "min_freq = 1\n",
    "itos = [o for o, c in freq.most_common(max_vocab) if c>=min_freq]\n",
    "itos.insert(0, '_pad_')\n",
    "itos.insert(0, '_unk_')\n",
    "stoi = defaultdict(lambda:0, {v:k for k,v in enumerate(itos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34160"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(stoi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the language model training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_lm = np.array([stoi[p] for p in tok_trn])\n",
    "val_lm = np.array([stoi[p] for p in tok_val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save results\n",
    "pickle.dump(itos, open(f'{token_files}itos.pkl', 'wb'))\n",
    "np.save(f'{token_files}trn_lm.npy', trn_lm)\n",
    "np.save(f'{token_files}val_lm.npy', val_lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34160"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#load the results so we can pick it up from here \n",
    "itos = pickle.load(open(f'{token_files}itos.pkl', 'rb'))\n",
    "trn_lm = np.load(f'{token_files}trn_lm.npy')\n",
    "val_lm = np.load(f'{token_files}val_lm.npy')\n",
    "\n",
    "stoi = defaultdict(lambda:0, {v:k for k,v in enumerate(itos)})\n",
    "vocab_size = len(itos)\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_bos people are in a room discussing around a computer printer . x_bos the girl is playing on a swing set . x_bos a cleaning woman in a bright uniform is pushing a cart . x_bos animals playing in a field x_bos a person is taking a picture of some kids . x_bos protesters joining on a city street . x_bos bicyclist riding their bikes across a metal bridge . x_bos a man is holding a flashlight . x_bos the team swiftly moves their traditional boat down the river . x_bos a man with long hair and a pink shirt "
     ]
    }
   ],
   "source": [
    "for word in val_lm[:100]:\n",
    "    print(itos[word], end=\" \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the sentence similarity dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from enum import Enum\n",
    "\n",
    "class Entail(Enum):\n",
    "    entailment = 0\n",
    "    contradiction = 1\n",
    "    neutral = 2\n",
    "       \n",
    "def load_sentence_pairs(json_file):\n",
    "    content = []\n",
    "    with open(json_file) as fp:\n",
    "        while True:\n",
    "            line = fp.readline()\n",
    "            if line:\n",
    "                content.append(json.loads(line))\n",
    "            else:\n",
    "                break\n",
    "\n",
    "    s0s = []\n",
    "    s1s = []\n",
    "    labels = []\n",
    "    avg_len = []\n",
    "    for item in content:\n",
    "        l = item['gold_label']\n",
    "        s0 = BOS+\" \"+fixup(item['sentence1'])\n",
    "        s1 = BOS+\" \"+fixup(item['sentence2'])\n",
    "\n",
    "        average_len = (len(s0)+len(s1))/2\n",
    "        try:\n",
    "            label = Entail[l].value\n",
    "            s0s.append(s0)\n",
    "            s1s.append(s1)\n",
    "            labels.append(label)\n",
    "            avg_len.append(average_len)\n",
    "        except KeyError:\n",
    "            pass\n",
    "        \n",
    "    s0s = Tokenizer().proc_all_mp(partition_by_cores(s0s))\n",
    "    s1s = Tokenizer().proc_all_mp(partition_by_cores(s1s))\n",
    "    return np.array((s0s, s1s, labels, avg_len)).transpose()    \n",
    "\n",
    "sentence_pairs_train = load_sentence_pairs(f'{snli_root}/snli_1.0_train.jsonl')\n",
    "sentence_pairs_dev = load_sentence_pairs(f'{snli_root}snli_1.0_dev.jsonl')\n",
    "sentence_pairs_test = load_sentence_pairs(f'{snli_root}snli_1.0_test.jsonl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f'{token_files}trn_snli.npy', sentence_pairs_train)\n",
    "np.save(f'{token_files}dev_snli.npy', sentence_pairs_dev)\n",
    "np.save(f'{token_files}test_snli.npy', sentence_pairs_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(sentence_pairs):\n",
    "    for i in range(len(sentence_pairs)):\n",
    "        item = sentence_pairs[i]\n",
    "        tok0 = [stoi[p] for p in item[0]]\n",
    "        tok1 =[stoi[p] for p in item[1]]\n",
    "        sentence_pairs[i] = np.array([tok0, tok1, item[2], item[3]])\n",
    "\n",
    "tokenize(sentence_pairs_train)\n",
    "tokenize(sentence_pairs_dev)\n",
    "tokenize(sentence_pairs_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f'{token_files}snli_tok_train.npy', sentence_pairs_train)\n",
    "np.save(f'{token_files}snli_tok_dev.npy', sentence_pairs_dev)\n",
    "np.save(f'{token_files}snli_tok_test.npy', sentence_pairs_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check our work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " x_bos a person on a horse jumps over a broken down airplane .\n",
      " x_bos a person is training his horse for a competition .\n",
      " x_bos two women are embracing while holding to go packages .\n",
      " x_bos the sisters are hugging goodbye while holding to go packages after just eating lunch .\n",
      " x_bos this church choir sings to the masses as they sing joyous songs from the book at a church .\n",
      " x_bos the church has cracks in the ceiling .\n"
     ]
    }
   ],
   "source": [
    "itos = pickle.load(open(f'{token_files}itos.pkl', 'rb'))\n",
    "\n",
    "dev = np.load(f'{token_files}snli_tok_dev.npy')\n",
    "train = np.load(f'{token_files}snli_tok_train.npy')\n",
    "test = np.load(f'{token_files}snli_tok_test.npy')\n",
    "\n",
    "def print_sentence(s):\n",
    "    sentence = \"\"\n",
    "    for tok in s:\n",
    "        sentence += \" \"+itos[tok]\n",
    "    print(sentence)\n",
    "\n",
    "print_sentence(train[0][0])\n",
    "print_sentence(train[0][1])\n",
    "\n",
    "print_sentence(dev[0][0])\n",
    "print_sentence(dev[0][1])\n",
    "\n",
    "print_sentence(test[0][0])\n",
    "print_sentence(test[0][1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " x_bos two women are embracing while holding to go packages .\n",
      " x_bos the sisters are hugging goodbye while holding to go packages after just eating lunch .\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3, 15, 47, 11, 2243, 30, 48, 18, 381, 3644, 4]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}