BPE_Tests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import sys\n",
    "from bpe.encoder import Encoder\n",
    "from nltk.tokenize import wordpunct_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_path = \"test_text.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(corpus_path, encoding=\"utf8\") as infile:\n",
    "    lines = list(map(str.strip, infile))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_word_ngrams(vocab):\n",
    "    ngrams_count = {}\n",
    "    for k, _ in vocab.items():\n",
    "        if k != '__eow' and k != '__sow':\n",
    "            length = len(wordpunct_tokenize(k))\n",
    "            if length not in ngrams_count:\n",
    "                ngrams_count[length] = 1\n",
    "            else:\n",
    "                ngrams_count[length] += 1\n",
    "    return ngrams_count\n",
    "\n",
    "def count_char_ngrams(vocab):\n",
    "    ngrams_count = {}\n",
    "    for k, _ in vocab.items():\n",
    "        if k != '__eow' and k != '__sow':\n",
    "            length = len(k)\n",
    "            if length not in ngrams_count:\n",
    "                ngrams_count[length] = 1\n",
    "            else:\n",
    "                ngrams_count[length] += 1\n",
    "    return ngrams_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text encoder (word ngrams and char-ngrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder()\n",
    "encoder.set_params(tokenize_symbols=True, vocab_size=1000, pct_bpe=0.9, word_ngram_max=3, ngram_max=5)\n",
    "encoder.fit(lines)\n",
    "text = \"There is a leader and he is winner in sports. inner city is a level of the 'Earthworm Jim' game\"\n",
    "print(encoder.transform(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(encoder.word_vocab.items())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(encoder.bpe_vocab.items())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in count_word_ngrams(encoder.word_vocab).items():\n",
    "    print(\"Word ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "for k, v in count_char_ngrams(encoder.bpe_vocab).items():\n",
    "    print(\"Char ngram size {}: {} times\".format(k, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Default encoder (words and char-ngrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(lines[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder()\n",
    "encoder.set_params(tokenize_symbols=True, ngram_max=4, vocab_size=100, pct_bpe=0.9)\n",
    "encoder.fit(lines[0])\n",
    "text = \"There is a leader and he is winner\"\n",
    "print(encoder.transform(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(encoder.word_vocab.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(encoder.bpe_vocab.items())[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in count_word_ngrams(encoder.word_vocab).items():\n",
    "    print(\"Word ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "for k, v in count_char_ngrams(encoder.bpe_vocab).items():\n",
    "    print(\"Char ngram size {}: {} times\".format(k, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder (words only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder()\n",
    "encoder.set_params(tokenize_symbols=False)\n",
    "encoder.fit(lines[0])\n",
    "text = \"There is a leader and he is winner\"\n",
    "print(encoder.transform(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(encoder.word_vocab.items())[2:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(encoder.bpe_vocab.items())[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in count_word_ngrams(encoder.word_vocab).items():\n",
    "    print(\"Word ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "for k, v in count_char_ngrams(encoder.bpe_vocab).items():\n",
    "    print(\"Char ngram size {}: {} times\".format(k, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder (words ngrams only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder()\n",
    "encoder.set_params(tokenize_symbols=False)\n",
    "encoder.fit(lines)\n",
    "text = \"There is a leader and he is winner\"\n",
    "print(encoder.transform(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(encoder.word_vocab.items())[2:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(encoder.bpe_vocab.items())[2:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in count_word_ngrams(encoder.word_vocab).items():\n",
    "    print(\"Word ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "for k, v in count_char_ngrams(encoder.bpe_vocab).items():\n",
    "    print(\"Char ngram size {}: {} times\".format(k, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sample tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiple word n-grams, Multiple character n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text = [\n",
    "    \"Hello world from earth\",\n",
    "    \"Hello George who has two ears\",\n",
    "    \"Hello Georgia which has a big capital city\",\n",
    "    \"A litte George said hello\",\n",
    "    \"Linking ideas in domain is a good idea\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text_single = \"Hello world from earth. Hello George who has two ears. Hello Georgia which has a big capital city. A litte George said hello. Linking ideas in domain is a good idea\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['__sow', 'S', 'a', 'y', '__eow', '__sow', 'H', 'ello', '__eow', '__sow', 't', 'o', '__eow', '__sow', 'g', 'u', 'y', 's', '__eow', '__sow', 'f', 'r', 'o', 'm', '__eow', '__sow', 'Geor', 'g', 'i', 'a', '__eow']\n",
      "\n",
      "Word ngram size 2: 5 times\n",
      "\n",
      "Char ngram size 2: 14 times\n",
      "Char ngram size 3: 7 times\n",
      "Char ngram size 4: 4 times\n",
      "\n",
      "Word vocabulary - {'Hello world': 0, 'world from': 1, 'from earth': 2, 'Hello George': 3, 'George who': 4}\n",
      "Ngrams vocabulary - {'in': 5, 'ea': 6, 'id': 7, 'it': 8, 'as': 9, 'ai': 10, 'de': 11, 'ide': 12, 'dea': 13, 'idea': 14, 'Ge': 15, 'eo': 16, 'or': 17, 'rg': 18, 'Geo': 19, 'eor': 20, 'org': 21, 'Geor': 22, 'eorg': 23, 'el': 24, 'll': 25, 'lo': 26, 'ell': 27, 'llo': 28, 'ello': 29}\n"
     ]
    }
   ],
   "source": [
    "encoder = Encoder()\n",
    "encoder.set_params(tokenize_symbols=True, vocab_size=30, pct_bpe=0.8, ngram_min=2, ngram_max=4, word_ngram_min=2)\n",
    "encoder.fit(train_text)\n",
    "text = \"Say Hello to guys from Georgia\"\n",
    "print(encoder.transform(text))\n",
    "\n",
    "print()\n",
    "\n",
    "for k, v in count_word_ngrams(encoder.word_vocab).items():\n",
    "    print(\"Word ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "for k, v in count_char_ngrams(encoder.bpe_vocab).items():\n",
    "    print(\"Char ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "print(\"Word vocabulary - {}\".format(encoder.word_vocab))\n",
    "print(\"Ngrams vocabulary - {}\".format(encoder.bpe_vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word n-grams, Multiple character n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['__sow', 'S', 'a', 'y', '__eow', 'Hello', '__sow', 't', 'o', '__eow', '__sow', 'g', 'u', 'y', 's', '__eow', '__sow', 'f', 'r', 'om', '__eow', '__sow', 'Georg', 'i', 'a', '__eow']\n",
      "\n",
      "Word ngram size 1: 2 times\n",
      "\n",
      "Char ngram size 2: 17 times\n",
      "Char ngram size 3: 6 times\n",
      "Char ngram size 4: 3 times\n",
      "Char ngram size 5: 2 times\n",
      "\n",
      "Word vocabulary - {'.': 0, 'Hello': 1}\n",
      "Ngrams vocabulary - {'or': 2, 'ea': 3, 'in': 4, 'Ge': 5, 'eo': 6, 'rg': 7, 'Geo': 8, 'eor': 9, 'org': 10, 'Geor': 11, 'eorg': 12, 'Georg': 13, 'as': 14, 'it': 15, 'id': 16, 'wo': 17, 'om': 18, 'ar': 19, 'ear': 20, 'ge': 21, 'rge': 22, 'orge': 23, 'eorge': 24, 'wh': 25, 'ha': 26, 'has': 27, 'ai': 28, 'de': 29}\n"
     ]
    }
   ],
   "source": [
    "encoder = Encoder()\n",
    "encoder.set_params(tokenize_symbols=True, vocab_size=30, pct_bpe=0.9, ngram_min=2, ngram_max=5)\n",
    "encoder.fit(train_text_single)\n",
    "text = \"Say Hello to guys from Georgia\"\n",
    "print(encoder.transform(text))\n",
    "\n",
    "print()\n",
    "\n",
    "for k, v in count_word_ngrams(encoder.word_vocab).items():\n",
    "    print(\"Word ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "for k, v in count_char_ngrams(encoder.bpe_vocab).items():\n",
    "    print(\"Char ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "print(\"Word vocabulary - {}\".format(encoder.word_vocab))\n",
    "print(\"Ngrams vocabulary - {}\".format(encoder.bpe_vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## no word n-grams, character n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['__sow', 'S', 'a', 'y', '__eow', '__sow', 'H', 'el', 'lo', '__eow', '__sow', 't', 'o', '__eow', '__sow', 'g', 'u', 'y', 's', '__eow', '__sow', 'f', 'r', 'om', '__eow', '__sow', 'Ge', 'or', 'g', 'i', 'a', '__eow']\n",
      "\n",
      "\n",
      "Char ngram size 2: 20 times\n",
      "\n",
      "Word vocabulary - {}\n",
      "Ngrams vocabulary - {'in': 0, 'ea': 1, 'id': 2, 'it': 3, 'or': 4, 'as': 5, 'ar': 6, 'ai': 7, 'de': 8, 'Ge': 9, 'eo': 10, 'rg': 11, 'el': 12, 'll': 13, 'lo': 14, 'wo': 15, 'om': 16, 'wh': 17, 'ha': 18, 'rs': 19}\n"
     ]
    }
   ],
   "source": [
    "encoder = Encoder()\n",
    "encoder.set_params(tokenize_symbols=True, vocab_size=20, pct_bpe=1, word_ngram_min=2, word_ngram_max=5)\n",
    "encoder.fit(train_text)\n",
    "text = \"Say Hello to guys from Georgia\"\n",
    "print(encoder.transform(text))\n",
    "\n",
    "print()\n",
    "\n",
    "for k, v in count_word_ngrams(encoder.word_vocab).items():\n",
    "    print(\"Word ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "for k, v in count_char_ngrams(encoder.bpe_vocab).items():\n",
    "    print(\"Char ngram size {}: {} times\".format(k, v))\n",
    "    \n",
    "print()\n",
    "\n",
    "print(\"Word vocabulary - {}\".format(encoder.word_vocab))\n",
    "print(\"Ngrams vocabulary - {}\".format(encoder.bpe_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:trees-dev]",
   "language": "python",
   "name": "conda-env-trees-dev-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}