kanakanji_convert_4_check.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wzz_FRgi7C7P"
   },
   "outputs": [],
   "source": [
    "# ディープラーニングを使ったかな漢字変換モデル\n",
    "# @yoh_okunoさんのモデルをtf.keras＆jupyterで動くようにしたものです。\n",
    "# tf.keras化の過程で若干オリジナルと異なる部分が出ているかもしれません。\n",
    "# オリジナルのモデル:\n",
    "# https://github.com/yohokuno/neural_ime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "z-BOEOPe7C3h"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BYk_Je9Be2sS"
   },
   "outputs": [],
   "source": [
    "# prepare dataset\n",
    "\n",
    "from utility import load_train_data\n",
    "from collections import Counter\n",
    "import os\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ct8_vGQFe2sX"
   },
   "outputs": [],
   "source": [
    "# import tensorflow\n",
    "import tensorflow as tf\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yiyf4vD15lj6"
   },
   "outputs": [],
   "source": [
    "# import ramdon for reproducability\n",
    "import random as rn\n",
    "\n",
    "np.random.seed(42)\n",
    "rn.seed(12345)\n",
    "tf.set_random_seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OtFy0MBNF1e0"
   },
   "outputs": [],
   "source": [
    "# prepare model\n",
    "\n",
    "def gru(units, backword_flg):\n",
    "    # CuDNNGRU does not compabible with GRU for tensorflow 1.12\n",
    "    return tf.keras.layers.GRU(units, \n",
    "                               return_sequences=True, \n",
    "                               return_state=True, \n",
    "                               recurrent_activation='sigmoid', \n",
    "                               recurrent_initializer='glorot_uniform',\n",
    "                               go_backwards=backword_flg)\n",
    "    \n",
    "    \n",
    "class KanaKanjiModel(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, rnn_units, batch_size):\n",
    "        super(KanaKanjiModel, self).__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.rnn_units = rnn_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru_f = gru(self.rnn_units, False)\n",
    "        self.gru_b = gru(self.rnn_units, True)\n",
    "        self.vocab_size = vocab_size\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "        \n",
    "    def call(self, x, hidden, training):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru_f(x, initial_state = hidden)\n",
    "        output = tf.keras.layers.Dropout(0.5)(output, training=training)\n",
    "        output = self.fc(output)\n",
    "        \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_size, self.rnn_units))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "r3BOezFR5lj_"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "87l4j9ZB5lkB"
   },
   "outputs": [],
   "source": [
    "# rebuild moel\n",
    "hidden_size = 400\n",
    "embedding_dim = hidden_size\n",
    "vocabulary_size = 50000\n",
    "\n",
    "model = KanaKanjiModel(vocabulary_size, embedding_dim, hidden_size, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LR1KT5GPXm5w"
   },
   "outputs": [],
   "source": [
    "optimizer3 = tf.train.AdamOptimizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "UGkUGNAc5lkE",
    "outputId": "701a2d95-2088-4313-c5e9-fc9a3844da76"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "status:  <tensorflow.python.training.checkpointable.util.CheckpointLoadStatus object at 0x130a929e8>\n"
     ]
    }
   ],
   "source": [
    "# load model\n",
    "\n",
    "# load weight\n",
    "checkpoint_dir = './ck_20190209'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer3,\n",
    "                                 model=model)\n",
    "\n",
    "status = checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))\n",
    "# status = checkpoint.restore(\"./training_checkpoints_ver4/ckpt-3\")\n",
    "print(\"status: \", status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uBCNB72u5lkI"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./ck_20190209/ckpt-5'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.train.latest_checkpoint(checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MOVk5YnD5lkN"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ylGHPGAq5lkP"
   },
   "outputs": [],
   "source": [
    "# this is based on decode.py of the original scripts\n",
    "\n",
    "import collections\n",
    "import heapq\n",
    "import operator\n",
    "\n",
    "hiragana_added = []\n",
    "\n",
    "def load_dictionary(model_directory):\n",
    "    vocabulary_path = os.path.join(model_directory, 'vocabulary.txt')\n",
    "    vocabulary = []\n",
    "    for line in open(vocabulary_path):\n",
    "        line = line.rstrip('\\n')\n",
    "        target, source = line.split('/', 1)\n",
    "        vocabulary.append((target, source))\n",
    "\n",
    "    dictionary = collections.defaultdict(list)\n",
    "    for i, (target, source) in enumerate(vocabulary):\n",
    "        dictionary[source].append((target, i))\n",
    "\n",
    "    return dictionary\n",
    "\n",
    "def create_lattice(input_, dictionary):\n",
    "    lattice = [[[] for _ in range(len(input_) + 1)] for _ in range(len(input_) + 2)]\n",
    "    _, unk_id = dictionary['_UNK'][0]\n",
    "\n",
    "    for i in range(1, len(input_) + 1):\n",
    "        for j in range(i):\n",
    "            key = input_[j:i]\n",
    "            if key in dictionary:\n",
    "                for target, word_id in dictionary[key]:\n",
    "                    lattice[i][j].append((target, word_id))\n",
    "            elif len(key) == 1:\n",
    "                # Create _UNK node with verbatim target when single character key is not found in the dictionary.\n",
    "                lattice[i][j].append((key, unk_id))\n",
    "\n",
    "    _, eos_id = dictionary['_EOS'][0]\n",
    "    lattice[-1][-1].append(('', eos_id))\n",
    "    return lattice\n",
    "\n",
    "\n",
    "def initialize_queues(lattice, rnn_predictor, dictionary):\n",
    "    # Initialize priority queues for keeping hypotheses\n",
    "    # A hypothesis is a tuple of (cost, string, state, prediction)\n",
    "    # cost is total negative log probability\n",
    "    # state.shape == [hidden_size * layer_size]\n",
    "    # prediction.shape == [vocabulary_size]\n",
    "    \n",
    "    hiragana_added = []\n",
    "    \n",
    "    hidden_f = rnn_predictor.initialize_hidden_state()\n",
    "    hidden_b = hidden_f\n",
    "    \n",
    "    _, bos_id = dictionary['_BOS'][0]\n",
    "    \n",
    "    input_x0 = tf.expand_dims([bos_id], 0)\n",
    "    \n",
    "    bos_predictions, hidden_f = rnn_predictor(input_x0, hidden_f, False)\n",
    "    \n",
    "    \n",
    "    # logits to probability\n",
    "    bos_predictions = tf.squeeze(bos_predictions, 0)\n",
    "    bos_predictions = tf.squeeze(bos_predictions, 0)\n",
    "    bos_predictions = -1 * tf.nn.log_softmax(bos_predictions, axis=0)\n",
    "    \n",
    "    hidden_f = tf.expand_dims(hidden_f, 0)\n",
    "    \n",
    "    bos_hypothesis = (0.0, '', hidden_f[0], bos_predictions)\n",
    "    queues = [[] for _ in range(len(lattice))]\n",
    "    queues[0].append(bos_hypothesis)\n",
    "    return queues\n",
    "\n",
    "def search(lattice, queues, rnn_predictor, beam_size, viterbi_size):\n",
    "    # Breadth first search with beam pruning and viterbi-like pruning\n",
    "    for i in range(len(lattice)):\n",
    "        queue = []\n",
    "\n",
    "        # create hypotheses without predicting next word\n",
    "        for j in range(len(lattice[i])):\n",
    "            for target, word_id in lattice[i][j]:\n",
    "                \n",
    "                # if word_id == 350:\n",
    "                #    print(\"word_id: \", word_id)\n",
    "                \n",
    "                word_queue = []\n",
    "                for previous_cost, previous_string, previous_state_f, previous_prediction in queues[j]:\n",
    "                    # if logits is bigger, better.\n",
    "                    \n",
    "                    # seems to need give huge priority to first word\n",
    "                    cost = previous_cost + previous_prediction[word_id]\n",
    "                    \n",
    "                    string = previous_string + target\n",
    "                    hypothesis = (cost, string, word_id, previous_state_f)\n",
    "                    word_queue.append(hypothesis)\n",
    "                \n",
    "                # prune word_queue to viterbi size\n",
    "                if viterbi_size > 0:\n",
    "                    word_queue = heapq.nsmallest(viterbi_size, word_queue, key=operator.itemgetter(0))\n",
    "                    \n",
    "                queue += word_queue\n",
    "                \n",
    "        # prune queue to beam size\n",
    "        if beam_size > 0:\n",
    "            queue = heapq.nsmallest(beam_size, queue, key=operator.itemgetter(0))\n",
    "        \n",
    "        # predict next word and state before continue\n",
    "        for cost, string, word_id, previous_state_f in queue:\n",
    "            \n",
    "            input_x0 = tf.expand_dims([word_id], 0)\n",
    "              \n",
    "            predictions, state_f = rnn_predictor(input_x0, [previous_state_f], False)\n",
    "            \n",
    "            # logits to probability\n",
    "            predictions = tf.squeeze(predictions, 0)\n",
    "            predictions = tf.squeeze(predictions, 0)\n",
    "            predictions = -1 * tf.nn.log_softmax(predictions, axis=0)\n",
    "            # print(\"predictions.shape: \", predictions.shape)\n",
    "        \n",
    "            state_f = tf.expand_dims(state_f, 0)\n",
    "            hypothesis = (cost, string, state_f[0], predictions)\n",
    "            queues[i].append(hypothesis)\n",
    "\n",
    "    return queues\n",
    "\n",
    "def decode(source, dictionary, rnn_predictor, beam_size, viterbi_size):\n",
    "    lattice = create_lattice(source, dictionary)\n",
    "    queues = initialize_queues(lattice, rnn_predictor, dictionary)\n",
    "    queues = search(lattice, queues, rnn_predictor, beam_size, viterbi_size)\n",
    "\n",
    "    candidates = []\n",
    "    for cost, string, _, _ in queues[-1]:\n",
    "        candidates.append((string, cost))\n",
    "\n",
    "    top_result = candidates[0][0]\n",
    "    return top_result, candidates, lattice, queues\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jc5wSKEx5lkS"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_6Ovasav5lkU"
   },
   "outputs": [],
   "source": [
    "def convert_kana_to_kanji(line, rnn_predictor):\n",
    "    \n",
    "    # Load settings and vocabulary\n",
    "    model_directory = \"models\"\n",
    "    dictionary = load_dictionary(model_directory)\n",
    "    \n",
    "    # debug\n",
    "    print_nbest = True \n",
    "    print_lattice = False\n",
    "    print_queue = False\n",
    "    \n",
    "    # parameters\n",
    "    beam_size = 5\n",
    "    viterbi_size = 50000\n",
    "    \n",
    "    \n",
    "    # Iterate input file line by line\n",
    "    line = line.rstrip('\\n')\n",
    "\n",
    "    # Decode - this might take ~10 seconds per line\n",
    "    result, candidates, lattice, queues = decode(line, dictionary, rnn_predictor, beam_size, viterbi_size)\n",
    "\n",
    "    # Print decoded results\n",
    "    if not print_nbest:\n",
    "        print(result)\n",
    "    else:\n",
    "        for string, cost in candidates:\n",
    "            print(string, cost)\n",
    "\n",
    "    # Print lattice for debug\n",
    "    if print_lattice:\n",
    "        for i in range(len(lattice)):\n",
    "            for j in range(len(lattice[i])):\n",
    "                print('i = {}, j = {}'.format(i, j))\n",
    "                for target, word_id in lattice[i][j]:\n",
    "                    print(target, word_id)\n",
    "\n",
    "    # Print queues for debug\n",
    "    if print_queue:\n",
    "        for i, queue in enumerate(queues):\n",
    "            print('queue', i)\n",
    "            for cost, string, state_f, prediction in queue:\n",
    "                # print(string, cost)\n",
    "                print(string, cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 2627
    },
    "colab_type": "code",
    "id": "PHDOpmA_5lkZ",
    "outputId": "c0f06f51-1228-4d88-c691-f2608f3d39eb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/kouohhashi/anaconda3/envs/vui2/lib/python3.6/site-packages/tensorflow/python/distribute/distribute_lib.py:1677: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "パリの歴史 tf.Tensor(24.277733, shape=(), dtype=float32)\n",
      "パリの歴誌 tf.Tensor(37.341175, shape=(), dtype=float32)\n",
      "パリの歴し tf.Tensor(38.27497, shape=(), dtype=float32)\n",
      "パリの歴史 tf.Tensor(38.383877, shape=(), dtype=float32)\n",
      "パリノ歴史 tf.Tensor(38.963135, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# test the model\n",
    "convert_kana_to_kanji(\"ぱりのれきし\", model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tfGjADvb5lkc"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GHStcBxZ5lke"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "kanakanji_convert_3_check.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}