6_deep_rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import theano, numpy\n",
    "from theano import tensor as T\n",
    "import os\n",
    "import copy\n",
    "import sys\n",
    "import gzip\n",
    "# import cPickle\n",
    "import pickle\n",
    "import urllib\n",
    "import random\n",
    "import stat\n",
    "import subprocess\n",
    "import sys\n",
    "import timeit\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import numpy as np\n",
    "from collections import OrderedDict\n",
    "sys.setrecursionlimit(1500)\n",
    "\n",
    "# get home directory\n",
    "from os.path import expanduser\n",
    "PREFIX = expanduser(\"./atis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def download(origin, destination):\n",
    "    '''\n",
    "    download the corresponding atis file\n",
    "    from http://www-etud.iro.umontreal.ca/~mesnilgr/atis/\n",
    "    '''\n",
    "    print ('Downloading data from %s' % origin)\n",
    "    urllib.urlretrieve(origin, destination)\n",
    "\n",
    "\n",
    "# data loading functions\n",
    "def atisfold(fold):\n",
    "    assert fold in range(5)\n",
    "    filename = os.path.join(PREFIX, 'atis.fold'+str(fold)+'.pkl.gz')\n",
    "    f = gzip.open(filename, 'rb')\n",
    "    train_set, valid_set, test_set, dicts = pickle.load(f, encoding='byte')\n",
    "    return train_set, valid_set, test_set, dicts\n",
    "def shuffle(lol, seed):\n",
    "    '''\n",
    "    lol :: list of list as input\n",
    "    seed :: seed the shuffling\n",
    "\n",
    "    shuffle inplace each list in the same order\n",
    "    '''\n",
    "    for l in lol:\n",
    "        random.seed(seed)\n",
    "        random.shuffle(l)\n",
    "\n",
    "def contextwin(l, win):\n",
    "    '''\n",
    "    win :: int corresponding to the size of the window\n",
    "    given a list of indexes composing a sentence\n",
    "\n",
    "    l :: array containing the word indexes\n",
    "\n",
    "    it will return a list of list of indexes corresponding\n",
    "    to context windows surrounding each word in the sentence\n",
    "    '''\n",
    "    assert (win % 2) == 1\n",
    "    assert win >= 1\n",
    "    l = list(l)\n",
    "\n",
    "    lpadded = win // 2 * [-1] + l + win // 2 * [-1]\n",
    "    out = [lpadded[i:(i + win)] for i in range(len(l))]\n",
    "\n",
    "    assert len(out) == len(l)\n",
    "    return out\n",
    "def conlleval(p, g, w, filename):\n",
    "    '''\n",
    "    INPUT:\n",
    "    p :: predictions\n",
    "    g :: groundtruth\n",
    "    w :: corresponding words\n",
    "    OUTPUT:\n",
    "    filename :: name of the file where the predictions\n",
    "    are written. it will be the input of conlleval.pl script\n",
    "    for computing the performance in terms of precision\n",
    "    recall and f1 score\n",
    "    '''\n",
    "    out = ''\n",
    "    for sl, sp, sw in zip(g, p, w):\n",
    "        out += 'BOS O O\\n'\n",
    "        for wl, wp, w in zip(sl, sp, sw):\n",
    "            out += w + ' ' + wl + ' ' + wp + '\\n'\n",
    "        out += 'EOS O O\\n\\n'\n",
    "\n",
    "    f = open(filename,'w')\n",
    "    f.writelines(out)\n",
    "    f.close()\n",
    "    \n",
    "    return get_perf(filename)\n",
    "\n",
    "def get_perf(filename):\n",
    "    ''' run conlleval.pl perl script to obtain\n",
    "    precision/recall and F1 score '''\n",
    "    _conlleval = os.path.join(PREFIX, 'conlleval.pl')\n",
    "    if not os.path.isfile(_conlleval):\n",
    "        #download('http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl') \n",
    "        os.system('wget https://www.comp.nus.edu.sg/%7Ekanmy/courses/practicalNLP_2008/packages/conlleval.pl')\n",
    "        os.chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions\n",
    "\n",
    "    proc = subprocess.Popen([\"perl\", _conlleval], stdin=subprocess.PIPE, stdout=subprocess.PIPE)\n",
    "    stdout, _ = proc.communicate(open(filename).read())\n",
    "    #print _conlleval\n",
    "    #print \"STDOUT: \"\n",
    "    #print stdout\n",
    "    for line in stdout.split('\\n'):\n",
    "        if 'accuracy' in line:\n",
    "            out = line.split()\n",
    "            break\n",
    "    \n",
    "    # out = ['accuracy:', '16.26%;', 'precision:', '0.00%;', 'recall:', '0.00%;', 'FB1:', '0.00']\n",
    "    \n",
    "    precision = float(out[3][:-2])\n",
    "    recall    = float(out[5][:-2])\n",
    "    f1score   = float(out[7])\n",
    "\n",
    "    return {'p':precision, 'r':recall, 'f1':f1score}\n",
    "\n",
    "def get_perfo(filename):\n",
    "    ''' \n",
    "    work around for using a PERL script in python\n",
    "    dirty but still works.\n",
    "    '''\n",
    "    tempfile = str(random.randint(1,numpy.iinfo('i').max)) + '.txt'\n",
    "    if not isfile(PREFIX + 'conlleval.pl'):\n",
    "        os.system('wget https://www.comp.nus.edu.sg/%7Ekanmy/courses/practicalNLP_2008/packages/conlleval.pl')\n",
    "        #download('http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl') \n",
    "        chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions\n",
    "    if len(PREFIX) > 0:\n",
    "        chmod(PREFIX + 'conlleval.pl', stat.S_IRWXU) # give the execute permissions\n",
    "        cmd = PREFIX + 'conlleval.pl < %s | grep accuracy > %s'%(filename,tempfile)\n",
    "    else:\n",
    "        cmd = './conlleval.pl < %s | grep accuracy > %s'%(filename,tempfile)\n",
    "    print (cmd)\n",
    "    out = os.system(cmd)\n",
    "    out = open(tempfile).readlines()[0].split()\n",
    "    os.system('rm %s'%tempfile)\n",
    "    precision = float(out[6][:-2])\n",
    "    recall    = float(out[8][:-2])\n",
    "    f1score   = float(out[10])\n",
    "    return {'p':precision, 'r':recall, 'f1':f1score}\n",
    "\n",
    "def arraysentence2string(arr):\n",
    "    return \" \".join([idx2word[x] for x in arr if x > 0])\n",
    "\n",
    "def arraylabels2string(arr):\n",
    "    return \" \".join([idx2label[x] for x in arr if x > 0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preliminaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.12274046,  0.13564919,  0.14991554,  0.2236477 ,  0.20236481,\n",
       "         0.16568229]])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.nnet.softmax([[0.1,0.2,0.3,0.7,0.6,0.4]]).eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "import theano.tensor as T\n",
    "\n",
    "k = T.iscalar(\"k\")\n",
    "A = T.vector(\"A\")\n",
    "\n",
    "# Symbolic description of the result\n",
    "result, updates = theano.scan(fn=lambda prior_result, A: prior_result * A,\n",
    "                              outputs_info=T.ones_like(A),\n",
    "                              non_sequences=A,\n",
    "                              n_steps=k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# We only care about A**k, but scan has provided us with A**1 through A**k.\n",
    "# Discard the values that we don't care about. Scan is smart enough to\n",
    "# notice this and not waste memory saving them.\n",
    "final_result = result[-1]\n",
    "\n",
    "# compiled function that returns A**k\n",
    "power = theano.function(inputs=[A,k], outputs=result, updates=updates)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0.   1.   2.   3.   4.   5.   6.   7.   8.   9.]\n",
      " [  0.   1.   4.   9.  16.  25.  36.  49.  64.  81.]]\n"
     ]
    }
   ],
   "source": [
    "print (power(range(10),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 10)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "power(range(10),2).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "LookupError",
     "evalue": "unknown encoding: byte",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-104-b0dd2a97b20f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# load the dataset\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtrain_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdic\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0matisfold\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0midx2label\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdic\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'labels2idx'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miteritems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0midx2word\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdic\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'words2idx'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miteritems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-98-53f80c2559c0>\u001b[0m in \u001b[0;36matisfold\u001b[1;34m(fold)\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[0mfilename\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mPREFIX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'atis.fold'\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfold\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'.pkl.gz'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgzip\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 15\u001b[1;33m     \u001b[0mtrain_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdicts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'byte'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     16\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mtrain_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_set\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdicts\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlol\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mLookupError\u001b[0m: unknown encoding: byte"
     ]
    }
   ],
   "source": [
    "# load the dataset\n",
    "train_set, valid_set, test_set, dic = atisfold(0)\n",
    "\n",
    "idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems())\n",
    "idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems())\n",
    "\n",
    "train_lex, train_ne, train_y = train_set\n",
    "valid_lex, valid_ne, valid_y = valid_set\n",
    "test_lex, test_ne, test_y = test_set\n",
    "\n",
    "vocsize = len(dic['words2idx'])\n",
    "print (\"Dictionary size: \"+str(vocsize))\n",
    "nclasses = len(dic['labels2idx'])\n",
    "print (\"Number of class labels: \"+str(nclasses))\n",
    "nsentences = len(train_lex)\n",
    "\n",
    "groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y]\n",
    "words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex]\n",
    "groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y]\n",
    "words_test = [map(lambda x: idx2word[x], w) for w in test_lex]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'UTF-8'"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import locale\n",
    "locale.getpreferredencoding()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How does the data look like?\n",
    "### In order to represent sentences, a vocabulary is built and each word corresponds to an index. A sentence is an array of indices."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The first sentence in the training data set\n",
    "train_lex[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The first sentence in the training data set\n",
    "example_sentence = train_lex[0]\n",
    "print example_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print arraysentence2string(example_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The first label in the training data set\n",
    "example_labels = train_y[0]\n",
    "print example_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print arraylabels2string(example_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Context window\n",
    "#### Each word is considered within its context. That is we look before and after the word. If the word is at the beginning/end of a sentence, we use padding (-1)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "win_size = 5 \n",
    "csample = contextwin(example_sentence, win_size)\n",
    "\n",
    "for x in csample:\n",
    "        print x\n",
    "        print arraysentence2string(x)+'\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import NearestNeighbors\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class RNNSLU(object):\n",
    "    ''' elman neural net model '''\n",
    "    def __init__(self, nh, nfo, nh2, nc, ne, de, cs):\n",
    "        '''\n",
    "        nh1 :: dimension of the first hidden layer\n",
    "        nfo :: number of output elements of first RNN\n",
    "        nh2 :: dimension of the second hidden layer\n",
    "        nc :: number of classes\n",
    "        ne :: number of word embeddings in the vocabulary / size of vocabulary\n",
    "        de :: dimension of the word embeddings\n",
    "        cs :: word window context size\n",
    "        '''\n",
    "        # parameters of the model\n",
    "        self.emb = theano.shared(name='embeddings',\n",
    "                                 value=0.2 * numpy.random.uniform(-1.0, 1.0,\n",
    "                                 (ne+1, de))\n",
    "                                 # add one for padding at the end\n",
    "                                 .astype(theano.config.floatX))\n",
    "        self.wx = theano.shared(name='wx',\n",
    "                                value=0.2 * numpy.random.uniform(-1.0, 1.0,\n",
    "                                (de * cs, nh))\n",
    "                                .astype(theano.config.floatX))\n",
    "        self.wh = theano.shared(name='wh',\n",
    "                                value=0.2 * numpy.random.uniform(-1.0, 1.0,\n",
    "                                (nh, nh))\n",
    "                                .astype(theano.config.floatX))\n",
    "        self.w = theano.shared(name='w',\n",
    "                               value=0.2 * numpy.random.uniform(-1.0, 1.0,\n",
    "                               (nh, nfo))\n",
    "                               .astype(theano.config.floatX))\n",
    "        self.bh = theano.shared(name='bh',\n",
    "                                value=numpy.zeros(nh,\n",
    "                                dtype=theano.config.floatX))\n",
    "        self.b = theano.shared(name='b',\n",
    "                               value=numpy.zeros(nfo,\n",
    "                               dtype=theano.config.floatX))\n",
    "        self.h0 = theano.shared(name='h0',\n",
    "                                value=numpy.zeros(nh,\n",
    "                                dtype=theano.config.floatX))\n",
    "        \n",
    "        \n",
    "        \n",
    "        ###\n",
    "        self.wx2 = theano.shared(name='wx2',\n",
    "                                value=0.2 * numpy.random.uniform(-1.0, 1.0,\n",
    "                                (nfo, nh2))\n",
    "                                .astype(theano.config.floatX))\n",
    "        self.wh2 = theano.shared(name='wh2',\n",
    "                                value=0.2 * numpy.random.uniform(-1.0, 1.0,\n",
    "                                (nh2, nh2))\n",
    "                                .astype(theano.config.floatX))\n",
    "        self.w2 = theano.shared(name='w2',\n",
    "                               value=0.2 * numpy.random.uniform(-1.0, 1.0,\n",
    "                               (nh2, nc))\n",
    "                               .astype(theano.config.floatX))\n",
    "        self.bh2 = theano.shared(name='bh2',\n",
    "                                value=numpy.zeros(nh2,\n",
    "                                dtype=theano.config.floatX))\n",
    "        self.b2 = theano.shared(name='b2',\n",
    "                               value=numpy.zeros(nc,\n",
    "                               dtype=theano.config.floatX))\n",
    "        self.h20 = theano.shared(name='h20',\n",
    "                                value=numpy.zeros(nh2,\n",
    "                                dtype=theano.config.floatX))\n",
    "        \n",
    "        \n",
    "\n",
    "        # bundle the parameters together, nice for gradient computation\n",
    "        self.params = [self.emb, self.wx, self.wh, self.w,\n",
    "                       self.bh, self.b, self.h0, self.wx2, self.wh2, self.w2,\n",
    "                       self.bh2, self.b2, self.h20]\n",
    "        \n",
    "        # we perform training on a complete sentence, which is broken down into context window elements\n",
    "        \n",
    "        # input is a matrix full of indices:\n",
    "        # as many columns as context window size\n",
    "        # as many lines as words in the sentence (each word is presented within context window)\n",
    "        idxs = T.imatrix()\n",
    "        \n",
    "        # map the input to the embedding space\n",
    "        x = self.emb[idxs].reshape((idxs.shape[0], de*cs))\n",
    "        \n",
    "        # labels for the words in the stenence\n",
    "        y_sentence = T.ivector('y_sentence')\n",
    "        \n",
    "        \n",
    "        # basic Elmann recurrent networks\n",
    "        \n",
    "        def recurrence(x_t, h_tm1, h2_tm1):\n",
    "            \n",
    "            # hidden state\n",
    "            h_t = T.nnet.sigmoid(T.dot(x_t, self.wx)\n",
    "                                 + T.dot(h_tm1, self.wh) + self.bh)\n",
    "            \n",
    "            # output\n",
    "            #s_t = T.nnet.softmax(T.dot(h_t, self.w) + self.b)\n",
    "            \n",
    "            tmp = T.nnet.sigmoid(T.dot(h_t, self.w) + self.b)\n",
    "            \n",
    "            \n",
    "            ###### 2.0*T.dot(s_t, self.wx2)+ T.dot(h2_tm1, self.wh2) + \n",
    "            h2_t = T.nnet.sigmoid(T.dot(tmp, self.wx2)+ T.dot(h2_tm1, self.wh2) +self.bh2)\n",
    "            \n",
    "            s_t = T.nnet.softmax(T.dot(h2_t, self.w2) + self.b2)\n",
    "            \n",
    "            return [h_t, h2_t, s_t]\n",
    "\n",
    "        # looping over the words in the sentence\n",
    "        [h, h2, s], _ = theano.scan(fn=recurrence, #first argument is the time sliced input\n",
    "                                # input\n",
    "                                sequences=x,\n",
    "                                # initial value, must have the same dimension as output [h_t, s_t]\n",
    "                                outputs_info=[self.h0, self.h20, None],\n",
    "                                # number of steps in the loop (corresponds to words in the sentence)\n",
    "                                n_steps=x.shape[0])\n",
    "\n",
    "        # output of the RNN is softmax result (probabilities)\n",
    "        p_y_given_x_sentence = s[:, 0, :]\n",
    "        \n",
    "        # for each word return the class with maximum probability\n",
    "        y_pred = T.argmax(p_y_given_x_sentence, axis=1)\n",
    "        \n",
    "\n",
    "        # cost and gradients and learning rate\n",
    "        lr = T.scalar('lr')\n",
    "\n",
    "        # cost function is the negative log-likelihood (NLL)\n",
    "        \n",
    "        \n",
    "        # x.shape[0] is (symbolically) the number of rows in x, i.e.,\n",
    "        # number of words in the sentence (call it n) in the minibatch\n",
    "        # T.arange(x.shape[0]) is a symbolic vector which will contain\n",
    "        # [0,1,2,... n-1] T.log(self.p_y_given_x_sentence) is a matrix of\n",
    "        # Log-Probabilities (call it LP) with one row per word and\n",
    "        # one column per class LP[T.arange(x.shape[0]),y_sentence] is a vector\n",
    "        # v containing [LP[0,y_sentence[0]], LP[1,y_sentence[1]], LP[2,y_sentence[2]], ...,\n",
    "        # LP[n-1,y_sentence[n-1]]] and T.mean(LP[T.arange(x.shape[0]),y_sentence]) is\n",
    "        # the mean (across minibatch examples) of the elements in v,\n",
    "        # i.e., the mean log-likelihood across the minibatch.\n",
    "        \n",
    "        sentence_nll = -T.mean(T.log(p_y_given_x_sentence)\n",
    "                               [T.arange(x.shape[0]), y_sentence])\n",
    "        \n",
    "        # now let theano do the ugly work and compute the SYMBOLIC gradients\n",
    "        sentence_gradients = T.grad(sentence_nll, self.params)\n",
    "        \n",
    "        # the updates is an ordered dicitionary, where each pair is a parameter and how it is updated (gradient descent)\n",
    "        sentence_updates = OrderedDict((p, p - lr*g)\n",
    "                                       for p, g in\n",
    "                                       zip(self.params, sentence_gradients))\n",
    "    \n",
    "\n",
    "        # theano functions to compile\n",
    "        \n",
    "        # classification function: \n",
    "        # INPUT: sentence (put in context windows)\n",
    "        # RETURN: vector (class for each word)\n",
    "        self.classify = theano.function(inputs=[idxs], outputs=y_pred)\n",
    "        \n",
    "        # training function:\n",
    "        # INPUT: sentence (put in context windows), class labels for each word (vector), and a learning rate\n",
    "        # RETURN: cost (negative log likelihood)\n",
    "        self.sentence_train = theano.function(inputs=[idxs, y_sentence, lr],\n",
    "                                              outputs=sentence_nll, \n",
    "                                              updates=sentence_updates)\n",
    "        \n",
    "        # normalization function (normalization of the embedding)\n",
    "        # update the embedding, such that it stays on the unit-ball (L2 norm equal to 1)\n",
    "        # dimshuffle(0, 'x') ->  make a column out of a 1d vector (N to Nx1)\n",
    "        self.normalize = theano.function(inputs=[],\n",
    "                                         updates={self.emb:\n",
    "                                                  self.emb /\n",
    "                                                  T.sqrt((self.emb**2)\n",
    "                                                  .sum(axis=1))\n",
    "                                                  .dimshuffle(0, 'x')})\n",
    "\n",
    "\n",
    "    def train(self, x, y, window_size, learning_rate):\n",
    "\n",
    "        cwords = contextwin(x, window_size)\n",
    "        words = map(lambda x: numpy.asarray(x).astype('int32'), cwords)\n",
    "        labels = y\n",
    "\n",
    "        self.sentence_train(words, labels, learning_rate)\n",
    "        self.normalize()\n",
    "\n",
    "    def save(self, folder):\n",
    "        for param in self.params:\n",
    "            numpy.save(os.path.join(folder,\n",
    "                       param.name + '.npy'), param.get_value())\n",
    "\n",
    "    def load(self, folder):\n",
    "        for param in self.params:\n",
    "            param.set_value(numpy.load(os.path.join(folder,\n",
    "                            param.name + '.npy')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Training parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "param = {\n",
    "    'fold': 3,\n",
    "    # 5 folds 0,1,2,3,4\n",
    "    'data': 'atis',\n",
    "    'lr': 0.0970806646812754,\n",
    "    'verbose': 1,\n",
    "    'decay': True,\n",
    "    # decay on the learning rate if improvement stops\n",
    "    'win': 5,\n",
    "    # number of words in the context window\n",
    "    'nhidden1': 100,\n",
    "    'nhidden2': 50,\n",
    "    'nfo': 300,\n",
    "    # number of hidden units\n",
    "    'seed': 345,\n",
    "    'emb_dimension': 10,\n",
    "    # dimension of word embedding\n",
    "    'nepochs': 80,\n",
    "    # 60 is recommended\n",
    "    'savemodel': False}\n",
    "print param\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "folder_name = \"RNN\"\n",
    "folder = os.path.join(PREFIX, folder_name)\n",
    "if not os.path.exists(folder):\n",
    "    os.mkdir(folder)\n",
    "\n",
    "\n",
    "\n",
    "# instanciate the model\n",
    "numpy.random.seed(param['seed'])\n",
    "random.seed(param['seed'])\n",
    "\n",
    "rnn = RNNSLU(nh=param['nhidden1'],\n",
    "             nfo=param['nfo'],\n",
    "             nh2=param['nhidden2'],\n",
    "             nc=nclasses,\n",
    "             ne=vocsize,\n",
    "             de=param['emb_dimension'],\n",
    "             cs=param['win'])\n",
    "\n",
    "# train with early stopping on validation set\n",
    "best_f1 = -numpy.inf\n",
    "param['clr'] = param['lr']\n",
    "for e in xrange(param['nepochs']):\n",
    "\n",
    "    # shuffle\n",
    "    shuffle([train_lex, train_ne, train_y], param['seed'])\n",
    "\n",
    "    param['ce'] = e\n",
    "    tic = timeit.default_timer()\n",
    "\n",
    "    # for each sentence in the training compute gradient and update\n",
    "    for i, (x, y) in enumerate(zip(train_lex, train_y)):\n",
    "        rnn.train(x, y, param['win'], param['clr'])\n",
    "        print '[learning] epoch %i >> %2.2f%%' % (\n",
    "            e, (i + 1) * 100. / nsentences),\n",
    "        print 'completed in %.2f (sec) <<\\r' % (timeit.default_timer() - tic),\n",
    "        sys.stdout.flush()\n",
    "\n",
    "    # evaluation // back into the real world : idx -> words\n",
    "    predictions_test = [map(lambda x: idx2label[x],\n",
    "                        rnn.classify(numpy.asarray(\n",
    "                        contextwin(x, param['win'])).astype('int32')))\n",
    "                        for x in test_lex]\n",
    "    predictions_valid = [map(lambda x: idx2label[x],\n",
    "                         rnn.classify(numpy.asarray(\n",
    "                         contextwin(x, param['win'])).astype('int32')))\n",
    "                         for x in valid_lex]\n",
    "\n",
    "    # evaluation // compute the accuracy using conlleval.pl\n",
    "    res_test = conlleval(predictions_test,\n",
    "                         groundtruth_test,\n",
    "                         words_test,\n",
    "                         folder + '/current.test.txt')\n",
    "    res_valid = conlleval(predictions_valid,\n",
    "                          groundtruth_valid,\n",
    "                          words_valid,\n",
    "                          folder + '/current.valid.txt')\n",
    "\n",
    "    if res_valid['f1'] > best_f1:\n",
    "\n",
    "        if param['savemodel']:\n",
    "            rnn.save(folder)\n",
    "\n",
    "        best_rnn = copy.deepcopy(rnn)\n",
    "        best_f1 = res_valid['f1']\n",
    "\n",
    "        if param['verbose']:\n",
    "            print('NEW BEST: epoch', e,\n",
    "                  'valid F1', res_valid['f1'],\n",
    "                  'best test F1', res_test['f1'])\n",
    "\n",
    "        param['vf1'], param['tf1'] = res_valid['f1'], res_test['f1']\n",
    "        param['vp'], param['tp'] = res_valid['p'], res_test['p']\n",
    "        param['vr'], param['tr'] = res_valid['r'], res_test['r']\n",
    "        param['be'] = e\n",
    "\n",
    "        subprocess.call(['mv', folder + '/current.test.txt',\n",
    "                        folder + '/best.test.txt'])\n",
    "        subprocess.call(['mv', folder + '/current.valid.txt',\n",
    "                        folder + '/best.valid.txt'])\n",
    "    else:\n",
    "        if param['verbose']:\n",
    "            print ''\n",
    "\n",
    "    # learning rate decay if no improvement in 10 epochs\n",
    "    if param['decay'] and abs(param['be']-param['ce']) >= 10:\n",
    "        param['clr'] *= 0.5\n",
    "        rnn = best_rnn\n",
    "\n",
    "    if param['clr'] < 1e-5:\n",
    "        break\n",
    "\n",
    "print('BEST RESULT: epoch', param['be'],\n",
    "      'valid F1', param['vf1'],\n",
    "      'best test F1', param['tf1'],\n",
    "      'with the model', folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now we have a look at the embedding of the words in the vector space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print rnn.emb.eval().shape\n",
    "# make sure the row-wise norm is equal to 1\n",
    "assert(np.sum(np.apply_along_axis(np.linalg.norm, 1, rnn.emb.eval())) == rnn.emb.eval().shape[0]),\"Embedding not properly normalized: Each row has to have norm 1.\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now we compute check the nearest neighbors for all elements for sanity check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nbrs = NearestNeighbors(n_neighbors=10, algorithm='ball_tree').fit(rnn.emb.eval())\n",
    "distances, indices = nbrs.kneighbors(rnn.emb.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# What are the different label categories?\n",
    "idx2word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# show the nearest neighbor to the word 'philadelphia'\n",
    "idx = 376\n",
    "[idx2word[x] for x in indices[idx]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# show the nearest neighbor to the word 'saturday'\n",
    "idx = 416\n",
    "[idx2word[x] for x in indices[idx]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# show the nearest neighbor to the word 'august'\n",
    "idx = 65\n",
    "[idx2word[x] for x in indices[idx]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# show the nearest neighbor to the word 'breakfast'\n",
    "idx = 80\n",
    "[idx2word[x] for x in indices[idx]]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}