rnn_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Implements a Recurrent Neural Netweork with the\n",
    "recurrent weight matrix intialized by identity matrix\n",
    "\n",
    "Optimizer used: RMSprop\n",
    "Loss function: categorical_crossentropy\n",
    "Reference: http://arxiv.org/pdf/1504.00941v2.pdf\n",
    "\"\"\"\n",
    "\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers.recurrent import SimpleRNN\n",
    "from keras.layers.core import Dense, Activation\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.initializations import identity, normal \n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# architecture details\n",
    "output_classes = 10\n",
    "hidden_units = 100\n",
    "\n",
    "# learning rate parameter\n",
    "learning_rate = 1e-6\n",
    "\n",
    "# running details\n",
    "num_epochs = 900\n",
    "batch_size = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load the mnist data and split it between train and test sets\n",
    "# train set: 60,000 examples (28 x 28 images)\n",
    "# test set: 10,000 examples (28 x 28 images)\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "# Flatten the examples to a vector of length 28 x 28 = 784\n",
    "X_train = X_train.reshape(X_train.shape[0], -1, 1)\n",
    "X_test = X_test.reshape(X_test.shape[0], -1, 1)\n",
    "\n",
    "# Convert the data type to float before normalizing\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "\n",
    "# Normalize the pixel values such \n",
    "# that they lie between 0 to 1\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "# convert class labels to binary class vectors \n",
    "# eg., 2 -> [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]\n",
    "Y_train = np_utils.to_categorical(y_train, output_classes)\n",
    "Y_test = np_utils.to_categorical(y_test, output_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Build the sequential model\n",
    "model = Sequential()\n",
    "model.add(SimpleRNN(output_dim=hidden_units,\n",
    "                    init=lambda shape, name: normal(shape, scale=0.001, name=name),\n",
    "                    inner_init=lambda shape, name: identity(shape, scale=1.0, name=name),\n",
    "                    activation='relu',\n",
    "                    input_shape=X_train.shape[1:]))\n",
    "model.add(Dense(output_classes))\n",
    "model.add(Activation('softmax'))\n",
    "rmsprop = RMSprop(lr=learning_rate)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=rmsprop, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                       Output Shape        Param #     Connected to                     \n",
      "====================================================================================================\n",
      "simplernn_4 (SimpleRNN)            (None, 100)         10200       simplernn_input_4[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "dense_7 (Dense)                    (None, 10)          1010        simplernn_4[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "activation_7 (Activation)          (None, 10)          0           dense_7[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 11210\n",
      "____________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print (model.summary())\n",
    "#print (model.to_json())\n",
    "#print (model.get_config())\n",
    "#print (model.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('RNN Model Evaluation:')\n",
    "\n",
    "# Train the model for a fixed number of epochs\n",
    "model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=num_epochs, verbose=1, \n",
    "            validation_data=(X_test, Y_test))\n",
    "\n",
    "# Compute the loss on the input data, batch by batch\n",
    "scores = model.evaluate(X_test, Y_test, verbose=0)\n",
    "\n",
    "print('RNN Model Test score:', scores[0])\n",
    "print('RNN Model Test accuracy:', scores[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}