lab-12-0-rnn-basics-keras-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-0 rnn basics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "\n",
    "print(tf.__version__)\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One hot encoding for each char in 'hello'\n",
    "h = [1, 0, 0, 0]\n",
    "e = [0, 1, 0, 0]\n",
    "l = [0, 0, 1, 0]\n",
    "o = [0, 0, 0, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One cell: 4 (input_dim) in 2 (hidden_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23348727/cc981856-fce7-11e6-83ea-4b187473466b.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_data: [[[1. 0. 0. 0.]]], shape: (1, 1, 4)\n",
      "outputs: [[[0.32261637 0.5036928 ]]], shape: (1, 1, 2)\n",
      "states: [[0.32261637 0.5036928 ]], shape: (1, 2)\n"
     ]
    }
   ],
   "source": [
    "# One cell RNN input_dim (4) -> output_dim (2)\n",
    "x_data = np.array([[h]], dtype=np.float32)\n",
    "\n",
    "hidden_size = 2\n",
    "cell = layers.SimpleRNNCell(units=hidden_size) # creating SimpleRNNCell\n",
    "rnn = layers.RNN(cell, return_sequences=True, return_state=True) # analogous to tf.nn.dynamic_rnn\n",
    "outputs, states = rnn(x_data)\n",
    "\n",
    "print('x_data: {}, shape: {}'.format(x_data, x_data.shape))\n",
    "print('outputs: {}, shape: {}'.format(outputs, outputs.shape))\n",
    "print('states: {}, shape: {}'.format(states, states.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_data: [[[1. 0. 0. 0.]]], shape: (1, 1, 4)\n",
      "outputs: [[[-0.5746312  0.6728443]]], shape: (1, 1, 2)\n",
      "states: [[-0.5746312  0.6728443]], shape: (1, 2)\n"
     ]
    }
   ],
   "source": [
    "# equivalent to above case\n",
    "rnn = layers.SimpleRNN(units=hidden_size, return_sequences=True,\n",
    "                       return_state=True) # layers.SimpleRNNCell + layers.RNN\n",
    "\n",
    "outputs, states = rnn(x_data)\n",
    "\n",
    "print('x_data: {}, shape: {}'.format(x_data, x_data.shape))\n",
    "print('outputs: {}, shape: {}'.format(outputs, outputs.shape))\n",
    "print('states: {}, shape: {}'.format(states, states.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unfolding to n sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23383634/649efd0a-fd82-11e6-925d-8041242743b0.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_data: [[[1. 0. 0. 0.]\n",
      "  [0. 1. 0. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 0. 1.]]], shape: (1, 5, 4) \n",
      "\n",
      "outputs: [[[ 0.36337885  0.73452437]\n",
      "  [ 0.23541063 -0.28505793]\n",
      "  [-0.19638212 -0.54805404]\n",
      "  [-0.589804   -0.65221256]\n",
      "  [-0.8427679   0.19108507]]], shape: (1, 5, 2) \n",
      "\n",
      "states: [[-0.8427679   0.19108507]], shape: (1, 2)\n"
     ]
    }
   ],
   "source": [
    "# One cell RNN input_dim (4) -> output_dim (2). sequence: 5\n",
    "x_data = np.array([[h, e, l, l, o]], dtype=np.float32)\n",
    "\n",
    "hidden_size = 2\n",
    "rnn = layers.SimpleRNN(units=2, return_sequences=True, return_state=True)    \n",
    "outputs, states = rnn(x_data)\n",
    "\n",
    "print('x_data: {}, shape: {} \\n'.format(x_data, x_data.shape))\n",
    "print('outputs: {}, shape: {} \\n'.format(outputs, outputs.shape))\n",
    "print('states: {}, shape: {}'.format(states, states.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batching input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23383681/9943a9fc-fd82-11e6-8121-bd187994e249.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_data: [[[1. 0. 0. 0.]\n",
      "  [0. 1. 0. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 0. 1.]]\n",
      "\n",
      " [[0. 1. 0. 0.]\n",
      "  [0. 0. 0. 1.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]]\n",
      "\n",
      " [[0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 1. 0. 0.]\n",
      "  [0. 1. 0. 0.]\n",
      "  [0. 0. 1. 0.]]], shape: (3, 5, 4) \n",
      "\n",
      "outputs: [[[-0.56743866 -0.23173441]\n",
      "  [ 0.8334968   0.30382484]\n",
      "  [-0.93795335 -0.53330624]\n",
      "  [-0.0874121   0.45240858]\n",
      "  [-0.52966034 -0.5007928 ]]\n",
      "\n",
      " [[ 0.58572567  0.00302648]\n",
      "  [-0.8668559  -0.19943324]\n",
      "  [-0.10968345  0.14605513]\n",
      "  [-0.6695559  -0.29440066]\n",
      "  [-0.30860662  0.20964848]]\n",
      "\n",
      " [[-0.73496014 -0.17280702]\n",
      "  [-0.23251031  0.10149368]\n",
      "  [ 0.723902   -0.06399264]\n",
      "  [-0.0544093  -0.03742061]\n",
      "  [-0.7118617  -0.12900047]]], shape: (3, 5, 2) \n",
      "\n",
      "states: [[-0.52966034 -0.5007928 ]\n",
      " [-0.30860662  0.20964848]\n",
      " [-0.7118617  -0.12900047]], shape: (3, 2)\n"
     ]
    }
   ],
   "source": [
    "# One cell RNN input_dim (4) -> output_dim (2). sequence: 5, batch 3\n",
    "# 3 batches 'hello', 'eolll', 'lleel'\n",
    "x_data = np.array([[h, e, l, l, o],\n",
    "                   [e, o, l, l, l],\n",
    "                   [l, l, e, e, l]], dtype=np.float32)\n",
    "\n",
    "hidden_size = 2\n",
    "rnn = layers.SimpleRNN(units=2, return_sequences=True, return_state=True)    \n",
    "outputs, states = rnn(x_data)\n",
    "\n",
    "print('x_data: {}, shape: {} \\n'.format(x_data, x_data.shape))\n",
    "print('outputs: {}, shape: {} \\n'.format(outputs, outputs.shape))\n",
    "print('states: {}, shape: {}'.format(states, states.shape))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}