RNN 196.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "#Based on tutorial found at: https://www.tensorflow.org/tutorials/text/text_generation\n",
    "\n",
    "#Importing required packages\n",
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "import numpy as np\n",
    "import os\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of text: 22443 characters\n",
      "81 unique characters\n"
     ]
    }
   ],
   "source": [
    "#Reading in previously scraped text\n",
    "text = ''\n",
    "text += open('reddit_scrape.txt', 'rb').read().decode(encoding='utf-8')\n",
    "\n",
    "#Provide information about scraped dataset\n",
    "print ('Length of text: {} characters'.format(len(text)))\n",
    "vocab = sorted(set(text))\n",
    "print ('{} unique characters'.format(len(vocab)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process The Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Vectorizing text\n",
    "char2idx = {u:i for i, u in enumerate(vocab)}\n",
    "idx2char = np.array(vocab)\n",
    "\n",
    "text_as_int = np.array([char2idx[c] for c in text])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given a character, or a sequence of characters, what is the most probable next character? This is the task we're training the model to perform. The input to the model will be a sequence of characters, and we train the model to predict the output—the following character at each time step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating training examples\n",
    "seq_length = 100\n",
    "examples_per_epoch = len(text)//seq_length\n",
    "\n",
    "# Create training examples / targets\n",
    "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n",
    "\n",
    "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Method to duplicate and shift to form input and target\n",
    "def split_input_target(chunk):\n",
    "    input_text = chunk[:-1]\n",
    "    target_text = chunk[1:]\n",
    "    return input_text, target_text\n",
    "\n",
    "dataset = sequences.map(split_input_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating training batches\n",
    "BATCH_SIZE = 64\n",
    "steps_per_epoch = examples_per_epoch//BATCH_SIZE\n",
    "\n",
    "BUFFER_SIZE = 10000\n",
    "\n",
    "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Length of the vocabulary in chars\n",
    "vocab_size = len(vocab)\n",
    "\n",
    "# The embedding dimension \n",
    "embedding_dim = 256\n",
    "\n",
    "# Number of RNN units\n",
    "rnn_units = 1024\n",
    "\n",
    "if tf.test.is_gpu_available():\n",
    "    rnn = tf.keras.layers.CuDNNGRU\n",
    "else:\n",
    "    import functools\n",
    "    rnn = functools.partial(tf.keras.layers.GRU, recurrent_activation='sigmoid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The function to build the RNN model\n",
    "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n",
    "    model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Embedding(vocab_size, embedding_dim, \n",
    "                              batch_input_shape=[batch_size, None]),\n",
    "    rnn(rnn_units,\n",
    "        return_sequences=True, \n",
    "        recurrent_initializer='glorot_uniform',\n",
    "        stateful=True),\n",
    "    tf.keras.layers.Dense(vocab_size)\n",
    "  ])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Calling build_model with all of our previously defined parameters\n",
    "model = build_model(\n",
    "  vocab_size = len(vocab), \n",
    "  embedding_dim=embedding_dim, \n",
    "  rnn_units=rnn_units, \n",
    "  batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At this point the problem can be treated as a standard classification problem. Given the previous RNN state, and the input this time step, predict the class of the next character."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss(labels, logits):\n",
    "    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n",
    "    #return tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)\n",
    "\n",
    "model.compile(optimizer = tf.train.AdamOptimizer(), loss = loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Directory where the checkpoints will be saved\n",
    "checkpoint_dir = './rnn_ckpt'\n",
    "\n",
    "# Name of the checkpoint files\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n",
    "\n",
    "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,save_weights_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Epochs indicate the amount of times all of the training vectors are used to update the weights in your model\n",
    "# The lower you set this number the faster the next cell will run, but you may find your model's performance lacking\n",
    "# There are diminishing returns with a large amount of epochs so the solution isn't neccessarily to set EPOCHS = int_max\n",
    "# Try experimenting with different amounts\n",
    "# The way this model is designed you can load the checkpoint from any epoch if you experience diminishing returns in loss\n",
    "EPOCHS=256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/256\n",
      "3/3 [==============================] - 3s 1s/step - loss: 4.3388\n",
      "Epoch 2/256\n",
      "3/3 [==============================] - 1s 266ms/step - loss: 7.0323\n",
      "Epoch 3/256\n",
      "3/3 [==============================] - 1s 275ms/step - loss: 4.0463\n",
      "Epoch 4/256\n",
      "3/3 [==============================] - 1s 483ms/step - loss: 4.1246\n",
      "Epoch 5/256\n",
      "3/3 [==============================] - 1s 361ms/step - loss: 4.0662\n",
      "Epoch 6/256\n",
      "3/3 [==============================] - 1s 291ms/step - loss: 3.9142\n",
      "Epoch 7/256\n",
      "3/3 [==============================] - 1s 379ms/step - loss: 3.7226\n",
      "Epoch 8/256\n",
      "3/3 [==============================] - 1s 229ms/step - loss: 3.5006\n",
      "Epoch 9/256\n",
      "3/3 [==============================] - 1s 315ms/step - loss: 3.1673\n",
      "Epoch 10/256\n",
      "3/3 [==============================] - 0s 139ms/step - loss: 3.0503\n",
      "Epoch 11/256\n",
      "3/3 [==============================] - 1s 210ms/step - loss: 2.9955\n",
      "Epoch 12/256\n",
      "3/3 [==============================] - 1s 304ms/step - loss: 2.9215\n",
      "Epoch 13/256\n",
      "3/3 [==============================] - 1s 299ms/step - loss: 2.8854\n",
      "Epoch 14/256\n",
      "3/3 [==============================] - 1s 311ms/step - loss: 2.8356\n",
      "Epoch 15/256\n",
      "3/3 [==============================] - 1s 227ms/step - loss: 2.7854\n",
      "Epoch 16/256\n",
      "3/3 [==============================] - 1s 230ms/step - loss: 2.7160\n",
      "Epoch 17/256\n",
      "3/3 [==============================] - 1s 302ms/step - loss: 2.6714\n",
      "Epoch 18/256\n",
      "3/3 [==============================] - 1s 214ms/step - loss: 2.6371\n",
      "Epoch 19/256\n",
      "3/3 [==============================] - 1s 309ms/step - loss: 2.5948\n",
      "Epoch 20/256\n",
      "3/3 [==============================] - 1s 296ms/step - loss: 2.5548\n",
      "Epoch 21/256\n",
      "3/3 [==============================] - 1s 310ms/step - loss: 2.5186\n",
      "Epoch 22/256\n",
      "3/3 [==============================] - 1s 214ms/step - loss: 2.4814\n",
      "Epoch 23/256\n",
      "3/3 [==============================] - 1s 218ms/step - loss: 2.4550\n",
      "Epoch 24/256\n",
      "3/3 [==============================] - 1s 289ms/step - loss: 2.4235\n",
      "Epoch 25/256\n",
      "3/3 [==============================] - 1s 215ms/step - loss: 2.4051\n",
      "Epoch 26/256\n",
      "3/3 [==============================] - 0s 135ms/step - loss: 2.3746\n",
      "Epoch 27/256\n",
      "3/3 [==============================] - 1s 314ms/step - loss: 2.3545\n",
      "Epoch 28/256\n",
      "3/3 [==============================] - 1s 208ms/step - loss: 2.3365\n",
      "Epoch 29/256\n",
      "3/3 [==============================] - 1s 322ms/step - loss: 2.3286\n",
      "Epoch 30/256\n",
      "3/3 [==============================] - 1s 239ms/step - loss: 2.3078\n",
      "Epoch 31/256\n",
      "3/3 [==============================] - 1s 225ms/step - loss: 2.2873\n",
      "Epoch 32/256\n",
      "3/3 [==============================] - 1s 226ms/step - loss: 2.2788\n",
      "Epoch 33/256\n",
      "3/3 [==============================] - 1s 233ms/step - loss: 2.2701\n",
      "Epoch 34/256\n",
      "3/3 [==============================] - 1s 223ms/step - loss: 2.2525\n",
      "Epoch 35/256\n",
      "3/3 [==============================] - 0s 137ms/step - loss: 2.2292\n",
      "Epoch 36/256\n",
      "3/3 [==============================] - 0s 135ms/step - loss: 2.2319\n",
      "Epoch 37/256\n",
      "3/3 [==============================] - 1s 245ms/step - loss: 2.2157\n",
      "Epoch 38/256\n",
      "3/3 [==============================] - 0s 132ms/step - loss: 2.1975\n",
      "Epoch 39/256\n",
      "3/3 [==============================] - 1s 254ms/step - loss: 2.1882\n",
      "Epoch 40/256\n",
      "3/3 [==============================] - 0s 127ms/step - loss: 2.1795\n",
      "Epoch 41/256\n",
      "3/3 [==============================] - 1s 235ms/step - loss: 2.1655\n",
      "Epoch 42/256\n",
      "3/3 [==============================] - 0s 143ms/step - loss: 2.1519\n",
      "Epoch 43/256\n",
      "3/3 [==============================] - 0s 162ms/step - loss: 2.1339\n",
      "Epoch 44/256\n",
      "3/3 [==============================] - 0s 131ms/step - loss: 2.1344\n",
      "Epoch 45/256\n",
      "3/3 [==============================] - 0s 141ms/step - loss: 2.1303\n",
      "Epoch 46/256\n",
      "3/3 [==============================] - 0s 127ms/step - loss: 2.1098\n",
      "Epoch 47/256\n",
      "3/3 [==============================] - 0s 166ms/step - loss: 2.1017\n",
      "Epoch 48/256\n",
      "3/3 [==============================] - 0s 144ms/step - loss: 2.0867\n",
      "Epoch 49/256\n",
      "3/3 [==============================] - 1s 269ms/step - loss: 2.0871\n",
      "Epoch 50/256\n",
      "3/3 [==============================] - 0s 123ms/step - loss: 2.0646\n",
      "Epoch 51/256\n",
      "3/3 [==============================] - 0s 146ms/step - loss: 2.0545\n",
      "Epoch 52/256\n",
      "3/3 [==============================] - 0s 131ms/step - loss: 2.0497\n",
      "Epoch 53/256\n",
      "3/3 [==============================] - 1s 248ms/step - loss: 2.0358\n",
      "Epoch 54/256\n",
      "3/3 [==============================] - 1s 212ms/step - loss: 2.0252\n",
      "Epoch 55/256\n",
      "3/3 [==============================] - 0s 140ms/step - loss: 2.0185\n",
      "Epoch 56/256\n",
      "3/3 [==============================] - 0s 131ms/step - loss: 1.9980\n",
      "Epoch 57/256\n",
      "3/3 [==============================] - 1s 358ms/step - loss: 1.9942\n",
      "Epoch 58/256\n",
      "3/3 [==============================] - 1s 253ms/step - loss: 1.9787\n",
      "Epoch 59/256\n",
      "3/3 [==============================] - 1s 249ms/step - loss: 1.9725\n",
      "Epoch 60/256\n",
      "3/3 [==============================] - 0s 121ms/step - loss: 1.9577\n",
      "Epoch 61/256\n",
      "3/3 [==============================] - 0s 142ms/step - loss: 1.9416\n",
      "Epoch 62/256\n",
      "3/3 [==============================] - 1s 208ms/step - loss: 1.9342\n",
      "Epoch 63/256\n",
      "3/3 [==============================] - 0s 126ms/step - loss: 1.9207\n",
      "Epoch 64/256\n",
      "3/3 [==============================] - 0s 132ms/step - loss: 1.9063\n",
      "Epoch 65/256\n",
      "3/3 [==============================] - 0s 151ms/step - loss: 1.8974\n",
      "Epoch 66/256\n",
      "3/3 [==============================] - 1s 211ms/step - loss: 1.8780\n",
      "Epoch 67/256\n",
      "3/3 [==============================] - 0s 132ms/step - loss: 1.8668\n",
      "Epoch 68/256\n",
      "3/3 [==============================] - 0s 135ms/step - loss: 1.8663\n",
      "Epoch 69/256\n",
      "3/3 [==============================] - 0s 125ms/step - loss: 1.8360\n",
      "Epoch 70/256\n",
      "3/3 [==============================] - 1s 222ms/step - loss: 1.8304\n",
      "Epoch 71/256\n",
      "3/3 [==============================] - 0s 126ms/step - loss: 1.8165\n",
      "Epoch 72/256\n",
      "3/3 [==============================] - 0s 139ms/step - loss: 1.8033\n",
      "Epoch 73/256\n",
      "3/3 [==============================] - 1s 206ms/step - loss: 1.7882\n",
      "Epoch 74/256\n",
      "3/3 [==============================] - 0s 149ms/step - loss: 1.7748\n",
      "Epoch 75/256\n",
      "3/3 [==============================] - 1s 237ms/step - loss: 1.7606\n",
      "Epoch 76/256\n",
      "3/3 [==============================] - 0s 128ms/step - loss: 1.7445\n",
      "Epoch 77/256\n",
      "3/3 [==============================] - 0s 142ms/step - loss: 1.7232\n",
      "Epoch 78/256\n",
      "3/3 [==============================] - 1s 209ms/step - loss: 1.7160\n",
      "Epoch 79/256\n",
      "3/3 [==============================] - 0s 133ms/step - loss: 1.6968\n",
      "Epoch 80/256\n",
      "3/3 [==============================] - 0s 130ms/step - loss: 1.6831\n",
      "Epoch 81/256\n",
      "3/3 [==============================] - 0s 132ms/step - loss: 1.6634\n",
      "Epoch 82/256\n",
      "3/3 [==============================] - 0s 151ms/step - loss: 1.6434\n",
      "Epoch 83/256\n",
      "3/3 [==============================] - 1s 224ms/step - loss: 1.6319\n",
      "Epoch 84/256\n",
      "3/3 [==============================] - 0s 151ms/step - loss: 1.6121\n",
      "Epoch 85/256\n",
      "3/3 [==============================] - 0s 138ms/step - loss: 1.5890\n",
      "Epoch 86/256\n",
      "3/3 [==============================] - 0s 129ms/step - loss: 1.5678\n",
      "Epoch 87/256\n",
      "3/3 [==============================] - 0s 124ms/step - loss: 1.5593\n",
      "Epoch 88/256\n",
      "3/3 [==============================] - 0s 128ms/step - loss: 1.5339\n",
      "Epoch 89/256\n",
      "3/3 [==============================] - 0s 142ms/step - loss: 1.5142\n",
      "Epoch 90/256\n",
      "3/3 [==============================] - 1s 186ms/step - loss: 1.4905\n",
      "Epoch 91/256\n",
      "3/3 [==============================] - 1s 182ms/step - loss: 1.4624\n",
      "Epoch 92/256\n",
      "3/3 [==============================] - 1s 189ms/step - loss: 1.4435\n",
      "Epoch 93/256\n",
      "3/3 [==============================] - 1s 235ms/step - loss: 1.4211\n",
      "Epoch 94/256\n",
      "3/3 [==============================] - 1s 217ms/step - loss: 1.4055\n",
      "Epoch 95/256\n",
      "3/3 [==============================] - 1s 183ms/step - loss: 1.3823\n",
      "Epoch 96/256\n",
      "3/3 [==============================] - 1s 186ms/step - loss: 1.3576\n",
      "Epoch 97/256\n",
      "3/3 [==============================] - 1s 192ms/step - loss: 1.3320\n",
      "Epoch 98/256\n",
      "3/3 [==============================] - 1s 211ms/step - loss: 1.2990\n",
      "Epoch 99/256\n",
      "3/3 [==============================] - 1s 335ms/step - loss: 1.2782\n",
      "Epoch 100/256\n",
      "3/3 [==============================] - 1s 190ms/step - loss: 1.2541\n",
      "Epoch 101/256\n",
      "3/3 [==============================] - 1s 182ms/step - loss: 1.2221\n",
      "Epoch 102/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 1.1928\n",
      "Epoch 103/256\n",
      "3/3 [==============================] - 1s 190ms/step - loss: 1.1675\n",
      "Epoch 104/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 1.1367\n",
      "Epoch 105/256\n",
      "3/3 [==============================] - 1s 182ms/step - loss: 1.1147\n",
      "Epoch 106/256\n",
      "3/3 [==============================] - 1s 190ms/step - loss: 1.0821\n",
      "Epoch 107/256\n",
      "3/3 [==============================] - 1s 195ms/step - loss: 1.0560\n",
      "Epoch 108/256\n",
      "3/3 [==============================] - 1s 190ms/step - loss: 1.0236\n",
      "Epoch 109/256\n",
      "3/3 [==============================] - 1s 187ms/step - loss: 1.0007\n",
      "Epoch 110/256\n",
      "3/3 [==============================] - 1s 198ms/step - loss: 0.9583\n",
      "Epoch 111/256\n",
      "3/3 [==============================] - 1s 220ms/step - loss: 0.9280\n",
      "Epoch 112/256\n",
      "3/3 [==============================] - 1s 211ms/step - loss: 0.8993\n",
      "Epoch 113/256\n",
      "3/3 [==============================] - 1s 208ms/step - loss: 0.8746\n",
      "Epoch 114/256\n",
      "3/3 [==============================] - 1s 226ms/step - loss: 0.8326\n",
      "Epoch 115/256\n",
      "3/3 [==============================] - 1s 218ms/step - loss: 0.8098\n",
      "Epoch 116/256\n",
      "3/3 [==============================] - 1s 206ms/step - loss: 0.7850\n",
      "Epoch 117/256\n",
      "3/3 [==============================] - 1s 213ms/step - loss: 0.7594\n",
      "Epoch 118/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 0.7253\n",
      "Epoch 119/256\n",
      "3/3 [==============================] - 1s 208ms/step - loss: 0.6972\n",
      "Epoch 120/256\n",
      "3/3 [==============================] - 1s 204ms/step - loss: 0.6663\n",
      "Epoch 121/256\n",
      "3/3 [==============================] - 1s 208ms/step - loss: 0.6359\n",
      "Epoch 122/256\n",
      "3/3 [==============================] - 1s 218ms/step - loss: 0.6129\n",
      "Epoch 123/256\n",
      "3/3 [==============================] - 1s 193ms/step - loss: 0.5943\n",
      "Epoch 124/256\n",
      "3/3 [==============================] - 1s 211ms/step - loss: 0.5653\n",
      "Epoch 125/256\n",
      "3/3 [==============================] - 1s 185ms/step - loss: 0.5415\n",
      "Epoch 126/256\n",
      "3/3 [==============================] - 1s 217ms/step - loss: 0.5177\n",
      "Epoch 127/256\n",
      "3/3 [==============================] - 1s 188ms/step - loss: 0.4880\n",
      "Epoch 128/256\n",
      "3/3 [==============================] - 1s 203ms/step - loss: 0.4728\n",
      "Epoch 129/256\n",
      "3/3 [==============================] - 1s 272ms/step - loss: 0.4521\n",
      "Epoch 130/256\n",
      "3/3 [==============================] - 1s 206ms/step - loss: 0.4255\n",
      "Epoch 131/256\n",
      "3/3 [==============================] - 1s 189ms/step - loss: 0.4146\n",
      "Epoch 132/256\n",
      "3/3 [==============================] - 1s 220ms/step - loss: 0.3997\n",
      "Epoch 133/256\n",
      "3/3 [==============================] - 1s 237ms/step - loss: 0.3851\n",
      "Epoch 134/256\n",
      "3/3 [==============================] - 1s 217ms/step - loss: 0.3699\n",
      "Epoch 135/256\n",
      "3/3 [==============================] - 1s 200ms/step - loss: 0.3541\n",
      "Epoch 136/256\n",
      "3/3 [==============================] - 1s 206ms/step - loss: 0.3356\n",
      "Epoch 137/256\n",
      "3/3 [==============================] - 1s 208ms/step - loss: 0.3256\n",
      "Epoch 138/256\n",
      "3/3 [==============================] - 1s 241ms/step - loss: 0.3177\n",
      "Epoch 139/256\n",
      "3/3 [==============================] - 1s 205ms/step - loss: 0.3028\n",
      "Epoch 140/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 0.2955\n",
      "Epoch 141/256\n",
      "3/3 [==============================] - 1s 189ms/step - loss: 0.2884\n",
      "Epoch 142/256\n",
      "3/3 [==============================] - 1s 211ms/step - loss: 0.2788\n",
      "Epoch 143/256\n",
      "3/3 [==============================] - 1s 235ms/step - loss: 0.2687\n",
      "Epoch 144/256\n",
      "3/3 [==============================] - 1s 198ms/step - loss: 0.2596\n",
      "Epoch 145/256\n",
      "3/3 [==============================] - 1s 179ms/step - loss: 0.2558\n",
      "Epoch 146/256\n",
      "3/3 [==============================] - 1s 192ms/step - loss: 0.2450\n",
      "Epoch 147/256\n",
      "3/3 [==============================] - 1s 210ms/step - loss: 0.2437\n",
      "Epoch 148/256\n",
      "3/3 [==============================] - 1s 189ms/step - loss: 0.2365\n",
      "Epoch 149/256\n",
      "3/3 [==============================] - 1s 209ms/step - loss: 0.2290\n",
      "Epoch 150/256\n",
      "3/3 [==============================] - 1s 186ms/step - loss: 0.2262\n",
      "Epoch 151/256\n",
      "3/3 [==============================] - 1s 186ms/step - loss: 0.2202\n",
      "Epoch 152/256\n",
      "3/3 [==============================] - 1s 194ms/step - loss: 0.2159\n",
      "Epoch 153/256\n",
      "3/3 [==============================] - 1s 192ms/step - loss: 0.2158\n",
      "Epoch 154/256\n",
      "3/3 [==============================] - 1s 186ms/step - loss: 0.2120\n",
      "Epoch 155/256\n",
      "3/3 [==============================] - 1s 197ms/step - loss: 0.2018\n",
      "Epoch 156/256\n",
      "3/3 [==============================] - 1s 205ms/step - loss: 0.1984\n",
      "Epoch 157/256\n",
      "3/3 [==============================] - 1s 185ms/step - loss: 0.1966\n",
      "Epoch 158/256\n",
      "3/3 [==============================] - 1s 182ms/step - loss: 0.1917\n",
      "Epoch 159/256\n",
      "3/3 [==============================] - 1s 205ms/step - loss: 0.1931\n",
      "Epoch 160/256\n",
      "3/3 [==============================] - 1s 217ms/step - loss: 0.1824\n",
      "Epoch 161/256\n",
      "3/3 [==============================] - 1s 198ms/step - loss: 0.1861\n",
      "Epoch 162/256\n",
      "3/3 [==============================] - 1s 228ms/step - loss: 0.1837\n",
      "Epoch 163/256\n",
      "3/3 [==============================] - 1s 191ms/step - loss: 0.1779\n",
      "Epoch 164/256\n",
      "3/3 [==============================] - 1s 191ms/step - loss: 0.1721\n",
      "Epoch 165/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 0.1777\n",
      "Epoch 166/256\n",
      "3/3 [==============================] - 1s 209ms/step - loss: 0.1713\n",
      "Epoch 167/256\n",
      "3/3 [==============================] - 1s 191ms/step - loss: 0.1683\n",
      "Epoch 168/256\n",
      "3/3 [==============================] - 1s 191ms/step - loss: 0.1664\n",
      "Epoch 169/256\n",
      "3/3 [==============================] - 1s 205ms/step - loss: 0.1633\n",
      "Epoch 170/256\n",
      "3/3 [==============================] - 1s 210ms/step - loss: 0.1590\n",
      "Epoch 171/256\n",
      "3/3 [==============================] - 1s 192ms/step - loss: 0.1550\n",
      "Epoch 172/256\n",
      "3/3 [==============================] - 1s 278ms/step - loss: 0.1626\n",
      "Epoch 173/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 0.1589\n",
      "Epoch 174/256\n",
      "3/3 [==============================] - 1s 217ms/step - loss: 0.1519\n",
      "Epoch 175/256\n",
      "3/3 [==============================] - 1s 194ms/step - loss: 0.1526\n",
      "Epoch 176/256\n",
      "3/3 [==============================] - 1s 213ms/step - loss: 0.1540\n",
      "Epoch 177/256\n",
      "3/3 [==============================] - 1s 194ms/step - loss: 0.1529\n",
      "Epoch 178/256\n",
      "3/3 [==============================] - 1s 210ms/step - loss: 0.1527\n",
      "Epoch 179/256\n",
      "3/3 [==============================] - 1s 188ms/step - loss: 0.1496\n",
      "Epoch 180/256\n",
      "3/3 [==============================] - 1s 233ms/step - loss: 0.1475\n",
      "Epoch 181/256\n",
      "3/3 [==============================] - 1s 186ms/step - loss: 0.1464\n",
      "Epoch 182/256\n",
      "3/3 [==============================] - 1s 212ms/step - loss: 0.1422\n",
      "Epoch 183/256\n",
      "3/3 [==============================] - 1s 185ms/step - loss: 0.1385\n",
      "Epoch 184/256\n",
      "3/3 [==============================] - 1s 185ms/step - loss: 0.1391\n",
      "Epoch 185/256\n",
      "3/3 [==============================] - 1s 203ms/step - loss: 0.1439\n",
      "Epoch 186/256\n",
      "3/3 [==============================] - 1s 183ms/step - loss: 0.1390\n",
      "Epoch 187/256\n",
      "3/3 [==============================] - 1s 209ms/step - loss: 0.1414\n",
      "Epoch 188/256\n",
      "3/3 [==============================] - 1s 258ms/step - loss: 0.1361\n",
      "Epoch 189/256\n",
      "3/3 [==============================] - 1s 200ms/step - loss: 0.1354\n",
      "Epoch 190/256\n",
      "3/3 [==============================] - 1s 234ms/step - loss: 0.1349\n",
      "Epoch 191/256\n",
      "3/3 [==============================] - 1s 212ms/step - loss: 0.1304\n",
      "Epoch 192/256\n",
      "3/3 [==============================] - 1s 200ms/step - loss: 0.1296\n",
      "Epoch 193/256\n",
      "3/3 [==============================] - 1s 198ms/step - loss: 0.1348\n",
      "Epoch 194/256\n",
      "3/3 [==============================] - 1s 189ms/step - loss: 0.1286\n",
      "Epoch 195/256\n",
      "3/3 [==============================] - 1s 195ms/step - loss: 0.1318\n",
      "Epoch 196/256\n",
      "3/3 [==============================] - 1s 184ms/step - loss: 0.1313\n",
      "Epoch 197/256\n",
      "3/3 [==============================] - 1s 214ms/step - loss: 0.1263\n",
      "Epoch 198/256\n",
      "3/3 [==============================] - 1s 220ms/step - loss: 0.1251\n",
      "Epoch 199/256\n",
      "3/3 [==============================] - 1s 193ms/step - loss: 0.1242\n",
      "Epoch 200/256\n",
      "3/3 [==============================] - 1s 186ms/step - loss: 0.1245\n",
      "Epoch 201/256\n",
      "3/3 [==============================] - 1s 211ms/step - loss: 0.1214\n",
      "Epoch 202/256\n",
      "3/3 [==============================] - 1s 208ms/step - loss: 0.1214\n",
      "Epoch 203/256\n",
      "3/3 [==============================] - 1s 205ms/step - loss: 0.1211\n",
      "Epoch 204/256\n",
      "3/3 [==============================] - 1s 200ms/step - loss: 0.1246\n",
      "Epoch 205/256\n",
      "3/3 [==============================] - 1s 190ms/step - loss: 0.1219\n",
      "Epoch 206/256\n",
      "3/3 [==============================] - 1s 219ms/step - loss: 0.1216\n",
      "Epoch 207/256\n",
      "3/3 [==============================] - 1s 179ms/step - loss: 0.1174\n",
      "Epoch 208/256\n",
      "3/3 [==============================] - 1s 197ms/step - loss: 0.1189\n",
      "Epoch 209/256\n",
      "3/3 [==============================] - 1s 228ms/step - loss: 0.1178\n",
      "Epoch 210/256\n",
      "3/3 [==============================] - 1s 183ms/step - loss: 0.1148\n",
      "Epoch 211/256\n",
      "3/3 [==============================] - 1s 180ms/step - loss: 0.1091\n",
      "Epoch 212/256\n",
      "3/3 [==============================] - 1s 181ms/step - loss: 0.1178\n",
      "Epoch 213/256\n",
      "3/3 [==============================] - 1s 227ms/step - loss: 0.1157\n",
      "Epoch 214/256\n",
      "3/3 [==============================] - 1s 188ms/step - loss: 0.1110\n",
      "Epoch 215/256\n",
      "3/3 [==============================] - 1s 209ms/step - loss: 0.1142\n",
      "Epoch 216/256\n",
      "3/3 [==============================] - 1s 201ms/step - loss: 0.1135\n",
      "Epoch 217/256\n",
      "3/3 [==============================] - 1s 252ms/step - loss: 0.1117\n",
      "Epoch 218/256\n",
      "3/3 [==============================] - 1s 206ms/step - loss: 0.1068\n",
      "Epoch 219/256\n",
      "3/3 [==============================] - 1s 215ms/step - loss: 0.1060\n",
      "Epoch 220/256\n",
      "3/3 [==============================] - 1s 193ms/step - loss: 0.1055\n",
      "Epoch 221/256\n",
      "3/3 [==============================] - 1s 202ms/step - loss: 0.1022\n",
      "Epoch 222/256\n",
      "3/3 [==============================] - 1s 202ms/step - loss: 0.1021\n",
      "Epoch 223/256\n",
      "3/3 [==============================] - 1s 194ms/step - loss: 0.1033\n",
      "Epoch 224/256\n",
      "3/3 [==============================] - 1s 190ms/step - loss: 0.1041\n",
      "Epoch 225/256\n",
      "3/3 [==============================] - 1s 219ms/step - loss: 0.1042\n",
      "Epoch 226/256\n",
      "3/3 [==============================] - 1s 240ms/step - loss: 0.0994\n",
      "Epoch 227/256\n",
      "3/3 [==============================] - 1s 188ms/step - loss: 0.1040\n",
      "Epoch 228/256\n",
      "3/3 [==============================] - 1s 224ms/step - loss: 0.1024\n",
      "Epoch 229/256\n",
      "3/3 [==============================] - 1s 187ms/step - loss: 0.1006\n",
      "Epoch 230/256\n",
      "3/3 [==============================] - 1s 189ms/step - loss: 0.1033\n",
      "Epoch 231/256\n",
      "3/3 [==============================] - 1s 183ms/step - loss: 0.1061\n",
      "Epoch 232/256\n",
      "3/3 [==============================] - 1s 188ms/step - loss: 0.0968\n",
      "Epoch 233/256\n",
      "3/3 [==============================] - 1s 180ms/step - loss: 0.0998\n",
      "Epoch 234/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 0.1000\n",
      "Epoch 235/256\n",
      "3/3 [==============================] - 1s 194ms/step - loss: 0.0999\n",
      "Epoch 236/256\n",
      "3/3 [==============================] - 1s 187ms/step - loss: 0.0984\n",
      "Epoch 237/256\n",
      "3/3 [==============================] - 1s 203ms/step - loss: 0.0974\n",
      "Epoch 238/256\n",
      "3/3 [==============================] - 1s 196ms/step - loss: 0.0979\n",
      "Epoch 239/256\n",
      "3/3 [==============================] - 1s 209ms/step - loss: 0.0929\n",
      "Epoch 240/256\n",
      "3/3 [==============================] - 1s 188ms/step - loss: 0.0959\n",
      "Epoch 241/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 0.0954\n",
      "Epoch 242/256\n",
      "3/3 [==============================] - 1s 280ms/step - loss: 0.0937\n",
      "Epoch 243/256\n",
      "3/3 [==============================] - 1s 196ms/step - loss: 0.0938\n",
      "Epoch 244/256\n",
      "3/3 [==============================] - 1s 252ms/step - loss: 0.0967\n",
      "Epoch 245/256\n",
      "3/3 [==============================] - 1s 185ms/step - loss: 0.0982\n",
      "Epoch 246/256\n",
      "3/3 [==============================] - 1s 193ms/step - loss: 0.0904\n",
      "Epoch 247/256\n",
      "3/3 [==============================] - 1s 188ms/step - loss: 0.0964\n",
      "Epoch 248/256\n",
      "3/3 [==============================] - 1s 210ms/step - loss: 0.0896\n",
      "Epoch 249/256\n",
      "3/3 [==============================] - 1s 205ms/step - loss: 0.0950\n",
      "Epoch 250/256\n",
      "3/3 [==============================] - 1s 207ms/step - loss: 0.0926\n",
      "Epoch 251/256\n",
      "3/3 [==============================] - 1s 201ms/step - loss: 0.0930\n",
      "Epoch 252/256\n",
      "3/3 [==============================] - 1s 189ms/step - loss: 0.0917\n",
      "Epoch 253/256\n",
      "3/3 [==============================] - 1s 204ms/step - loss: 0.0913\n",
      "Epoch 254/256\n",
      "3/3 [==============================] - 1s 185ms/step - loss: 0.0933\n",
      "Epoch 255/256\n",
      "3/3 [==============================] - 1s 186ms/step - loss: 0.0897\n",
      "Epoch 256/256\n",
      "3/3 [==============================] - 1s 221ms/step - loss: 0.0883\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(dataset.repeat(), epochs=EPOCHS, steps_per_epoch=steps_per_epoch, callbacks=[checkpoint_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./rnn_ckpt/ckpt_256'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.train.latest_checkpoint(checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (1, None, 256)            20736     \n",
      "_________________________________________________________________\n",
      "cu_dnngru_1 (CuDNNGRU)       (1, None, 1024)           3938304   \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (1, None, 81)             83025     \n",
      "=================================================================\n",
      "Total params: 4,042,065\n",
      "Trainable params: 4,042,065\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n",
    "\n",
    "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n",
    "\n",
    "model.build(tf.TensorShape([1, None]))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num_generate is the number of characters you'd like to generate\n",
    "# model is the model we've defined, this is simply passed in as \"model\"\n",
    "# start_string is what kicks off our RNN, setting an initial state to predict from\n",
    "def generate_text(num_generate, model, start_string):\n",
    "  # Evaluation step (generating text using the learned model)\n",
    "\n",
    "  # Converting our start string to numbers (vectorizing) \n",
    "    input_eval = [char2idx[s] for s in start_string]\n",
    "    input_eval = tf.expand_dims(input_eval, 0)\n",
    "\n",
    "  # Empty string to store our results\n",
    "    text_generated = []\n",
    "\n",
    "  # Low temperatures results in more predictable text.\n",
    "  # Higher temperatures results in more surprising text.\n",
    "  # Experiment to find the best setting.\n",
    "    temperature = 1.0\n",
    "\n",
    "  # Here batch size == 1\n",
    "    model.reset_states()\n",
    "    for i in range(num_generate):\n",
    "        predictions = model(input_eval)\n",
    "      # remove the batch dimension\n",
    "        predictions = tf.squeeze(predictions, 0)\n",
    "\n",
    "      # using a multinomial distribution to predict the word returned by the model\n",
    "        predictions = predictions / temperature\n",
    "        predicted_id = tf.multinomial(predictions, num_samples=1)[-1,0].numpy()\n",
    "      \n",
    "      # We pass the predicted word as the next input to the model\n",
    "      # along with the previous hidden state\n",
    "        input_eval = tf.expand_dims([predicted_id], 0)\n",
    "      \n",
    "        text_generated.append(idx2char[predicted_id])\n",
    "\n",
    "    return (start_string + ''.join(text_generated))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-16-8660c366e4ad>:28: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.random.categorical` instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'[WP]IFOR] AT THE MOON\". You plepsed and takes over and Jack drives home at night, and accidentally runs over another Jack, a plumber. The Skviors round years, mice for on the apocollpscellice you lan a lifetime spacily the lang smowary rearons. Aporay of'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_text(250, model, start_string=u\"[WP]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open(\"Generated_reddit.txt\",\"a\")\n",
    "file.write(generate_text(65000, model, start_string=u\"[WP]\"))\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'random' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-68a7a958b8ee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m25\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m250\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_string\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34mu\"[WP]\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'\\n'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'random' is not defined"
     ]
    }
   ],
   "source": [
    "file = open(\"reddit_posts.txt\",\"a\")\n",
    "i=0\n",
    "while i < 25:\n",
    "    text = generate_text(random.randint(30,250), model, start_string=u\"[WP]\")\n",
    "    text = text + '\\n'\n",
    "    file.write(text)\n",
    "    i+=1\n",
    "\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}