6.2 - Parameter Search With Custom Grid Search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run the Bottom Code before resetting the thing!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (aver.min(), str(grid_result['parameters'][np.argwhere(aver == aver.min())]  )))\n",
    "\n",
    "print(\"Average Rsults\")\n",
    "# prints all of the results\n",
    "for i in range(grid_results.size):\n",
    "    params = grid_results[i]['parameters']\n",
    "    for l, p in zip(aver, params):\n",
    "        print l, p\n",
    "\n",
    "print(\"All Results\")\n",
    "# prints all of the results\n",
    "for i in range(grid_results.size):\n",
    "    loss = grid_results[i]['results']\n",
    "    params = grid_results[i]['parameters']\n",
    "    for l, p in zip(loss, params):\n",
    "        print l, p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# imports\n",
    "import numpy             as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas            as pd\n",
    "import math\n",
    "import keras\n",
    "from scipy.stats  import norm\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, LSTM, Dropout\n",
    "from keras        import backend as K\n",
    "from datetime     import datetime\n",
    "from keras.optimizers      import RMSprop\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from matplotlib.pylab      import rcParams\n",
    "%matplotlib inline\n",
    "rcParams['figure.figsize'] = 15, 6\n",
    "\n",
    "# number of inputs to be fed\n",
    "look_back    = 288\n",
    "# number of outputs to be generated\n",
    "look_forward = 24\n",
    "# the number of stations\n",
    "stations     = 66"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepares the Dataset For the Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The interval between each dataset (original data in 5 minute intervals)\n",
    "time_grouping = '5min'\n",
    "# load the dataset\n",
    "dataframe = pd.read_csv('ignored_assets/paxout_table.csv', engine='python', nrows=288*21)\n",
    "dataframe['time_bucket'] = pd.to_datetime(dataframe['time_bucket'])\n",
    "dataframe = dataframe.set_index('time_bucket')\n",
    "# dataframe['total'] = dataframe.sum(axis=1)\n",
    "dataframe['day_hour'] = dataframe.index.round(time_grouping)\n",
    "dataframe = dataframe.groupby('day_hour').sum()\n",
    "# removes the timestamp at column 67\n",
    "dataset_orig = dataframe.values[:,:stations]\n",
    "dataset_orig = dataset_orig.astype('float32')\n",
    "# scale the dataset\n",
    "scaler = MinMaxScaler(feature_range=(0, 1))\n",
    "dataset = scaler.fit_transform(dataset_orig)\n",
    "\n",
    "# convert an array of values into a dataset matrix, adjusted to make a dateset that is 66 wide\n",
    "def create_dataset(dataset, look_back=1, look_forward=2):\n",
    "    dataX, dataY = [], []\n",
    "    np.array(dataY)\n",
    "    for i in range(len(dataset)-look_back-1):\n",
    "        a = dataset[i:(i+look_back),:]\n",
    "        # Makes sure that the array isn't the last 2 or 3 or whatever bits. It has to be the full 24\n",
    "        if len(dataset[i + look_back:i+look_back+look_forward, 0]) == look_forward:\n",
    "            dataX.append(a.T)\n",
    "            dataY.append(dataset[i + look_back:i+look_back+look_forward, :].T)\n",
    "    return np.array(dataX), np.array(dataY)\n",
    "\n",
    "# split into train and test sets\n",
    "train_size = 288*14\n",
    "test_size = len(dataset) - train_size\n",
    "train, test = dataset[0:train_size,:], dataset[train_size:len(dataset)-look_forward,:]\n",
    "\n",
    "# reshape into X=[t, t-1, t-2,..., t-look_back] and Y=[t+1, t+2,... t+look_forward]\n",
    "trainX, trainY = create_dataset(train, look_back, look_forward)\n",
    "trainX, trainY = create_dataset(test, look_back, look_forward)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Loss Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def top_heavy_loss (y_true, y_pred):\n",
    "    w = np.arange(1,0, -1./24) ** 2\n",
    "    w = (w / w.sum())[:,None]\n",
    "    W = K.variable(value = w)\n",
    "    return K.dot( K.abs(y_pred-y_true), W)\n",
    "\n",
    "def top_heavy_tail_heavy (y_true, y_pred):\n",
    "    # This is the normal weights that decrease\n",
    "    # w = np.arange(1, 0, -1./24)**1.3\n",
    "    # This is the Weights with one difference, The last is weighted as equally as the first\n",
    "    w = np.append(np.arange(1,1./23, -1./24)**2.5, [1])\n",
    "    w = w / w.sum()\n",
    "    w = w[:,None]\n",
    "    W = K.variable(value = w)\n",
    "    return K.dot( K.abs(y_pred-y_true), W)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Custom Grid Search Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "  My Custom Grid Search Function:\n",
    "    * trainX       - The input data for the model\n",
    "    * trainY       - The target data for the model\n",
    "    * create_model - The function which returns a compiled model\n",
    "    * param_grid   - The parameters for the function to grid search through\n",
    "    * batch_size   - The default batch size if not otherwise specified directly or as a parameter to grid search\n",
    "    * epochs       - The default number of epochs if not otherwise specified directly or as a parameter to grid search through\n",
    "    * verbose      - The verbose level when training the individual prameters\n",
    "    * seed         - The seed to be applied to each model \n",
    "'''\n",
    "def custom_grid_search(trainX, trainY, create_model, param_grid, batch_size=25, epochs=10, verbose=3, seed=None):\n",
    "    # creates an array of the keys\n",
    "    keys = np.array(param_grid.keys())\n",
    "    # feeds the keys, the full dictionary, and a fresh dictionary to the recursive function\n",
    "    # to get an array of dictionaries of every possible combination of parameters\n",
    "    indiv_params = combine_parameters (keys, param_grid, dict())\n",
    "    # An array which will keep track of the accuracy of each iteration of the parameters\n",
    "    results = np.zeros(indiv_params.size)\n",
    "    # Runs each of the possible combinations\n",
    "    for i in range(indiv_params.size):\n",
    "        # prevents changing the original set of parameters\n",
    "        indiv_param = indiv_params[i].copy()\n",
    "        # Saves batch_size and epochs for the fitting of the model, not the creation/compilation of\n",
    "        if 'batch_size' in keys:\n",
    "            batch_size = indiv_param.pop('batch_size', None)\n",
    "        if 'epochs' in keys:\n",
    "            epochs = indiv_param.pop('epochs', None)\n",
    "        # if a seed has been specified, use it\n",
    "        if seed != None:\n",
    "            np.random.seed(seed)\n",
    "        # Creates the model and trains it\n",
    "        model = create_model(**indiv_param)\n",
    "        model.fit(trainX, trainY, epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "        # Stores this iteration's results\n",
    "        results[i] = np.mean(model.evaluate(trainX, trainY, verbose=3))\n",
    "        # Clears the backend session in an attempt to save memory with each model\n",
    "        K.clear_session()\n",
    "    # finds the index of the result with the lowest loss value\n",
    "    best_i = np.argwhere(results == min(results) )\n",
    "    \n",
    "    best_results = results[best_i].reshape(1)[0]\n",
    "    best_params  = indiv_params[best_i].reshape(1)[0]\n",
    "    # Returns all the results and their respective parameters in order\n",
    "    return dict(results=results,\n",
    "                parameters=indiv_params,\n",
    "                best_results = best_results,\n",
    "                best_parameters = best_params)\n",
    "\n",
    "'''\n",
    "  Given a Dictionary, the keys yet to be looped through, and a specific instance of the dictionary \n",
    "  it returns an array of dictionaries showing all unique combinations of the values\n",
    "  \n",
    "  e.g. given a dictionary (a=[0, 1], b=[0, 1]), it will return the following array\n",
    "  array([\n",
    "    dict(a=0, b=0),\n",
    "    dict(a=0, b=1),\n",
    "    dict(a=1, b=0),\n",
    "    dict(a=1, b-1)\n",
    "  ])\n",
    "  \n",
    "  The Structure is as follows:\n",
    "\n",
    "    The function is given an array of keys ['a', 'b'] and a dictionary (a=[0,1], b=[0, 1])\n",
    "\n",
    "    It loops through the first key, in this case a ∈ {0, 1} and removes the first key from the array,\n",
    "    its new value being ['b']\n",
    "\n",
    "    It then loops through all the values of the first key, creating a variable called specific_dict, \n",
    "    which is a dictionary with the given value of a (as well as any previously defined keys in specific_dict)\n",
    "      (a=0), and (a=1)\n",
    "\n",
    "    It then calls itself, providing the updated list of keys yet to be iterated through, the full dictionary,\n",
    "    and the specific_dict each for loop\n",
    "\n",
    "    Once the list of keys yet to be iterated through is of size 1, it ceases the recursion\n",
    "  \n",
    "'''\n",
    "def combine_parameters(keys, full_dict, specific_dict):\n",
    "    result = np.array([])\n",
    "    # If the size of the keys array is 1, that means this is the final key to be looped through, cease recursion\n",
    "    if keys.size == 1:\n",
    "        for i in range(len(full_dict[keys[0]])):\n",
    "            specific_dict = specific_dict.copy()\n",
    "            # the result will be the specific dict followed by every value in this \n",
    "            specific_dict[keys[0]] = full_dict[keys[0]][i]\n",
    "            result = np.append(result, specific_dict)\n",
    "        return result\n",
    "    else:\n",
    "        # keeps the first key\n",
    "        current_key = keys[0]\n",
    "        # removes the first key from the list of keys that havent been looped through yet\n",
    "        keys = keys[1:]\n",
    "        # for every value of the current_key\n",
    "        for i in range(len(full_dict[current_key])):\n",
    "            specific_dict = specific_dict.copy()\n",
    "            specific_dict[current_key] = full_dict[current_key][i]\n",
    "            result = np.append(result, combine_parameters(keys, full_dict, specific_dict))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tests for Optimal Dropout Rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Best: 0.063363 using {'dropout_rate': 0.1}\n",
      "0.0633632068847 {'dropout_rate': 0.1}\n",
      "0.0705592154632 {'dropout_rate': 0.2}\n",
      "0.0720145687669 {'dropout_rate': 0.3}\n",
      "0.067590751186 {'dropout_rate': 0.4}\n",
      "0.0684426791815 {'dropout_rate': 0.5}\n",
      "0.0732081863056 {'dropout_rate': 0.6}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# creates a model but does not compile the model\n",
    "def create_model(dropout_rate = 0.3):\n",
    "    layers            = 4\n",
    "    neurons           = 128\n",
    "    model = Sequential()\n",
    "    model.add(Dropout(dropout_rate, input_shape=(stations, look_back)))\n",
    "    lstm = LSTM(256, recurrent_dropout=dropout_rate, return_sequences=True)\n",
    "    model.add(lstm)\n",
    "    for i in range(layers):\n",
    "        model.add(Dropout(dropout_rate))\n",
    "        model.add(Dense(neurons))\n",
    "    model.add(Dense(look_forward))\n",
    "    model.compile(loss=top_heavy_loss, optimizer=RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0), metrics=[top_heavy_loss])\n",
    "    return model\n",
    "\n",
    "# The Parameters to Search Through\n",
    "dropout_rate = [0.2, 0.3, 0.4, 0.5]\n",
    "\n",
    "# Defines it as a dictionary\n",
    "param_grid = dict(dropout_rate=dropout_rate)\n",
    "\n",
    "# Begins the Search For the Optimal Properties\n",
    "grid_result = custom_grid_search(trainX, trainY, create_model, epochs=30, batch_size=30, param_grid=param_grid)\n",
    "\n",
    "# clears \n",
    "K.clear_session()\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result['best_results'], str(grid_result['best_parameters'])))\n",
    "\n",
    "loss = grid_result['results']\n",
    "params = grid_result['parameters']\n",
    "\n",
    "for l, p in zip(loss, params):\n",
    "    print l, p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "K.clear_session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Tests Optimal Combination of Dropout, Layers, and Neurons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-471e5c02d26b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;31m# prints all of the results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgrid_results\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrid_results\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'results'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m     \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrid_results\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'parameters'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices"
     ]
    }
   ],
   "source": [
    "\n",
    "# creates a model but does not compile the model\n",
    "def create_model(dropout_rate = 0.3, \n",
    "                 layers=3, \n",
    "                 neurons=128):\n",
    "    model = Sequential()\n",
    "    model.add(Dropout(dropout_rate, input_shape=(stations, look_back)))\n",
    "    lstm = LSTM(256, recurrent_dropout=dropout_rate, return_sequences=True)\n",
    "    model.add(lstm)\n",
    "    for i in range(layers):\n",
    "        model.add(Dropout(dropout_rate))\n",
    "        model.add(Dense(neurons))\n",
    "    model.add(Dense(look_forward))\n",
    "    model.compile(loss=top_heavy_loss, optimizer=RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0), metrics=[top_heavy_loss])\n",
    "    return model\n",
    "\n",
    "# The Parameters to Search Through\n",
    "dropout_rate = [0.2, 0.225, 0.250, 0.275]\n",
    "neurons = [128, 256, 128*3]\n",
    "layers = [3, 4]\n",
    "# Defines it as a dictionary\n",
    "param_grid = dict(dropout_rate=dropout_rate,\n",
    "                  neurons = neurons,\n",
    "                  layers = layers)\n",
    "\n",
    "# The number of times to repeat the grid search for the average\n",
    "num_iterations = 7\n",
    "# this will store three grid_result's\n",
    "grid_results = np.zeros(num_iterations).astype('object')\n",
    "\n",
    "# creates a m x n matrix where m is the number of grid searches and n is the number of unique shit\n",
    "grid_losses = np.zeros((num_iterations, len(dropout_rate) * len(neurons) * len(layers)))\n",
    "\n",
    "for i in range(num_iterations):\n",
    "    # Begins the Search For the Optimal Properties\n",
    "    grid_result = custom_grid_search(trainX, trainY, create_model, epochs=30, batch_size=30, param_grid=param_grid)\n",
    "    # You can never be too careful\n",
    "    K.clear_session()\n",
    "    # stores the entire dictionary into a variable\n",
    "    grid_results[i] = grid_result\n",
    "    # stores the results of the parameters into the matrix\n",
    "    grid_losses[i,:] = grid_result['results']\n",
    "    \n",
    "# averages the losses of each of the runs\n",
    "aver = grid_losses.mean(axis=0)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (aver.min(), str(grid_result['parameters'][np.argwhere(aver == aver.min())]  )))\n",
    "\n",
    "# prints all of the results\n",
    "for i in range(grid_results.size):\n",
    "    loss = grid_results[i]['results']\n",
    "    params = grid_results[i]['parameters']\n",
    "    for l, p in zip(loss, params):\n",
    "        print l, p\n",
    "        \n",
    "K.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Epoch 1/10\n",
      "Epoch 2/10\n",
      "Epoch 3/10\n",
      "Epoch 4/10\n",
      "Epoch 5/10\n",
      "Epoch 6/10\n",
      "Epoch 7/10\n",
      "Epoch 8/10\n",
      "Epoch 9/10\n",
      "Epoch 10/10\n",
      "Best: 0.070339 using [[{'recurrent_dropout': 0.225, 'lstm_neurons': 512}]]\n",
      "0.0704788165472 {'recurrent_dropout': 0.2, 'lstm_neurons': 256}\n",
      "0.075821797466 {'recurrent_dropout': 0.2, 'lstm_neurons': 384}\n",
      "0.07353012424 {'recurrent_dropout': 0.2, 'lstm_neurons': 512}\n",
      "0.0691947197272 {'recurrent_dropout': 0.225, 'lstm_neurons': 256}\n",
      "0.0738768855432 {'recurrent_dropout': 0.225, 'lstm_neurons': 384}\n",
      "0.0706349243075 {'recurrent_dropout': 0.225, 'lstm_neurons': 512}\n",
      "0.072493628268 {'recurrent_dropout': 0.25, 'lstm_neurons': 256}\n",
      "0.0726812667037 {'recurrent_dropout': 0.25, 'lstm_neurons': 384}\n",
      "0.0682354924967 {'recurrent_dropout': 0.25, 'lstm_neurons': 512}\n",
      "0.0720485536928 {'recurrent_dropout': 0.2, 'lstm_neurons': 256}\n",
      "0.0713391372959 {'recurrent_dropout': 0.2, 'lstm_neurons': 384}\n",
      "0.070901777992 {'recurrent_dropout': 0.2, 'lstm_neurons': 512}\n",
      "0.0807724039667 {'recurrent_dropout': 0.225, 'lstm_neurons': 256}\n",
      "0.0717108008396 {'recurrent_dropout': 0.225, 'lstm_neurons': 384}\n",
      "0.0700436801135 {'recurrent_dropout': 0.225, 'lstm_neurons': 512}\n",
      "0.0716273671128 {'recurrent_dropout': 0.25, 'lstm_neurons': 256}\n",
      "0.0702364598819 {'recurrent_dropout': 0.25, 'lstm_neurons': 384}\n",
      "0.0731363405453 {'recurrent_dropout': 0.25, 'lstm_neurons': 512}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# creates a model but does not compile the model\n",
    "def create_model(lstm_neurons=256, recurrent_dropout = 0.2):\n",
    "    dropout_rate = 0.25, \n",
    "    layers = 3 \n",
    "    neurons = 128\n",
    "    model = Sequential()\n",
    "    model.add(Dropout(dropout_rate, input_shape=(stations, look_back)))\n",
    "    lstm = LSTM(lstm_neurons, recurrent_dropout=recurrent_dropout, return_sequences=True)\n",
    "    model.add(lstm)\n",
    "    for i in range(layers):\n",
    "        model.add(Dropout(dropout_rate))\n",
    "        model.add(Dense(neurons))\n",
    "    model.add(Dense(look_forward))\n",
    "    model.compile(loss=top_heavy_loss, optimizer=RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0), metrics=[top_heavy_loss])\n",
    "    return model\n",
    "\n",
    "# The Parameters to Search Through\n",
    "recurrent_dropout = [0.2, 0.225, 0.250]\n",
    "lstm_neurons = [256, 128*3, 512]\n",
    "\n",
    "# Defines it as a dictionary\n",
    "param_grid = dict(recurrent_dropout=recurrent_dropout,\n",
    "                  lstm_neurons = lstm_neurons)\n",
    "\n",
    "# The number of times to repeat the grid search for the average\n",
    "num_iterations = 2\n",
    "# this will store three grid_result's\n",
    "grid_results = np.zeros(num_iterations).astype('object')\n",
    "\n",
    "# creates a m x n matrix where m is the number of grid searches and n is the number of unique shit\n",
    "grid_losses = np.zeros((num_iterations, len(recurrent_dropout) * len(lstm_neurons)))\n",
    "\n",
    "for i in range(num_iterations):\n",
    "    # Begins the Search For the Optimal Properties\n",
    "    grid_result = custom_grid_search(trainX, trainY, create_model, epochs=10, batch_size=30, param_grid=param_grid)\n",
    "    # You can never be too careful\n",
    "    K.clear_session()\n",
    "    # stores the entire dictionary into a variable\n",
    "    grid_results[i] = grid_result\n",
    "    # stores the results of the parameters into the matrix\n",
    "    grid_losses[i,:] = grid_result['results']\n",
    "    \n",
    "# averages the losses of each of the runs\n",
    "aver = grid_losses.mean(axis=0)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (aver.min(), str(grid_result['parameters'][np.argwhere(aver == aver.min())]  )))\n",
    "\n",
    "# prints all of the results\n",
    "for i in range(grid_results.size):\n",
    "    loss = grid_results[i]['results']\n",
    "    params = grid_results[i]['parameters']\n",
    "    for l, p in zip(loss, params):\n",
    "        print l, p\n",
    "        \n",
    "K.clear_session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Finds the Optimal Combination of Dropout, Recurrent Dropout, Dense Layers, Dense Neurons, and LSTM Neurons (yeah no not running this)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n",
      "Epoch 26/30\n",
      "Epoch 27/30\n",
      "Epoch 28/30\n",
      "Epoch 29/30\n",
      "Epoch 30/30\n",
      "Epoch 1/30\n",
      "Epoch 2/30\n",
      "Epoch 3/30\n",
      "Epoch 4/30\n",
      "Epoch 5/30\n",
      "Epoch 6/30\n",
      "Epoch 7/30\n",
      "Epoch 8/30\n",
      "Epoch 9/30\n",
      "Epoch 10/30\n",
      "Epoch 11/30\n",
      "Epoch 12/30\n",
      "Epoch 13/30\n",
      "Epoch 14/30\n",
      "Epoch 15/30\n",
      "Epoch 16/30\n",
      "Epoch 17/30\n",
      "Epoch 18/30\n",
      "Epoch 19/30\n",
      "Epoch 20/30\n",
      "Epoch 21/30\n",
      "Epoch 22/30\n",
      "Epoch 23/30\n",
      "Epoch 24/30\n",
      "Epoch 25/30\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# creates a model but does not compile the model\n",
    "def create_model(dropout_rate = 0.3, \n",
    "                 layers=3, \n",
    "                 neurons=128,\n",
    "                 recurrent_dropout=0.2,\n",
    "                 lstm_neurons=128):\n",
    "    model = Sequential()\n",
    "    model.add(Dropout(dropout_rate, input_shape=(stations, look_back)))\n",
    "    lstm = LSTM(lstm_neurons, recurrent_dropout=recurrent_dropout, return_sequences=True)\n",
    "    model.add(lstm)\n",
    "    for i in range(layers):\n",
    "        model.add(Dropout(dropout_rate))\n",
    "        model.add(Dense(neurons))\n",
    "    model.add(Dense(look_forward))\n",
    "    model.compile(loss=top_heavy_loss, optimizer=RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0), metrics=[top_heavy_loss])\n",
    "    return model\n",
    "\n",
    "# The Parameters to Search Through\n",
    "# 4 * 3 * 2 * 3 * 4\n",
    "\n",
    "dropout_rate = [0.2, 0.225, 0.250, 0.275]\n",
    "neurons = [128, 256, 128*3]\n",
    "layers = [3, 4]\n",
    "recurrent_dropout = [0.2, 0.225, 0.250]\n",
    "lstm_neurons = [256, 128*3, 512, 128*5]\n",
    "\n",
    "# Defines it as a dictionary\n",
    "param_grid = dict(dropout_rate=dropout_rate,\n",
    "                  neurons = neurons,\n",
    "                  layers = layers,\n",
    "                  recurrent_dropout=recurrent_dropout,\n",
    "                  lstm_neurons=lstm_neurons)\n",
    "\n",
    "# The number of times to repeat the grid search for the average\n",
    "num_iterations = 2\n",
    "# this will store three grid_result's\n",
    "grid_results = np.zeros(num_iterations).astype('object')\n",
    "\n",
    "# creates a m x n matrix where m is the number of grid searches and n is the number of unique shit\n",
    "grid_losses = np.zeros((num_iterations, len(dropout_rate) * \n",
    "                                        len(neurons) * \n",
    "                                        len(layers) *\n",
    "                                        len(recurrent_dropout) * \n",
    "                                        len(lstm_neurons)))\n",
    "\n",
    "for i in range(num_iterations):\n",
    "    # Begins the Search For the Optimal Properties\n",
    "    grid_result = custom_grid_search(trainX, trainY, create_model, epochs=30, batch_size=30, param_grid=param_grid)\n",
    "    # You can never be too careful\n",
    "    K.clear_session()\n",
    "    # stores the entire dictionary into a variable\n",
    "    grid_results[i] = grid_result\n",
    "    # stores the results of the parameters into the matrix\n",
    "    grid_losses[i,:] = grid_result['results']\n",
    "    \n",
    "# averages the losses of each of the runs\n",
    "aver = grid_losses.mean(axis=0)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (aver.min(), str(grid_result['parameters'][np.argwhere(aver == aver.min())]  )))\n",
    "print(\"Average Rsults\")\n",
    "\n",
    "# prints all of the results\n",
    "for i in range(grid_results.size):\n",
    "    params = grid_results[i]['parameters']\n",
    "    for l, p in zip(aver, params):\n",
    "        print l, p\n",
    "\n",
    "print(\"All Results\")\n",
    "# prints all of the results\n",
    "for i in range(grid_results.size):\n",
    "    loss = grid_results[i]['results']\n",
    "    params = grid_results[i]['parameters']\n",
    "    for l, p in zip(loss, params):\n",
    "        print l, p\n",
    "        \n",
    "K.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}