ero-SpaceInvaders.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import gym                 # Retro Environment\n",
    "import random\n",
    "import warnings\n",
    "from skimage import transform # Help us to preprocess the frames\n",
    "from skimage.color import rgb2gray # Help us to gray our frames\n",
    "import matplotlib.pyplot as plt # Display graphs\n",
    "from collections import deque # Ordered collection with ends\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GameEnv:\n",
    "  \"\"\"\n",
    "  This Class creates Atari Game Enivroment and provides some preprocessing functions.\n",
    "  \"\"\"\n",
    "  \n",
    "  def __init__(self, game = 'SpaceInvaders-v0'):\n",
    "    self.env = gym.make(game)\n",
    "    self.n_actions = self.env.action_space.n\n",
    "    self.frame_size = self.env.observation_space.shape\n",
    "    self.hot_enc_actions = np.array(np.identity(self.n_actions).tolist()) \n",
    "    self.stack_size = 4\n",
    "    self.stacked_frames = deque([np.zeros((110,84), dtype=np.int) for i in range(self.stack_size)], maxlen=self.stack_size)\n",
    "    self.hyperparameters = {\n",
    "                           'learning_rate' : 0.00025,\n",
    "                           'total_episodes' : 20,#was 50 initially\n",
    "                           'max_steps' : 50000,#was initially 50000\n",
    "                           'btach_size': 64,\n",
    "                           'explore_start' : 1,\n",
    "                           'explore_end' : 0.01,\n",
    "                           'decay_rate' : 0.00001,\n",
    "                           'gamma' : 0.9,\n",
    "                           'pretrain_length' : 64,\n",
    "                           'memory_size' : 1000000,\n",
    "                           'state_size' : [110, 84, 4]\n",
    "                           }\n",
    "    self.training = False\n",
    "    self.render = False\n",
    "    \n",
    "    \n",
    "    \n",
    "#Converting the main frame into a simpler frame ready to be processed by CNN\n",
    "  def _preprocess_frame(self,frame):\n",
    "    gray_frame = rgb2gray(frame)\n",
    "    cropped_frame = gray_frame[8:-12,4:-12]\n",
    "    \n",
    "    # Normalize Pixel Values\n",
    "    normalized_frame = cropped_frame/255.0\n",
    "    \n",
    "    # Resize\n",
    "    # Thanks to Mikołaj Walkowiak\n",
    "    preprocessed_frame = transform.resize(normalized_frame, [110,84])\n",
    "    \n",
    "    return preprocessed_frame # 110x84x1 frame\n",
    "\n",
    "\n",
    "# arranges the frames into a stack of 4 frames to recognise the temporal differences\n",
    "  \n",
    "  def stack_frame(self, state, new_epis = False):\n",
    "    \n",
    "    frame = self._preprocess_frame(state)\n",
    "    \n",
    "    if new_epis:\n",
    "      self.stacked_frames  =  deque([frame for _ in range(self.stack_size)], maxlen=self.stack_size)\n",
    "    else:\n",
    "      self.stacked_frames.append(frame)\n",
    "    \n",
    "    self.stacked_state = np.stack(self.stacked_frames, axis=2)\n",
    "    return self.stacked_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DeepQNN:\n",
    "  \n",
    "  def __init__(self, gamenv):\n",
    "    self.gamenv = gamenv\n",
    "    self.decay_step = 0\n",
    "    with tf.variable_scope('DQNN'):\n",
    "      self._inputs = tf.placeholder(tf.float32, [None, *self.gamenv.hyperparameters['state_size']], name='inputs')\n",
    "      self._actions = tf.placeholder(tf.float32, [None, self.gamenv.n_actions], name='actions')\n",
    "      self.target_Q = tf.placeholder(tf.float32, [None], name=\"target\")\n",
    "      \n",
    "      self.conv1 = tf.layers.conv2d(inputs = self._inputs, \n",
    "                                    filters = 32,\n",
    "                                    kernel_size = [8,8],\n",
    "                                    strides = [4,4],\n",
    "                                    padding = 'VALID',\n",
    "                                    kernel_initializer=tf.contrib.layers.xavier_initializer_conv2d(),\n",
    "                                    name = 'Conv1')\n",
    "      self.actvf1 = tf.nn.elu(self.conv1, name='Elu1')\n",
    "      \n",
    "      self.conv2 = tf.layers.conv2d(inputs = self.conv1, \n",
    "                                    filters = 64,\n",
    "                                    kernel_size = [4,4],\n",
    "                                    strides = [2,2],\n",
    "                                    padding = 'VALID',\n",
    "                                    kernel_initializer=tf.contrib.layers.xavier_initializer_conv2d(),\n",
    "                                    name = 'Conv2')\n",
    "      self.actvf2 = tf.nn.elu(self.conv2, name='Elu2')\n",
    "      \n",
    "      self.conv3 = tf.layers.conv2d(inputs = self.conv2, \n",
    "                                    filters = 64,\n",
    "                                    kernel_size = [3,3],\n",
    "                                    strides = [2,2],\n",
    "                                    padding = 'VALID',\n",
    "                                    kernel_initializer=tf.contrib.layers.xavier_initializer_conv2d(),\n",
    "                                    name = 'Conv3')\n",
    "      self.actvf3 = tf.nn.elu(self.conv3, name='Elu3')\n",
    "      \n",
    "      self.flatten = tf.contrib.layers.flatten(self.actvf3)\n",
    "      self.fc = tf.layers.dense(inputs = self.flatten,\n",
    "                                units = 512,\n",
    "                                activation = tf.nn.elu,\n",
    "                                kernel_initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                                name=\"fc1\")\n",
    "            \n",
    "      self.output = tf.layers.dense(inputs = self.fc, \n",
    "                                   kernel_initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                                   units = self.gamenv.n_actions, \n",
    "                                   activation=None)\n",
    "      self.Q = tf.reduce_sum(tf.multiply(self.output, self._actions))\n",
    "            \n",
    "            # The loss is the difference between our predicted Q_values and the Q_target\n",
    "            # Sum(Qtarget - Q)^2\n",
    "      self.loss = tf.reduce_mean(tf.square(self.target_Q - self.Q))\n",
    "            \n",
    "      self.optimizer = tf.train.AdamOptimizer(self.gamenv.hyperparameters['learning_rate']).minimize(self.loss)\n",
    "      \n",
    "      \n",
    "  def predict_action(self, state, sess):\n",
    "    hyperp = self.gamenv.hyperparameters\n",
    "    explore_probability = hyperp['explore_end'] + (hyperp['explore_start'] - hyperp['explore_end']) * np.exp(-hyperp['decay_rate'] * self.decay_step)\n",
    "    \n",
    "    if explore_probability > np.random.rand():\n",
    "      action = self.gamenv.hot_enc_actions[self.gamenv.env.action_space.sample()]\n",
    "      \n",
    "    else:\n",
    "      Qs = sess.run(self.output,feed_dict = {self._inputs:state.reshape((1,*state.shape))})\n",
    "      action = self.gamenv.hot_enc_actions[np.argmax(Qs)]\n",
    "      \n",
    "    return action, explore_probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Memory:\n",
    "  def __init__(self, max_size):\n",
    "    self.buffer = deque(maxlen = max_size)\n",
    "    \n",
    "  def add(self, experience):\n",
    "    self.buffer.append(experience)\n",
    "    \n",
    "  def sample(self, batch_size):\n",
    "    buffer_len = len(self.buffer)\n",
    "    index = np.random.choice(np.arange(buffer_len), size = batch_size, replace = False)\n",
    "    return [self.buffer[i] for i in index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_populate_memory(memory, gamenv):\n",
    "  state = gamenv.env.reset()\n",
    "  state = gamenv.stack_frame(state,new_epis = True)\n",
    "  for i in range(gamenv.hyperparameters['btach_size']):\n",
    "    action = gamenv.hot_enc_actions[gamenv.env.action_space.sample()]\n",
    "    next_state, reward, done, info = gamenv.env.step(np.argmax(action))\n",
    "    next_state = gamenv.stack_frame(next_state, new_epis = False)\n",
    "    if done:\n",
    "      next_state = np.zeros(next_state.shape)\n",
    "      memory.add((state, action, reward, next_state, done))\n",
    "      state = gamenv.env.reset()\n",
    "      state = gamenv.stack_frame(state,new_epis = True)\n",
    "    else:\n",
    "      memory.add((state, action, reward, next_state, done))\n",
    "      state = next_state\n",
    "  return memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gym[atari] in /home/ero/anaconda3/lib/python3.7/site-packages (0.12.0)\n",
      "Requirement already satisfied: scipy in /home/ero/anaconda3/lib/python3.7/site-packages (from gym[atari]) (1.1.0)\n",
      "Requirement already satisfied: numpy>=1.10.4 in /home/ero/anaconda3/lib/python3.7/site-packages (from gym[atari]) (1.15.4)\n",
      "Requirement already satisfied: requests>=2.0 in /home/ero/anaconda3/lib/python3.7/site-packages (from gym[atari]) (2.21.0)\n",
      "Requirement already satisfied: six in /home/ero/anaconda3/lib/python3.7/site-packages (from gym[atari]) (1.12.0)\n",
      "Requirement already satisfied: pyglet>=1.2.0 in /home/ero/anaconda3/lib/python3.7/site-packages (from gym[atari]) (1.3.2)\n",
      "Requirement already satisfied: PyOpenGL; extra == \"atari\" in /home/ero/anaconda3/lib/python3.7/site-packages (from gym[atari]) (3.1.0)\n",
      "Requirement already satisfied: Pillow; extra == \"atari\" in /home/ero/anaconda3/lib/python3.7/site-packages (from gym[atari]) (5.3.0)\n",
      "Requirement already satisfied: atari-py>=0.1.4; extra == \"atari\" in /home/ero/anaconda3/lib/python3.7/site-packages (from gym[atari]) (0.1.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ero/anaconda3/lib/python3.7/site-packages (from requests>=2.0->gym[atari]) (2018.11.29)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /home/ero/anaconda3/lib/python3.7/site-packages (from requests>=2.0->gym[atari]) (2.8)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/ero/anaconda3/lib/python3.7/site-packages (from requests>=2.0->gym[atari]) (3.0.4)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /home/ero/anaconda3/lib/python3.7/site-packages (from requests>=2.0->gym[atari]) (1.24.1)\n",
      "Requirement already satisfied: future in /home/ero/anaconda3/lib/python3.7/site-packages (from pyglet>=1.2.0->gym[atari]) (0.17.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install gym[atari]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-3-93a95e7b02b4>:17: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.conv2d instead.\n",
      "WARNING:tensorflow:From /home/ero/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/ero/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/layers/python/layers/layers.py:1624: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.flatten instead.\n",
      "WARNING:tensorflow:From <ipython-input-3-93a95e7b02b4>:43: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "tf.reset_default_graph()\n",
    "spaceinvaders = GameEnv()\n",
    "# spaceinvaders.training = True\n",
    "dqnn = DeepQNN(spaceinvaders)\n",
    "memory = Memory(spaceinvaders.hyperparameters['memory_size'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memory = pre_populate_memory(memory, spaceinvaders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup TensorBoard Writer\n",
    "writer = tf.summary.FileWriter(\"./tensorboard/dqn/1\")\n",
    "\n",
    "## Losses\n",
    "tf.summary.scalar(\"Loss\", dqnn.loss)\n",
    "\n",
    "write_op = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "spaceinvaders.training = True\n",
    "spaceinvaders.render = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0 Total reward: 320.0 Explore P: 0.9931 Training Loss 23.3482\n",
      "Model Saved\n"
     ]
    }
   ],
   "source": [
    "# Saver will help us to save our model\n",
    "saver = tf.train.Saver()\n",
    "hyperp = spaceinvaders.hyperparameters\n",
    "rewards_list = []\n",
    "if spaceinvaders.training == True:\n",
    "    with tf.Session() as sess:\n",
    "        #saver.restore(sess, \"./models/model.ckpt\")\n",
    "        # Initialize the variables\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        # Initialize the decay rate (that will use to reduce epsilon) \n",
    "#         decay_step = 0\n",
    "        \n",
    "        for episode in range(hyperp['total_episodes']):\n",
    "            # Set step to 0\n",
    "            step = 0\n",
    "            \n",
    "            # Initialize the rewards of the episode\n",
    "            episode_rewards = []\n",
    "            \n",
    "            # Make a new episode and observe the first state\n",
    "            state = spaceinvaders.env.reset()\n",
    "            \n",
    "            # Remember that stack frame function also call our preprocess function.\n",
    "            state = spaceinvaders.stack_frame(state, True)\n",
    "            \n",
    "            while step < hyperp['max_steps']:\n",
    "                step += 1\n",
    "                \n",
    "                #Increase decay_step\n",
    "                dqnn.decay_step +=1\n",
    "                \n",
    "                # Predict the action to take and take it\n",
    "                action, explore_probability = dqnn.predict_action(state, sess)\n",
    "                \n",
    "                #Perform the action and get the next_state, reward, and done information\n",
    "                next_state, reward, done, _ = spaceinvaders.env.step(np.argmax(action))\n",
    "                \n",
    "                if spaceinvaders.render:\n",
    "                    spaceinvaders.env.render()\n",
    "                \n",
    "                # Add the reward to total reward\n",
    "                episode_rewards.append(reward)\n",
    "                \n",
    "                # If the game is finished\n",
    "                if done:\n",
    "                    # The episode ends so no next state\n",
    "                    next_state = np.zeros((110,84), dtype=np.int)\n",
    "                    \n",
    "                    next_state = spaceinvaders.stack_frame(next_state, False)\n",
    "\n",
    "                    # Set step = max_steps to end the episode\n",
    "                    step = hyperp['max_steps']\n",
    "\n",
    "                    # Get the total reward of the episode\n",
    "                    total_reward = np.sum(episode_rewards)\n",
    "\n",
    "                    print('Episode: {}'.format(episode),\n",
    "                                  'Total reward: {}'.format(total_reward),\n",
    "                                  'Explore P: {:.4f}'.format(explore_probability),\n",
    "                                'Training Loss {:.4f}'.format(loss))\n",
    "\n",
    "                    rewards_list.append((episode, total_reward))\n",
    "\n",
    "                    # Store transition <st,at,rt+1,st+1> in memory D\n",
    "                    memory.add((state, action, reward, next_state, done))\n",
    "\n",
    "                else:\n",
    "                    # Stack the frame of the next_state\n",
    "                    next_state = spaceinvaders.stack_frame(next_state, False)\n",
    "                \n",
    "                    # Add experience to memory\n",
    "                    memory.add((state, action, reward, next_state, done))\n",
    "\n",
    "                    # st+1 is now our current state\n",
    "                    state = next_state\n",
    "                    \n",
    "\n",
    "                ### LEARNING PART            \n",
    "                # Obtain random mini-batch from memory\n",
    "                batch = memory.sample(hyperp['btach_size'])\n",
    "                states_mb = np.array([each[0] for each in batch], ndmin=3)\n",
    "                actions_mb = np.array([each[1] for each in batch])\n",
    "                rewards_mb = np.array([each[2] for each in batch]) \n",
    "                next_states_mb = np.array([each[3] for each in batch], ndmin=3)\n",
    "                dones_mb = np.array([each[4] for each in batch])\n",
    "\n",
    "                target_Qs_batch = []\n",
    "\n",
    "                # Get Q values for next_state \n",
    "                Qs_next_state = sess.run(dqnn.output, feed_dict = {dqnn._inputs: next_states_mb})\n",
    "                \n",
    "                # Set Q_target = r if the episode ends at s+1, otherwise set Q_target = r + gamma*maxQ(s', a')\n",
    "                for i in range(0, len(batch)):\n",
    "                    terminal = dones_mb[i]\n",
    "\n",
    "                    # If we are in a terminal state, only equals reward\n",
    "                    if terminal:\n",
    "                        target_Qs_batch.append(rewards_mb[i])\n",
    "                        \n",
    "                    else:\n",
    "                        target = rewards_mb[i] + hyperp['gamma'] * np.max(Qs_next_state[i])\n",
    "                        target_Qs_batch.append(target)\n",
    "                        \n",
    "\n",
    "                targets_mb = np.array([each for each in target_Qs_batch])\n",
    "\n",
    "                loss, _ = sess.run([dqnn.loss, dqnn.optimizer],\n",
    "                                        feed_dict={dqnn._inputs: states_mb,\n",
    "                                                   dqnn.target_Q: targets_mb,\n",
    "                                                   dqnn._actions: actions_mb})\n",
    "\n",
    "                # Write TF Summaries\n",
    "                summary = sess.run(write_op, feed_dict={dqnn._inputs: states_mb,\n",
    "                                                       dqnn.target_Q: targets_mb,\n",
    "                                                       dqnn._actions: actions_mb})\n",
    "                writer.add_summary(summary, episode)\n",
    "                writer.flush()\n",
    "\n",
    "            # Save model every 5 episodes\n",
    "            if episode % 1 == 0:\n",
    "                save_path = saver.save(sess, \"./models/model.ckpt\")\n",
    "                print(\"Model Saved\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()\n",
    "step = 0\n",
    "with tf.Session() as sess:\n",
    "    total_test_rewards = []\n",
    "    \n",
    "    # Load the model\n",
    "    saver.restore(sess, \"./models/model.ckpt\")\n",
    "    \n",
    "    for episode in range(1):\n",
    "        total_rewards = 0\n",
    "        \n",
    "        state = spaceinvaders.env.reset()\n",
    "        state = spaceinvaders.stack_frame(state, True)\n",
    "        \n",
    "        print(\"****************************************************\")\n",
    "        print(\"EPISODE \", episode)\n",
    "        \n",
    "        while True:\n",
    "            # Reshape the state\n",
    "            state = state.reshape((1, *spaceinvaders.hyperparameters['state_size']))\n",
    "            # Get action from Q-network \n",
    "            # Estimate the Qs values state\n",
    "            Qs = sess.run(dqnn.output, feed_dict = {dqnn._inputs: state})\n",
    "            step +=1\n",
    "            # Take the biggest Q value (= the best action)\n",
    "            choice = np.argmax(Qs)\n",
    "#             print(choice)\n",
    "            action = spaceinvaders.hot_enc_actions[choice]\n",
    "#             print(choice)\n",
    "            #Perform the action and get the next_state, reward, and done information\n",
    "            next_state, reward, done, _ = spaceinvaders.env.step(choice)\n",
    "            spaceinvaders.env.render()\n",
    "            \n",
    "            total_rewards += reward\n",
    "\n",
    "            if done:\n",
    "                print (\"Score\", total_rewards)\n",
    "                total_test_rewards.append(total_rewards)\n",
    "                print(step)\n",
    "                break\n",
    "                \n",
    "                \n",
    "            next_state = spaceinvaders.stack_frame(next_state, False)\n",
    "            state = next_state\n",
    "            \n",
    "    spaceinvaders.env.close()\n",
    "    del(spaceinvaders)\n",
    "    spaceinvaders = GameEnv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}