optimization/multi_gpu/multi_gpus.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [Using GPUS](https://tensorflow.google.cn/programmers_guide/using_gpu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Supported devices\n",
    "They are represented as strings. For example:\n",
    "\n",
    "- \"/cpu:0\": The CPU of your machine.\n",
    "- \"/device:GPU:0\": The GPU of your machine, if you have one.\n",
    "- \"/device:GPU:1\": The second GPU of your machine, etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logging Device placement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 22.  28.]\n",
      " [ 49.  64.]]\n"
     ]
    }
   ],
   "source": [
    "# Creates a graph.\n",
    "a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')\n",
    "b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')\n",
    "c = tf.matmul(a, b)\n",
    "# Creates a session with log_device_placement set to True.\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "# Runs the op.\n",
    "print(sess.run(c))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manual device placement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 22.  28.]\n",
      " [ 49.  64.]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "# Creates a graph.\n",
    "with tf.device('/cpu:0'):\n",
    "  a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')\n",
    "  b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')\n",
    "c = tf.matmul(a, b)\n",
    "# Creates a session with log_device_placement set to True.\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "# Runs the op.\n",
    "print(sess.run(c))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Allowing GPU memory growth\n",
    "By default, TensorFlow maps nearly all of the GPU memory of all GPUs (subject to CUDA_VISIBLE_DEVICES) visible to the process.\n",
    "\n",
    "TensorFlow provides two Config options on the Session to control this.\n",
    "- ***allow_growth***\n",
    "\n",
    "```\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.Session(config=config, ...)\n",
    "```\n",
    "\n",
    "- ***per_process_gpu_memory***\n",
    "\n",
    "```\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.4\n",
    "session = tf.Session(config=config, ...)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using a single GPU on a multi-GPU system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "InvalidArgumentError",
     "evalue": "Cannot assign a device for operation 'MatMul_1': Operation was explicitly assigned to /device:GPU:2 but available devices are [ /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/gpu:0, /job:localhost/replica:0/task:0/gpu:1 ]. Make sure the device specification refers to a valid device.\n\t [[Node: MatMul_1 = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/device:GPU:2\"](a_1, b_1)]]\n\nCaused by op u'MatMul_1', defined at:\n  File \"/usr/lib/python2.7/runpy.py\", line 162, in _run_module_as_main\n    \"__main__\", fname, loader, pkg_name)\n  File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n    exec code in run_globals\n  File \"/home/yuzhile/work2017/env/tf/lib/python2.7/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/kernelapp.py\", line 478, in start\n    self.io_loop.start()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2718, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-3dba1abd7652>\", line 5, in <module>\n    c = tf.matmul(a, b)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py\", line 1816, in matmul\n    a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py\", line 1217, in _mat_mul\n    transpose_b=transpose_b, name=name)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n    op_def=op_def)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 2506, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 1269, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): Cannot assign a device for operation 'MatMul_1': Operation was explicitly assigned to /device:GPU:2 but available devices are [ /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/gpu:0, /job:localhost/replica:0/task:0/gpu:1 ]. Make sure the device specification refers to a valid device.\n\t [[Node: MatMul_1 = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/device:GPU:2\"](a_1, b_1)]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-3dba1abd7652>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0msess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mConfigProto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_device_placement\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;31m# Runs the op.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    787\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 789\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    790\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    995\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 997\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    998\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1130\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1132\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1133\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1150\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1152\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: Cannot assign a device for operation 'MatMul_1': Operation was explicitly assigned to /device:GPU:2 but available devices are [ /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/gpu:0, /job:localhost/replica:0/task:0/gpu:1 ]. Make sure the device specification refers to a valid device.\n\t [[Node: MatMul_1 = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/device:GPU:2\"](a_1, b_1)]]\n\nCaused by op u'MatMul_1', defined at:\n  File \"/usr/lib/python2.7/runpy.py\", line 162, in _run_module_as_main\n    \"__main__\", fname, loader, pkg_name)\n  File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n    exec code in run_globals\n  File \"/home/yuzhile/work2017/env/tf/lib/python2.7/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/kernelapp.py\", line 478, in start\n    self.io_loop.start()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2718, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2822, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-3dba1abd7652>\", line 5, in <module>\n    c = tf.matmul(a, b)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py\", line 1816, in matmul\n    a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py\", line 1217, in _mat_mul\n    transpose_b=transpose_b, name=name)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n    op_def=op_def)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 2506, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/home/yuzhile/work2017/env/tf/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py\", line 1269, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): Cannot assign a device for operation 'MatMul_1': Operation was explicitly assigned to /device:GPU:2 but available devices are [ /job:localhost/replica:0/task:0/cpu:0, /job:localhost/replica:0/task:0/gpu:0, /job:localhost/replica:0/task:0/gpu:1 ]. Make sure the device specification refers to a valid device.\n\t [[Node: MatMul_1 = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/device:GPU:2\"](a_1, b_1)]]\n"
     ]
    }
   ],
   "source": [
    "# Creates a graph.\n",
    "with tf.device('/device:GPU:2'):\n",
    "  a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')\n",
    "  b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')\n",
    "  c = tf.matmul(a, b)\n",
    "# Creates a session with log_device_placement set to True.\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "# Runs the op.\n",
    "print(sess.run(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 22.  28.]\n",
      " [ 49.  64.]]\n"
     ]
    }
   ],
   "source": [
    "# Creates a graph.\n",
    "with tf.device('/device:GPU:2'):\n",
    "  a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')\n",
    "  b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')\n",
    "  c = tf.matmul(a, b)\n",
    "# Creates a session with allow_soft_placement and log_device_placement set\n",
    "# to True.\n",
    "sess = tf.Session(config=tf.ConfigProto(\n",
    "      allow_soft_placement=True, log_device_placement=True))\n",
    "# Runs the op.\n",
    "print(sess.run(c))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using multiple GPUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  44.   56.]\n",
      " [  98.  128.]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "# Creates a graph.\n",
    "c = []\n",
    "for d in ['/device:GPU:0', '/device:GPU:1']:\n",
    "  with tf.device(d):\n",
    "    a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3])\n",
    "    b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2])\n",
    "    c.append(tf.matmul(a, b))\n",
    "with tf.device('/cpu:0'):\n",
    "  sum = tf.add_n(c)\n",
    "# Creates a session with log_device_placement set to True.\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "# Runs the op.\n",
    "print(sess.run(sum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}