FCN-trainer-from-gap.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Loaded\n",
      "conv1_1 Value Loaded\n",
      "Layer name: conv1_1\n",
      "Layer shape: (3, 3, 3, 64)\n",
      "conv1_1 Frozen\n",
      "conv1_1 (64,) Value Loaded\n",
      "\n",
      "\n",
      "conv1_2 Value Loaded\n",
      "Layer name: conv1_2\n",
      "Layer shape: (3, 3, 64, 64)\n",
      "L2 Loss Added\n",
      "conv1_2 (64,) Value Loaded\n",
      "\n",
      "\n",
      "conv2_1 Value Loaded\n",
      "Layer name: conv2_1\n",
      "Layer shape: (3, 3, 64, 128)\n",
      "L2 Loss Added\n",
      "conv2_1 (128,) Value Loaded\n",
      "\n",
      "\n",
      "conv2_2 Value Loaded\n",
      "Layer name: conv2_2\n",
      "Layer shape: (3, 3, 128, 128)\n",
      "L2 Loss Added\n",
      "conv2_2 (128,) Value Loaded\n",
      "\n",
      "\n",
      "conv3_1 Value Loaded\n",
      "Layer name: conv3_1\n",
      "Layer shape: (3, 3, 128, 256)\n",
      "L2 Loss Added\n",
      "conv3_1 (256,) Value Loaded\n",
      "\n",
      "\n",
      "conv3_2 Value Loaded\n",
      "Layer name: conv3_2\n",
      "Layer shape: (3, 3, 256, 256)\n",
      "L2 Loss Added\n",
      "conv3_2 (256,) Value Loaded\n",
      "\n",
      "\n",
      "conv3_3 Value Loaded\n",
      "Layer name: conv3_3\n",
      "Layer shape: (3, 3, 256, 256)\n",
      "L2 Loss Added\n",
      "conv3_3 (256,) Value Loaded\n",
      "\n",
      "\n",
      "conv4_1 Value Loaded\n",
      "Layer name: conv4_1\n",
      "Layer shape: (3, 3, 256, 512)\n",
      "L2 Loss Added\n",
      "conv4_1 (512,) Value Loaded\n",
      "\n",
      "\n",
      "conv4_2 Value Loaded\n",
      "Layer name: conv4_2\n",
      "Layer shape: (3, 3, 512, 512)\n",
      "L2 Loss Added\n",
      "conv4_2 (512,) Value Loaded\n",
      "\n",
      "\n",
      "conv4_3 Value Loaded\n",
      "Layer name: conv4_3\n",
      "Layer shape: (3, 3, 512, 512)\n",
      "L2 Loss Added\n",
      "conv4_3 (512,) Value Loaded\n",
      "\n",
      "\n",
      "conv5_1 Value Loaded\n",
      "Layer name: conv5_1\n",
      "Layer shape: (3, 3, 512, 512)\n",
      "L2 Loss Added\n",
      "conv5_1 (512,) Value Loaded\n",
      "\n",
      "\n",
      "conv5_2 Value Loaded\n",
      "Layer name: conv5_2\n",
      "Layer shape: (3, 3, 512, 512)\n",
      "L2 Loss Added\n",
      "conv5_2 (512,) Value Loaded\n",
      "\n",
      "\n",
      "conv5_3 Value Loaded\n",
      "Layer name: conv5_3\n",
      "Layer shape: (3, 3, 512, 512)\n",
      "L2 Loss Added\n",
      "conv5_3 (512,) Value Loaded\n",
      "\n",
      "\n",
      "Layer name: fc6\n",
      "Layer shape: [7, 7, 512, 4096]\n",
      "fc6 Value Loaded\n",
      "fc6 (4096,) Value Loaded\n",
      "\n",
      "\n",
      "fc6 dropout_added\n",
      "Layer name: fc7\n",
      "Layer shape: [1, 1, 4096, 4096]\n",
      "fc7 Value Loaded\n",
      "fc7 (4096,) Value Loaded\n",
      "\n",
      "\n",
      "fc7 dropout_added\n",
      "fc8 Value Randomly initialized\n",
      "L2 Loss Added\n",
      "Layer name: score_fr\n",
      "Layer shape: [1, 1, 4096, 19]\n",
      "fc8_biases Value Initialized 0\n",
      "\n",
      "\n",
      "Finished building Network.\n",
      "using Pretrained model\n",
      "model_checkpoint_path: \"/mnt/data3/FCN32_gap/rel2/ckpt_dir/fcn32_rel2_from_gap.ckpt-50\"\n",
      "all_model_checkpoint_paths: \"/mnt/data3/FCN32_gap/rel2/ckpt_dir/fcn32_rel2_from_gap.ckpt-50\"\n",
      "\n",
      "finished\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import matplotlib.cm as cm\n",
    "import copy\n",
    "import tensorflow as tf\n",
    "\n",
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "from IPython.display import clear_output\n",
    "import sys\n",
    "import csv\n",
    "import math\n",
    "from matplotlib import colors\n",
    "import fcn32_from_gap as fcn32_vgg\n",
    "import csv\n",
    "\n",
    "\n",
    "ckpt_dir = \"/mnt/data3/FCN32_gap/rel2/ckpt_dir/fcn32_rel2_from_gap.ckpt\"\n",
    "\n",
    "PRETRAINED_MODEL_PATH= \"/mnt/data3/FCN32_gap/rel2/ckpt_dir\"\n",
    "\n",
    "WEIGHT_PATH = \"/mnt/data3/gap/npy/vgg-rel2-gap-epoch-83.npy\"\n",
    "\n",
    "TRAINSET_PATH = '/mnt/data1/city/csv_files/cityscapes_train_wmask.csv'\n",
    "VALSET_PATH = '/mnt/data1/city/csv_files/cityscapes_val_wmask.csv'\n",
    "\n",
    "data_csv_path = 'fcn32_rel2_gap.csv'\n",
    "\n",
    "\n",
    "NUM_OF_CLASSESS = 19\n",
    "BATCH_SIZE = 1\n",
    "IMAGE_HEIGHT = 1024\n",
    "IMAGE_WIDTH = 2048\n",
    "NUM_CHANNELS = 3\n",
    "N_EPOCHS = 300\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "train_csv_file=open(TRAINSET_PATH,\"r\")\n",
    "train_filepaths=[]\n",
    "train_labels=[]\n",
    "train_masks=[]\n",
    "for line in train_csv_file:\n",
    "    filepath, label, mask= line.split(\",\")\n",
    "    train_filepaths.append(filepath)\n",
    "    train_labels.append(label)\n",
    "    train_masks.append(mask.split(\"\\r\")[0])\n",
    "    \n",
    "train_image_path, train_label_path, train_mask_path = tf.train.slice_input_producer([train_filepaths,train_labels, train_masks]\n",
    "                                           ,capacity = 2975)\n",
    "#csv_path = tf.train.string_input_producer([TRAINSET_PATH])\n",
    "#textReader = tf.TextLineReader()\n",
    "#_, content = textReader.read(csv_path)\n",
    "#one_ex = tf.decode_csv(content, record_defaults=[[\"\"], [\"\"], [\"\"]])\n",
    "\n",
    "train_image_content = tf.read_file(train_image_path)\n",
    "train_image = tf.image.decode_png(train_image_content, channels=3)\n",
    "train_image.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH, NUM_CHANNELS])\n",
    "train_image = tf.cast(train_image, tf.float32)\n",
    "\n",
    "train_label_content = tf.read_file(train_label_path)\n",
    "train_label = tf.image.decode_png(train_label_content, channels=1)\n",
    "train_label.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH, 1])\n",
    "train_label=tf.squeeze(train_label, squeeze_dims=[2])\n",
    "\n",
    "#record_bytes=2097152\n",
    "#reader=tf.FixedLengthRecordReader(record_bytes)\n",
    "train_mask_content =tf.read_file(train_mask_path)\n",
    "train_mask=tf.decode_raw(train_mask_content,tf.uint8)\n",
    "train_mask.set_shape([2097152])\n",
    "#mask=tf.cast(mask, tf.float32)\n",
    "\n",
    "\n",
    "val_csv_file=open(VALSET_PATH,\"r\")\n",
    "val_filepaths=[]\n",
    "val_labels=[]\n",
    "val_masks=[]\n",
    "for line in val_csv_file:\n",
    "    filepath, label, mask= line.split(\",\")\n",
    "    val_filepaths.append(filepath)\n",
    "    val_labels.append(label)\n",
    "    val_masks.append(mask.split(\"\\r\")[0])\n",
    "    \n",
    "val_image_path, val_label_path, val_mask_path = tf.train.slice_input_producer([val_filepaths,val_labels, val_masks]\n",
    "                                           ,capacity = 500)\n",
    "#csv_path = tf.train.string_input_producer([TRAINSET_PATH])\n",
    "#textReader = tf.TextLineReader()\n",
    "#_, content = textReader.read(csv_path)\n",
    "#one_ex = tf.decode_csv(content, record_defaults=[[\"\"], [\"\"], [\"\"]])\n",
    "\n",
    "val_image_content = tf.read_file(val_image_path)\n",
    "val_image = tf.image.decode_png(val_image_content, channels=3)\n",
    "val_image.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH, NUM_CHANNELS])\n",
    "val_image = tf.cast(val_image, tf.float32)\n",
    "\n",
    "val_label_content = tf.read_file(val_label_path)\n",
    "val_label = tf.image.decode_png(val_label_content, channels=1)\n",
    "val_label.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH, 1])\n",
    "val_label=tf.squeeze(val_label, squeeze_dims=[2])\n",
    "\n",
    "#record_bytes=2097152\n",
    "#reader=tf.FixedLengthRecordReader(record_bytes)\n",
    "val_mask_content =tf.read_file(val_mask_path)\n",
    "val_mask=tf.decode_raw(val_mask_content,tf.uint8)\n",
    "val_mask.set_shape([2097152])\n",
    "#mask=tf.cast(mask, tf.float32)\n",
    "\n",
    "\n",
    "\n",
    "train_image_batch, train_label_batch, train_mask_batch= tf.train.batch([train_image, train_label, train_mask], batch_size=BATCH_SIZE,\n",
    "                                                               capacity=100 + 3 * BATCH_SIZE,\n",
    "                                                               num_threads=7)\n",
    "\n",
    "val_image_batch, val_label_batch, val_mask_batch= tf.train.batch([val_image, val_label, val_mask], batch_size=BATCH_SIZE,\n",
    "                                                               capacity=50 + 3 * BATCH_SIZE,\n",
    "                                                               num_threads=7)\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    sess = tf.Session()\n",
    "    #images_tf = tf.placeholder(tf.float32,[None, 1024, 2048, 3])\n",
    "    #labels_tf = tf.placeholder(tf.int32,[None, 1024, 2048])\n",
    "    #masks_tf = tf.placeholder(tf.float32,[None,2097152])\n",
    "    train_mode = tf.placeholder(tf.bool)\n",
    "    \n",
    "    vgg_fcn = fcn32_vgg.FCN32VGG(WEIGHT_PATH)#('./vgg16.npy')\n",
    "    \n",
    "    if train_mode is not None:\n",
    "        images_tf=tf.cond(train_mode, lambda:train_image_batch, lambda:val_image_batch)\n",
    "    if train_mode is not None:\n",
    "        labels_tf=tf.cond(train_mode, lambda:train_label_batch, lambda:val_label_batch)\n",
    "    if train_mode is not None:\n",
    "        masks_tf=tf.cond(train_mode, lambda:train_mask_batch, lambda:val_mask_batch)\n",
    "                          \n",
    "    with tf.name_scope(\"content_vgg\"):\n",
    "        vgg_fcn.build(images_tf, train=train_mode, num_classes=19, random_init_fc8=True, debug=False)\n",
    "\n",
    "\n",
    "    #head=[]\n",
    "    #sum_of_weights=tf.reduce_sum(weights)\n",
    "    #labels_oh= tf.one_hot(labels_tf,19)\n",
    "    \n",
    "    labels_tf_flat=tf.reshape(tf.mul(tf.reshape(labels_tf,[-1]),masks_tf),[-1])\n",
    "    #mask_reshped=tf.reshape(masks_tf,[1024,2048])\n",
    "    #labels_tf_flat2= tf.gather(labels_tf_flat,tf.where(tf.not_equal(labels_tf_flat,19))[1])\n",
    "    logits = tf.reshape(vgg_fcn.upscore, (-1, NUM_OF_CLASSESS))\n",
    "    #logits = tf.gather(logits,tf.where(tf.not_equal(labels_tf_flat,19)))\n",
    "    labels_tf_flat = tf.cast(labels_tf_flat, tf.int32)\n",
    "\n",
    "    #epsilon = tf.constant(value=1e-4)\n",
    "    #logits = logits + epsilon\n",
    "    #labels_flat_sparse = tf.reshape(labels_oh, (-1, NUM_OF_CLASSESS))\n",
    "    #labels_flat_sparse=tf.cast(labels_flat_sparse, tf.float64)\n",
    "    #ww_flat=tf.reshape(weights,[-1])\n",
    "    #ww_flat = tf.cast(ww_flat, tf.float64)\n",
    "    #softmax = tf.nn.softmax(logits)\n",
    "    #softmax = tf.cast(softmax, tf.float64)\n",
    "    #test1=-tf.reduce_sum(labels_flat_sparse * tf.log(softmax), reduction_indices=[1])\n",
    "    #test2=tf.reduce_sum(test1)\n",
    "    #cross_entropy = tf.mul(-tf.reduce_sum(labels_flat_sparse * tf.log(softmax), reduction_indices=[1]),ww_flat)\n",
    "    #cross_entropy = -tf.reduce_sum(labels_flat_sparse * tf.log(softmax), reduction_indices=[1])\n",
    "    #mask_flat=tf.reshape(train_mask_batch,[-1])\n",
    "    masks_tf=tf.cast(masks_tf, tf.float32)\n",
    "    cross_entropy =tf.mul(tf.nn.sparse_softmax_cross_entropy_with_logits(logits,labels_tf_flat),masks_tf)\n",
    "    #cross_entropy_sum= tf.reduce_sum(cross_entropy,\n",
    "    #                                name='xentropy_mean')\n",
    "    #cross_entropy_mean = tf.reduce_mean(cross_entropy,name='xentropy_mean')\n",
    "    #cross_entropy_mean = cross_entropy_sum/tf.cast(sum_of_weights, tf.float32)\n",
    "    cross_entropy_mean = tf.reduce_mean(cross_entropy,\n",
    "                                        name='xentropy_mean')\n",
    "    l2_loss=tf.reduce_sum(tf.get_collection(\"losses\"))\n",
    "    total_loss=cross_entropy_mean +l2_loss\n",
    "    #loss_tf = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(vgg_fcn.upscore,\n",
    "     #                                                                     tf.squeeze(labels_tf, squeeze_dims=[3]),\n",
    "      #                                                                    name=\"entropy\")))\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    #train_op = tf.train.MomentumOptimizer(0.0001, 0.99).minimize(cross_entropy_mean)\n",
    "    train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(cross_entropy_mean)\n",
    "    predictions=tf.argmax(logits, 1)\n",
    "    #labels_tf_flat=tf.reshape(labels_tf_flat,[-1])\n",
    "    #labels_flat=tf.cast(labels_tf_flat, tf.int64)\n",
    "    pred_flat=tf.reshape(predictions,[-1])\n",
    "with tf.device('/cpu:0'):\n",
    "    miou, update_cm=tf.contrib.metrics.streaming_mean_iou(pred_flat,labels_tf_flat,19,weights=masks_tf)\n",
    "    if not os.path.exists(ckpt_dir):\n",
    "        os.makedirs(ckpt_dir)\n",
    "    saver = tf.train.Saver(max_to_keep=50)\n",
    "with tf.device('/gpu:0'):\n",
    "    print('Finished building Network.')\n",
    "    \n",
    "    init_op = tf.group(tf.initialize_all_variables(),\n",
    "                       tf.initialize_local_variables())\n",
    "    sess.run(init_op)\n",
    "        \n",
    "    if PRETRAINED_MODEL_PATH:\n",
    "        print \"using Pretrained model\"\n",
    "        ckpt = tf.train.get_checkpoint_state(PRETRAINED_MODEL_PATH)\n",
    "        print(ckpt)\n",
    "        saver.restore(sess,ckpt.model_checkpoint_path)\n",
    "        \n",
    "    # print(csv_path)\n",
    "    # For populating queues with batches, very important!\n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(sess=sess,coord=coord)\n",
    "    \n",
    "    print(\"finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epoch = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "total_train_loss_list,train_loss_list,plot_train_loss,plot_total_train_loss,plot_train_miou= [],[],[],[],[]\n",
    "val_loss_list,plot_val_loss,plot_val_miou= [],[],[]\n",
    "#for epoch in xrange(N_EPOCHS):\n",
    "cmap = colors.ListedColormap([[0.4667,0.0431,0.1254],[0.902,0.5882,0.5490],[0,0.2353,0.3921]\n",
    "                              ,[0,0,0.5569],[0.7451,0.6,0.6],[0,0,0.902]\n",
    "                              ,[0.8627,0.0784,0.2352],[0.6,0.6,0.6],[1,0,0]\n",
    "                              ,[0.502,0.251,0.502],[0.9569,0.1372,0.9098],[0.2745,0.5098,0.7059]\n",
    "                              ,[0.5961,0.9843,0.5961],[0.9804,0.6667,0.1176],[0.8627,0.8627,0]\n",
    "                              ,[0,0.3137,0.3922],[0,0,0.2745],[0.4196,0.5569,0.1372]\n",
    "                              ,[0.4,0.4,0.6118],[0,0,0]])\n",
    "norm=colors.NoNorm(0,19)\n",
    "try:\n",
    "    while not coord.should_stop() and epoch< N_EPOCHS:\n",
    "        epoch+=1\n",
    "        epoch_start_time = time.time()\n",
    "        for iteration in xrange(2975/BATCH_SIZE): \n",
    "            if (iteration%500==0):\n",
    "                _,train_loss,total_train_loss,_,train_pred_pic, train_label_pic = sess.run([train_op,cross_entropy_mean,total_loss,\n",
    "                                                            update_cm,vgg_fcn.pred_up,labels_tf],feed_dict={train_mode: True})\n",
    "                plt.figure(1)\n",
    "                plt.imshow(train_label_pic[0],cmap=cmap,norm=norm)\n",
    "                plt.figure(2)\n",
    "                plt.imshow(train_pred_pic[0],cmap=cmap,norm=norm)\n",
    "                #plt.figure(3)\n",
    "                #plt.imshow(output2[0],cmap=cmap,norm=norm)\n",
    "                plt.show()\n",
    "            else:\n",
    "                    _,train_loss,total_train_loss,_ = sess.run([train_op,cross_entropy_mean,total_loss,\n",
    "                                                            update_cm],feed_dict={train_mode: True})\n",
    "            if(math.isnan(train_loss)):\n",
    "                break\n",
    "            train_loss_list.append(train_loss)\n",
    "            total_train_loss_list.append(total_train_loss)\n",
    "            sys.stdout.write('\\r\\r'+ str(iteration))\n",
    "            sys.stdout.flush()\n",
    "                \n",
    "\n",
    "        train_iou=sess.run(miou)\n",
    "        train_loss_mean = np.mean(train_loss_list)\n",
    "        total_train_loss_mean=np.mean(total_train_loss_list)\n",
    "        plot_train_loss.append(train_loss_mean)\n",
    "        plot_total_train_loss.append(total_train_loss_mean)\n",
    "\n",
    "        train_loss_list = []\n",
    "        total_train_loss_list = []\n",
    "        plot_train_miou.append(train_iou)\n",
    "        \n",
    "        var_miou = filter(lambda x: x.name.endswith('total_confusion_matrix:0'), tf.local_variables())\n",
    "        init_miou=tf.initialize_variables(var_miou)\n",
    "        sess.run(init_miou)\n",
    "        \n",
    "        \n",
    "        \n",
    "        for iteration in xrange(500/BATCH_SIZE): \n",
    "            if (iteration%100==0):\n",
    "                val_loss,_,val_pred_pic, val_label_pic = sess.run([cross_entropy_mean,\n",
    "                                                            update_cm,vgg_fcn.pred_up,labels_tf],feed_dict={train_mode: False})\n",
    "                plt.figure(1)\n",
    "                plt.imshow(val_label_pic[0],cmap=cmap,norm=norm)\n",
    "                plt.figure(2)\n",
    "                plt.imshow(val_pred_pic[0],cmap=cmap,norm=norm)\n",
    "                #plt.figure(3)\n",
    "                #plt.imshow(output2[0],cmap=cmap,norm=norm)\n",
    "                plt.show()\n",
    "            else:\n",
    "                    val_loss,_ = sess.run([cross_entropy_mean,update_cm],feed_dict={train_mode: False})\n",
    "            val_loss_list.append(val_loss)\n",
    "            sys.stdout.write('\\r\\r' + str(iteration))\n",
    "            sys.stdout.flush()\n",
    "        \n",
    "        val_iou=sess.run(miou)\n",
    "        val_loss_mean = np.mean(val_loss_list)\n",
    "        plot_val_loss.append(val_loss_mean)\n",
    "        \n",
    "        val_loss_list = []\n",
    "        plot_val_miou.append(val_iou)\n",
    "        \n",
    "        var_miou = filter(lambda x: x.name.endswith('total_confusion_matrix:0'), tf.local_variables())\n",
    "        init_miou=tf.initialize_variables(var_miou)\n",
    "        sess.run(init_miou)\n",
    "        \n",
    "        clear_output()\n",
    "        print \"Epoch\", epoch\n",
    "        \n",
    "        print \"===========**Training ACCURACY**================\"\n",
    "        print 'Training mIOU: ', train_iou\n",
    "        print \"Training Loss:\", train_loss_mean \n",
    "        print \"Training Loss:\", total_train_loss_mean \n",
    "        \n",
    "        print \"===========**VALIDATION ACCURACY**================\"\n",
    "        print 'Validation mIOU: ', val_iou\n",
    "        print \"Validation Loss:\", val_loss_mean \n",
    "        \n",
    "        print \"=================**TIME**====================\"\n",
    "        print 'Time Elapsed for Epoch:' + str(epoch) + ' is ' + str(\n",
    "                (time.time() - epoch_start_time) / 60.) + ' minutes'\n",
    "        \n",
    "        plt.figure(1) \n",
    "        dd = plt.plot(plot_total_train_loss,'r',label=\"Training\")\n",
    "        plt.title(\"Total LOSS\")\n",
    "        plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n",
    "        plt.figure(2) \n",
    "        aa = plt.plot(plot_train_loss,'r',label=\"Training\")\n",
    "        aav = plt.plot(plot_val_loss,'g',label=\"Validation\")\n",
    "        plt.title(\"LOSS\")\n",
    "        plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n",
    "        plt.figure(3)\n",
    "        cc = plt.plot(plot_train_miou,'r',label=\"Training\")\n",
    "        aav = plt.plot(plot_val_miou,'g',label=\"Validation\")\n",
    "        plt.title(\"mIOU\")\n",
    "        plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n",
    "        plt.show()\n",
    "        saver.save(sess, ckpt_dir, global_step=epoch)\n",
    "        ofile  = open(data_csv_path, \"a\")\n",
    "        writer = csv.writer(ofile)\n",
    "        timep=((time.time() - epoch_start_time) / 60.)\n",
    "        writer.writerow([train_loss_mean,train_iou,timep,val_loss_mean,val_iou,epoch])\n",
    "        ofile.close()\n",
    "except tf.errors.OutOfRangeError:\n",
    "    print (\"out of range\")\n",
    "finally:\n",
    "    coord.request_stop()\n",
    "    coord.join(threads)\n",
    "    print(\"requesting stop\")\n",
    "        #print(iou[1])\n",
    "            #train_loss= sess.run(vgg_fcn.upscore,\n",
    "             #   feed_dict={ images_tf: train_imbatch})\n",
    "            #res=sess.run(image)\n",
    "            #plt.imshow(train_labatch[0])\n",
    "            #plt.show()\n",
    "    #    aa= tt * np.log10(train_loss)\n",
    "    #    print(len(aa))\n",
    "    #    print(len(aa[0]))\n",
    "    #print(sum)\n",
    "    #print(sum2)\n",
    "    #print(len(ww))\n",
    "    #print(len(ww[0]))\n",
    "    #print(len(ww[0][0]))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file saved /mnt/data3/FCN32_gap/rel2/npy/fcn32-rel2-gap-epoch-50.npy\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'/mnt/data3/FCN32_gap/rel2/npy/fcn32-rel2-gap-epoch-50.npy'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vgg_fcn.save_npy(sess, \"/mnt/data3/FCN32_gap/rel2/npy/fcn32-rel2-gap-epoch-\" + str(epoch) + \".npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}