prepare-finetuning-vgg16.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine-tuning an $\\alpha$-pooling model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use a custom caffe framework, which implements a SignedPowerLayer. Please make sure to clone and make it before using this script and add the path to caffe in the following box."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import sys\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "sys.path.append('/home/simon/Research/lib/caffe/python')\n",
    "sys.path.append('/home/simon/Research/finegrained/src/part_model_layer/part_autoencoder')\n",
    "sys.path.append('/home/simon/Research/generic/src/bilinear_logm/')\n",
    "\n",
    "import caffe\n",
    "import scipy.misc\n",
    "import h5py\n",
    "import scipy.io\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import glob\n",
    "import time\n",
    "%matplotlib inline  \n",
    "import os\n",
    "import matplotlib\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import google.protobuf\n",
    "import uuid\n",
    "import pyprind\n",
    "import random\n",
    "import google.protobuf.text_format\n",
    "caffe.set_device(0)\n",
    "caffe.set_mode_gpu()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following box contains most things you might want to adjust"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initial value for alpha, called gamma in this file\n",
    "gamma = 2.0\n",
    "chop_off_layer = 'relu5_3'\n",
    "# Resize images to this size before cropping for data augmentation\n",
    "resize_size = 640\n",
    "# Actual crop size\n",
    "crop_size = 560\n",
    "# The resolutions to extract alpha-pooling features from\n",
    "resolutions = [224,560]\n",
    "prefix_template = 'res%i/'\n",
    "# Number of object classes\n",
    "num_classes = 201\n",
    "init_model = './vgg16-training/vgg16_imagenet.caffemodel'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create parameter files\n",
    "# Net\n",
    "netparams_in = caffe.proto.caffe_pb2.NetParameter()\n",
    "protofile = './vgg16-training/train_val.prototxt'\n",
    "google.protobuf.text_format.Merge(open(protofile).read(),netparams_in)\n",
    "\n",
    "# Solver\n",
    "params = caffe.proto.caffe_pb2.SolverParameter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change to working dir\n",
    "working_dir = 'finetuning/finetuning_%s'%(str(uuid.uuid4()))\n",
    "try: os.makedirs(working_dir) \n",
    "except: pass\n",
    "os.chdir(working_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add second branch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this section, we take a prepared prototxt and adjust it for our needs. You might want to adjust the path to the image data here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare data layer\n",
    "lyr = netparams_in.layer\n",
    "lyr[0].image_data_param.source = '/home/simon/Datasets/CUB_200_2011/train_images.txt'\n",
    "lyr[0].image_data_param.root_folder = '/home/simon/Datasets/CUB_200_2011/images/'\n",
    "lyr[0].image_data_param.batch_size = 8\n",
    "lyr[0].image_data_param.smaller_side_size[0] = resize_size\n",
    "#lyr[0].image_data_param.smaller_side_size[1] = crop_size\n",
    "lyr[0].transform_param.crop_size = crop_size\n",
    "lyr[0].type = 'ImageData'\n",
    "\n",
    "lyr[1].image_data_param.source = '/home/simon/Datasets/CUB_200_2011/test_images.txt'\n",
    "lyr[1].image_data_param.root_folder = '/home/simon/Datasets/CUB_200_2011/images/'\n",
    "lyr[1].image_data_param.batch_size = 1\n",
    "lyr[1].image_data_param.smaller_side_size[0] = resize_size\n",
    "#lyr[1].image_data_param.smaller_side_size[1] = crop_size\n",
    "lyr[1].transform_param.crop_size = crop_size\n",
    "lyr[1].type = 'ImageData'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add batch norm\n",
    "netparams = caffe.proto.caffe_pb2.NetParameter()\n",
    "netparams.name = netparams_in.name\n",
    "\n",
    "bilinear_outputs = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input layers\n",
    "for idx, l in enumerate(netparams_in.layer):\n",
    "    if l.type in ['ImageData', 'Data']:\n",
    "        netparams.layer.add()\n",
    "        netparams.layer[-1].MergeFrom(l)\n",
    "\n",
    "for idx, l in enumerate(netparams_in.layer):\n",
    "    if l.type in ['ImageData', 'Data']:\n",
    "        netparams.layer.add()\n",
    "        netparams.layer[-1].name = 'zeros'\n",
    "        netparams.layer[-1].type = 'DummyData'\n",
    "        netparams.layer[-1].top.append('zeros')\n",
    "        netparams.layer[-1].dummy_data_param.shape.add()\n",
    "        netparams.layer[-1].dummy_data_param.shape[0].dim.extend([l.image_data_param.batch_size,1])\n",
    "        netparams.layer[-1].include.add()\n",
    "        netparams.layer[-1].include[0].phase = l.include[0].phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Resize layers\n",
    "for res_idx, res in enumerate(resolutions):\n",
    "    prefix = prefix_template%res \n",
    "    netparams.layer.add()\n",
    "    netparams.layer[-1].name = prefix + netparams_in.layer[0].top[0]\n",
    "    netparams.layer[-1].type = 'SpatialTransformer'\n",
    "    netparams.layer[-1].bottom.append(netparams_in.layer[0].top[0])\n",
    "    netparams.layer[-1].bottom.append('zeros')\n",
    "    netparams.layer[-1].top.append(netparams.layer[-1].name)\n",
    "    netparams.layer[-1].st_param.theta_1_1 = 1\n",
    "    netparams.layer[-1].st_param.theta_1_2 = 0\n",
    "    netparams.layer[-1].st_param.theta_1_3 = 0\n",
    "    netparams.layer[-1].st_param.theta_2_1 = 0\n",
    "    netparams.layer[-1].st_param.theta_2_2 = 1\n",
    "    #netparams.layer[-1].st_param.theta_2_3 = 0\n",
    "    netparams.layer[-1].st_param.to_compute_dU = False\n",
    "    netparams.layer[-1].st_param.output_H = res;\n",
    "    netparams.layer[-1].st_param.output_W = res;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for each resolution\n",
    "for res_idx, res in enumerate(resolutions):\n",
    "    # Add all  layers before chop_off\n",
    "    for idx, l in enumerate(netparams_in.layer):\n",
    "        if l.type in ['ImageData', 'Data']:\n",
    "            continue\n",
    "        netparams.layer.add()\n",
    "        netparams.layer[-1].MergeFrom(l)\n",
    "        prefix = prefix_template%res \n",
    "        netparams.layer[-1].name = prefix + netparams.layer[-1].name \n",
    "        for i in range(len(l.top)):\n",
    "            netparams.layer[-1].top[i] = prefix + netparams.layer[-1].top[i]\n",
    "        for i in range(len(l.bottom)):\n",
    "            netparams.layer[-1].bottom[i] = prefix + netparams.layer[-1].bottom[i]\n",
    "        for param_idx, p in enumerate(netparams.layer[-1].param):\n",
    "            p.name = '%s_param%i'%(l.name,param_idx)\n",
    "\n",
    "        if l.name == chop_off_layer:\n",
    "            break\n",
    "\n",
    "    # Add gamma layer\n",
    "    netparams.layer.add()\n",
    "    netparams.layer[-1].name = prefix + 'gamma_power'\n",
    "    netparams.layer[-1].type = 'SignedPower'\n",
    "    netparams.layer[-1].bottom.append(netparams.layer[-2].top[0])\n",
    "    netparams.layer[-1].top.append(netparams.layer[-1].name)\n",
    "    netparams.layer[-1].power_param.power = gamma - 1\n",
    "    netparams.layer[-1].param.add()\n",
    "    netparams.layer[-1].param[0].name = 'gamma_power'\n",
    "    netparams.layer[-1].param[0].lr_mult = 10\n",
    "    netparams.layer[-1].param[0].decay_mult = 0\n",
    "\n",
    "    # Add bilinear layers \n",
    "    netparams.layer.add()\n",
    "    netparams.layer[-1].name = prefix + 'bilinear'\n",
    "    netparams.layer[-1].type = 'CompactBilinear'\n",
    "    netparams.layer[-1].bottom.append(netparams.layer[-3].top[0])\n",
    "    netparams.layer[-1].bottom.append(netparams.layer[-2].top[0])\n",
    "    netparams.layer[-1].top.append(netparams.layer[-1].name)\n",
    "    netparams.layer[-1].compact_bilinear_param.num_output = 8192\n",
    "\n",
    "    bilinear_outputs.append(netparams.layer[-1].top[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalization layers\n",
    "if len(bilinear_outputs)>1:\n",
    "    netparams.layer.add()\n",
    "    netparams.layer[-1].name = 'bilinear_sum'\n",
    "    netparams.layer[-1].type = 'Eltwise'\n",
    "    for bi_out in bilinear_outputs:\n",
    "        netparams.layer[-1].bottom.append(bi_out)\n",
    "    netparams.layer[-1].top.append(netparams.layer[-1].name)\n",
    "\n",
    "if True:\n",
    "    netparams.layer.add()\n",
    "    netparams.layer[-1].name = 'bilinear_gamma_root'\n",
    "    netparams.layer[-1].type = 'SignedPower'\n",
    "    netparams.layer[-1].bottom.append(netparams.layer[-2].name)\n",
    "    netparams.layer[-1].top.append(netparams.layer[-1].name)\n",
    "    netparams.layer[-1].power_param.power = 0.5 #1.0 / (gamma)\n",
    "    netparams.layer[-1].param.add()\n",
    "    netparams.layer[-1].param[0].lr_mult = 0\n",
    "    netparams.layer[-1].param[0].decay_mult = 0\n",
    "\n",
    "if True:\n",
    "    netparams.layer.add()\n",
    "    netparams.layer[-1].name = 'bilinear_l2'\n",
    "    netparams.layer[-1].type = 'L2Normalize'\n",
    "    netparams.layer[-1].bottom.append(netparams.layer[-2].top[0])\n",
    "    netparams.layer[-1].top.append(netparams.layer[-1].name)\n",
    "\n",
    "# fc8\n",
    "netparams.layer.add()\n",
    "netparams.layer[-1].name = 'fc8_ft'\n",
    "netparams.layer[-1].type = 'InnerProduct'\n",
    "netparams.layer[-1].bottom.append(netparams.layer[-2].top[0])\n",
    "netparams.layer[-1].top.append(netparams.layer[-1].name) \n",
    "netparams.layer[-1].inner_product_param.num_output = num_classes\n",
    "[netparams.layer[-1].param.add() for _ in range(2)]\n",
    "netparams.layer[-1].param[0].lr_mult = 1\n",
    "netparams.layer[-1].param[0].decay_mult = 1\n",
    "netparams.layer[-1].param[1].lr_mult = 2\n",
    "netparams.layer[-1].param[1].decay_mult = 2\n",
    "\n",
    "# Accuracy\n",
    "netparams.layer.add()\n",
    "netparams.layer[-1].name = 'loss'\n",
    "netparams.layer[-1].type = 'SoftmaxWithLoss'\n",
    "netparams.layer[-1].bottom.append(netparams.layer[-2].top[0])\n",
    "netparams.layer[-1].bottom.append('label')\n",
    "netparams.layer[-1].top.append(netparams.layer[-1].name) \n",
    "\n",
    "# Softmax\n",
    "netparams.layer.add()\n",
    "netparams.layer[-1].name = 'Accuracy'\n",
    "netparams.layer[-1].type = 'Accuracy'\n",
    "netparams.layer[-1].bottom.append(netparams.layer[-3].top[0])\n",
    "netparams.layer[-1].bottom.append('label')\n",
    "netparams.layer[-1].top.append(netparams.layer[-1].name) \n",
    "netparams.layer[-1].include.add()\n",
    "netparams.layer[-1].include[0].phase = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Learning rates and decays and so on\n",
    "for l in netparams.layer:\n",
    "    if l.type in ['InnerProduct','Convolution','Scale']:\n",
    "        [l.param.add() for _ in range(2 - len(l.param))]\n",
    "        l.param[0].lr_mult = 1\n",
    "        l.param[0].decay_mult = 1\n",
    "        l.param[1].lr_mult = 2\n",
    "        l.param[1].decay_mult = 2\n",
    "    if l.type in ['InnerProduct']:\n",
    "        l.inner_product_param.weight_filler.type = \"gaussian\"\n",
    "        l.inner_product_param.weight_filler.ClearField('std')\n",
    "        l.inner_product_param.weight_filler.std = 0.01\n",
    "        l.inner_product_param.bias_filler.type = \"constant\"\n",
    "        l.inner_product_param.bias_filler.value = 0.0\n",
    "    if l.name in ['fc8_ft']:\n",
    "        l.inner_product_param.weight_filler.type = \"gaussian\"\n",
    "        l.inner_product_param.weight_filler.std = 0.000000001\n",
    "        l.inner_product_param.bias_filler.type = \"constant\"\n",
    "        l.inner_product_param.bias_filler.value = 0.01\n",
    "    if l.type in ['Convolution']:\n",
    "        l.convolution_param.weight_filler.type = \"gaussian\"\n",
    "        l.convolution_param.weight_filler.ClearField('std')\n",
    "        l.inner_product_param.weight_filler.std = 0.01\n",
    "        l.convolution_param.bias_filler.type = \"constant\"\n",
    "        l.convolution_param.bias_filler.value = 0.0\n",
    "    if l.type == \"BatchNorm\":\n",
    "        l.param[0].lr_mult = 0\n",
    "        l.param[1].lr_mult = 0\n",
    "        l.param[2].lr_mult = 0\n",
    "        l.batch_norm_param.ClearField('use_global_stats')\n",
    "#    if l.name in ['fc6','fc7']:\n",
    "#        l.inner_product_param.num_output = 2048"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Solver for fine-tuning\n",
    "solverfile = 'ft.solver'\n",
    "params = caffe.proto.caffe_pb2.SolverParameter()\n",
    "params.net = u'ft.prototxt'\n",
    "params.test_iter.append(int(len([None for _ in open(netparams.layer[1].image_data_param.source,'rt')]) / lyr[0].image_data_param.batch_size))\n",
    "params.test_interval = 10000\n",
    "params.test_initialization = True\n",
    "params.base_lr = 0.001\n",
    "params.display = 100\n",
    "params.max_iter = 1000000\n",
    "params.lr_policy = \"fixed\"\n",
    "params.power = 1\n",
    "#params.stepsize = 100000\n",
    "#params.gamma = 0.1\n",
    "#params.momentum = 0.9\n",
    "params.weight_decay = 0.0005\n",
    "params.snapshot = 10000\n",
    "#params.random_seed = 0\n",
    "params.snapshot_prefix = \"ft\"\n",
    "params.net = \"ft.prototxt\"\n",
    "params.iter_size = int(8/lyr[0].image_data_param.batch_size)\n",
    "#params.type = \"Nesterov\"\n",
    "assert params.iter_size > 0\n",
    "open(solverfile,'w').write(google.protobuf.text_format.MessageToString(params))\n",
    "open(params.net,'w').write(google.protobuf.text_format.MessageToString(netparams))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copy the weights from the pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net_origin = caffe.Net('../../'+protofile, '../../'+init_model, caffe.TEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net_target = caffe.Net('ft.prototxt',caffe.TEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for origin_param in net_origin.params.keys():\n",
    "    for res in resolutions:\n",
    "        prefix = prefix_template%res\n",
    "        target_param = prefix + origin_param\n",
    "        if target_param in net_target.params:\n",
    "            for idx in range(len(net_origin.params[origin_param])):\n",
    "                #print('Copying %s[%i] to %s[%i]'%(origin_param, idx, target_param, idx))\n",
    "                net_target.params[target_param][idx].data[...] = net_origin.params[origin_param][idx].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False: net_target.copy_from(init_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net_target.save('model_init')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del net_origin\n",
    "del net_target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Caffe LR init"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To speed everything up, we calculate features for each image and learn only the classifier with it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Calc the features\n",
    "def calc_features(net, n_images, blobs):\n",
    "    batchsize = net.blobs['data'].data.shape[0]\n",
    "    feats = dict()\n",
    "    for blob in blobs:\n",
    "        out_shape = list(net.blobs[blob].data.shape)\n",
    "        out_shape[0] = n_images\n",
    "        feats[blob] = np.zeros(tuple(out_shape),dtype=np.float16 if not blob=='label' else np.int32)\n",
    "    print('Need %.3f GiB'%(np.sum([x.nbytes for x in feats.values()])/1024/1024/1024))\n",
    "        \n",
    "    for it in pyprind.prog_bar(range(0,n_images,batchsize),update_interval=10):\n",
    "        net.forward()\n",
    "        for blob in blobs:\n",
    "            feats[blob][it:it+batchsize,...] = net.blobs[blob].data[:feats[blob][it:it+batchsize,...].shape[0],...]\n",
    "            \n",
    "    return [feats[blob] for blob in blobs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_images = [len([None for _ in open(netparams.layer[i].image_data_param.source,'r')]) for i in [0,1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_blob = [l.bottom[0] for l in netparams.layer if l.type == 'InnerProduct'][-1]\n",
    "last_blob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "solver = caffe.get_solver('ft.solver')\n",
    "solver.net.copy_from('model_init')\n",
    "train_feats,train_labels = calc_features(solver.net,num_images[0],[last_blob,'label'])\n",
    "del solver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    solver = caffe.get_solver('ft.solver')\n",
    "    solver.test_nets[0].copy_from('model_init')\n",
    "    val_feats,val_labels = calc_features(solver.test_nets[0],num_images[1],[last_blob, 'label'])\n",
    "    del solver.test_nets[0]\n",
    "    del solver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "netparams_fixed = caffe.proto.caffe_pb2.NetParameter()\n",
    "netparams_fixed.layer.add()\n",
    "netparams_fixed.layer[-1].name = 'data'\n",
    "netparams_fixed.layer[-1].type = 'Input'\n",
    "netparams_fixed.layer[-1].top.append(last_blob)\n",
    "netparams_fixed.layer[-1].input_param.shape.add()\n",
    "netparams_fixed.layer[-1].input_param.shape[0].dim.extend((32,) + train_feats.shape[1:])\n",
    "\n",
    "netparams_fixed.layer.add()\n",
    "netparams_fixed.layer[-1].name = 'label'\n",
    "netparams_fixed.layer[-1].type = 'Input'\n",
    "netparams_fixed.layer[-1].top.append('label')\n",
    "netparams_fixed.layer[-1].input_param.shape.add()\n",
    "netparams_fixed.layer[-1].input_param.shape[0].dim.extend((32,))\n",
    "# Add all layers after fc8\n",
    "approached_fc8 = False\n",
    "for l in netparams.layer:\n",
    "    if l.name == 'fc8_ft':\n",
    "        l.param[0].lr_mult = 1\n",
    "        l.param[0].decay_mult = 1\n",
    "        l.param[1].lr_mult = 1\n",
    "        l.param[1].decay_mult = 1\n",
    "        l.inner_product_param.weight_filler.std = 0.0001\n",
    "        l.inner_product_param.bias_filler.value = 0\n",
    "    approached_fc8 = approached_fc8 or l.name == 'fc8_ft'\n",
    "    if approached_fc8:\n",
    "        netparams_fixed.layer.add()\n",
    "        netparams_fixed.layer[-1].MergeFrom(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Solver\n",
    "solverfile = 'ft_fixed.solver'\n",
    "params = caffe.proto.caffe_pb2.SolverParameter()\n",
    "params.net = u'ft_fixed.prototxt'\n",
    "#params.test_iter.append(1450)\n",
    "#params.test_interval = 1000\n",
    "params.test_initialization = False\n",
    "params.base_lr = 1\n",
    "params.display = 100\n",
    "params.max_iter = 60000\n",
    "params.lr_policy = \"multistep\"\n",
    "params.stepvalue.extend([20000,30000,40000,50000])\n",
    "#params.power = 1\n",
    "#params.stepsize = 100000\n",
    "params.gamma = 0.25\n",
    "params.momentum = 0.9\n",
    "params.weight_decay = 0.000005\n",
    "params.snapshot = 10000000\n",
    "#params.random_seed = 0\n",
    "params.snapshot_prefix = \"ft_fixed\"\n",
    "params.iter_size = 1\n",
    "assert params.iter_size > 0\n",
    "open(solverfile,'w').write(google.protobuf.text_format.MessageToString(params))\n",
    "open(params.net,'w').write(google.protobuf.text_format.MessageToString(netparams_fixed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "solver = caffe.get_solver('ft_fixed.solver')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train\n",
    "for it in pyprind.prog_bar(range(params.max_iter)):\n",
    "    train_ids = random.sample(range(train_feats.shape[0]),32)\n",
    "    solver.net.blobs[last_blob].data[...] = train_feats[train_ids,...]\n",
    "    solver.net.blobs['label'].data[...] = train_labels[train_ids]\n",
    "    solver.step(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "solver.net.save('model_lr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del solver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "solver = caffe.get_solver('ft.solver')\n",
    "solver.net.copy_from('model_init')\n",
    "solver.net.copy_from('model_lr')\n",
    "solver.net.save('model_lr')\n",
    "del solver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    import sklearn\n",
    "\n",
    "    model = sklearn.linear_model.LogisticRegression(C=1000, solver='lbfgs',multi_class='multinomial', max_iter = 10000, tol = 1e-10)\n",
    "    %time model.fit(train_feats.reshape(train_feats.shape[0],-1), train_labels)\n",
    "    print(\"LR Accuracy is \")\n",
    "    print(model.score(val_feats.reshape(val_feats.shape[0],-1), val_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}