main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# coding: utf-8\n",
    "# Please use Python 2.\n",
    "# Check if all the required packages are installed.\n",
    "# Please use Theano as backend of Keras.\n",
    "# Using IPython or Jupyter Notebook to run this code is strongly recommended!\n",
    "# Run line by line or block by block, and you can config some important parameters\n",
    "# and output whatever you want!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import sys, os, collections, re, commands, csv\n",
    "import nltk, codecs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import DataFrame, Series\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from util import *\n",
    "from PairWiseRanker import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pkuas/chenqy/anaconda2/lib/python2.7/site-packages/IPython/core/magics/pylab.py:161: UserWarning: pylab import has clobbered these variables: ['copy']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    }
   ],
   "source": [
    "% pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "UNKNOWN_WORD = ':-)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir = 'data/'\n",
    "fn = {'trn':data_dir + 'WikiQA-train.tsv', 'dev':data_dir + 'WikiQA-dev.tsv', 'test':data_dir + 'WikiQA-test.tsv'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word2vec_dir = '/store1/chenqy/'\n",
    "EMBEDDING_DIM = 50\n",
    "word2vec_fn = word2vec_dir + 'glove.6B.%dd.txt' % EMBEDDING_DIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#read_data = lambda fn : pd.read_csv(fn, sep='\\t', header=0, index_col='QuestionID')\n",
    "def read_data(f):\n",
    "    data = pd.read_csv(f, sep='\\t', header=0, quoting=csv.QUOTE_NONE)\n",
    "    data['QAID'] = data['QuestionID'] + '*' + data['SentenceID']\n",
    "    data.index = data['QAID']\n",
    "    for col in ['Question', 'Sentence']:\n",
    "        data[col] = map(lambda x : codecs.decode(x.lower(), 'UTF-8'), data[col])\n",
    "    questions = dict(zip(data['QuestionID'], data['Question']))\n",
    "    return data, questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test (2351, 7) 2351\n",
      "dev (1130, 8) 1130\n",
      "trn (20360, 8) 20360\n"
     ]
    }
   ],
   "source": [
    "data, questions = {}, {}\n",
    "for k in fn:\n",
    "    data[k], questions[k] = read_data(fn[k])\n",
    "    print k, data[k].shape, len(set(data[k]['QAID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_trn, data_dev, data_test = data['trn'], data['dev'], data['test'] # for convienience"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev \n",
      "1    115\n",
      "2      9\n",
      "4      1\n",
      "3      1\n",
      "Name: Label, dtype: int64\n",
      "trn \n",
      "0    1245\n",
      "1     745\n",
      "2     103\n",
      "3      20\n",
      "7       2\n",
      "5       1\n",
      "6       1\n",
      "4       1\n",
      "Name: Label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# distribution of number of answers each question has\n",
    "for k in fn:\n",
    "    if k == 'test':\n",
    "        continue\n",
    "    grp = data[k]['Label'].groupby(data[k]['QuestionID'])\n",
    "    print k, '\\n', grp.agg(lambda v : v.sum()).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2118 126 243 2487 2487\n"
     ]
    }
   ],
   "source": [
    "t = set(questions['trn']), set(questions['dev']), set(questions['test'])\n",
    "print len(t[0]), len(t[1]), len(t[2]), len(t[0] | t[1] | t[2]), sum([len(v) for v in t])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# qtokens = {k:{qid:nltk.word_tokenize(q) for qid, q in questions[k].items() } for k in fn }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wnl = WordNetLemmatizer()\n",
    "regex_tokenizer = RegexpTokenizer(r'\\w+')\n",
    "lancaster = nltk.LancasterStemmer()\n",
    "def sentence2words(sent):\n",
    "    res = {}\n",
    "    res['tokens'] = nltk.word_tokenize(sent)\n",
    "    res['words'] = regex_tokenizer.tokenize(sent)\n",
    "    res['lemma'] = [wnl.lemmatize(w) for w in res['words']]\n",
    "    res['stems'] = [lancaster.stem(w) for w in res['words']]\n",
    "    res['words_not_stop'] = filter(lambda word : word not in stopwords.words('english'), res['words'])\n",
    "    res['stems_not_stop'] = [lancaster.stem(w) for w in res['words_not_stop']]\n",
    "    res['lemma_not_stop'] = [wnl.lemmatize(w) for w in res['words_not_stop']]\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tps = ['words', 'stems', 'words_not_stop', 'stems_not_stop', 'lemma', 'lemma_not_stop']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sent_words = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for k in fn:# fn:\n",
    "    for qid, q in questions[k].items():\n",
    "        sent_words[qid] = sentence2words(q)\n",
    "    d = data[k]\n",
    "    for qaid in d.index:\n",
    "        sent_words[qaid] = sentence2words(d.get_value(qaid, 'Sentence'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#first_word = lambda sent : sent[:sent.find(' ')]\n",
    "first_token = lambda tokens : tokens[0]\n",
    "second_token = lambda tokens : UNKNOWN_WORD if len(tokens) == 1 else tokens[1]\n",
    "first_two_tokens = lambda tokens : ' '.join(tokens[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for k in fn:\n",
    "    for col, func in [('token1', first_token), ('token2', second_token), ('token12', first_two_tokens)]:\n",
    "        qids = list(set(data[k]['QuestionID']))\n",
    "        tmp = Series(map(func, map(lambda qid : sent_words[qid]['words'], qids)), index=qids)\n",
    "        data[k][col] = tmp[data[k]['QuestionID']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_question_type(data_x):\n",
    "    qtype = deepcopy(Series(data_x['token1'], index=data_x.index))\n",
    "    qtype[data_x['token12'] == 'what year'] = 'time'\n",
    "    qtype[data_x['token1'] == 'when'] = 'time'\n",
    "    qtype[data_x['token12'] == 'how many'] = 'number'\n",
    "    qtype[data_x['token1'] == 'where'] = 'place'\n",
    "    qtype[data_x['token1'] == 'who'] = 'person'\n",
    "    return qtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 338, 286)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(data_trn.token1)), len(set(data_trn.token12)), len(set(data_trn.token2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for k in fn:\n",
    "    data[k]['qtype'] = get_question_type(data[k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test \n",
      "\n",
      "what     52.956189\n",
      "how      16.205870\n",
      "who      11.569545\n",
      "where    10.421097\n",
      "when      8.847299\n",
      "Name: token1, dtype: float64\n",
      "what is      22.288388\n",
      "what are      5.827308\n",
      "when did      5.784772\n",
      "how many      5.019141\n",
      "what year     3.487877\n",
      "where was     2.934921\n",
      "where is      2.892386\n",
      "what does     2.722246\n",
      "how did       2.254360\n",
      "who is        2.254360\n",
      "Name: token12, dtype: float64\n",
      "\n",
      "\n",
      "dev \n",
      "\n",
      "what        56.194690\n",
      "how         12.035398\n",
      "when        11.946903\n",
      "where       10.000000\n",
      "who          9.646018\n",
      "whatever     0.176991\n",
      "Name: token1, dtype: float64\n",
      "what is      14.424779\n",
      "what are     10.265487\n",
      "when did      9.203540\n",
      "how many      6.725664\n",
      "where is      5.132743\n",
      "what does     4.070796\n",
      "what was      3.982301\n",
      "what year     3.539823\n",
      "who is        2.743363\n",
      "when was      2.743363\n",
      "Name: token12, dtype: float64\n",
      "\n",
      "\n",
      "trn \n",
      "\n",
      "what     49.695481\n",
      "how      20.859528\n",
      "who      12.568762\n",
      "when      9.228880\n",
      "where     7.647348\n",
      "Name: token1, dtype: float64\n",
      "what is      19.965619\n",
      "how many      7.509823\n",
      "what are      4.705305\n",
      "when was      3.408644\n",
      "what does     3.163065\n",
      "when did      2.991159\n",
      "who is        2.946955\n",
      "where is      2.946955\n",
      "how did       2.254420\n",
      "what was      1.940079\n",
      "Name: token12, dtype: float64\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for k in fn:\n",
    "    print k, '\\n'\n",
    "    for tk in ['token1', 'token12']:\n",
    "        print data[k][tk].value_counts().head(10) / float(data[k].shape[0]) * 100\n",
    "    print '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "words_idf = dict()\n",
    "for tp in tps:\n",
    "    words_idf[tp] = dict()\n",
    "\n",
    "for tp in tps:\n",
    "    for qaid in sent_words:\n",
    "        for word in set(sent_words[qaid][tp]):\n",
    "            if (word in words_idf[tp]) == True:\n",
    "                words_idf[tp][word] = words_idf[tp][word] + 1\n",
    "            else:\n",
    "                words_idf[tp][word] = 1\n",
    "\n",
    "DocNum = len(sent_words)\n",
    "for tp in tps:\n",
    "    for w in words_idf[tp]:\n",
    "        words_idf[tp][w] = math.log(DocNum / words_idf[tp][w], 2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def count_num_cooccur_weight(x, y, tp):\n",
    "    z = set(x) & set(y)\n",
    "    s = 0\n",
    "    for w in z:\n",
    "        s = s + words_idf[tp][w]\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#word embedding:\n",
    "# readEmbedFile\n",
    "from word2vec_util import * \n",
    "wordvecs = readEmbedFile(word2vec_fn)\n",
    "dim = EMBEDDING_DIM\n",
    "\n",
    "wordset2 = set()\n",
    "for tp in tps:\n",
    "    for w in words_idf[tp]:\n",
    "        wordset2.add(w)\n",
    "\n",
    "new_wv=dict()\n",
    "for w in wordvecs:\n",
    "    if (w in wordset2) == True:\n",
    "        new_wv[w] = wordvecs[w]\n",
    "wordvecs = new_wv\n",
    "new_wv = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#using this!!!\n",
    "def count_wv_dis(x, y):\n",
    "    vx = [0 for c in range(dim)]\n",
    "    wn = 0\n",
    "    for w in x:\n",
    "        if w in wordvecs:\n",
    "            wn = wn + 1\n",
    "            for d in range(dim):\n",
    "                vx[d] += wordvecs[w][d]\n",
    "    if wn > 0:\n",
    "        for d in range(dim):\n",
    "            vx[d] /= wn\n",
    "    vy = [0 for c in range(dim)]\n",
    "    wn = 0\n",
    "    for w in y:\n",
    "        if w in wordvecs:\n",
    "            wn = wn + 1\n",
    "            for d in range(dim):\n",
    "                vy[d] += wordvecs[w][d]\n",
    "    if wn > 0:\n",
    "        for d in range(dim):\n",
    "            vy[d] /= wn\n",
    "    dis = 0;\n",
    "    for d in range(dim):\n",
    "        dis += (vx[d] - vy[d])*(vx[d] - vy[d])\n",
    "    return 0 - dis # larger, better\n",
    "\n",
    "def count_wv_dis_weight(x, y, tp):\n",
    "    vx = [0 for c in range(dim)]\n",
    "    wn = 0\n",
    "    for w in x:\n",
    "        if w in wordvecs:\n",
    "            wn = wn + words_idf[tp][w]\n",
    "            for d in range(dim):\n",
    "                vx[d] += wordvecs[w][d]*words_idf[tp][w]\n",
    "    if wn > 0:\n",
    "        for d in range(dim):\n",
    "            vx[d] /= wn\n",
    "    vy = [0 for c in range(dim)]\n",
    "    wn = 0\n",
    "    for w in y:\n",
    "        if w in wordvecs:\n",
    "            wn = wn + words_idf[tp][w]\n",
    "            for d in range(dim):\n",
    "                vy[d] += wordvecs[w][d]*words_idf[tp][w]\n",
    "    if wn > 0:\n",
    "        for d in range(dim):\n",
    "            vy[d] /= wn\n",
    "    dis = 0;\n",
    "    for d in range(dim):\n",
    "        dis += (vx[d] - vy[d])*(vx[d] - vy[d])\n",
    "    return 0 - dis # larger, better\n",
    "\n",
    "# for k in fn:\n",
    "#     d = data[k]\n",
    "#     for tp in tps:\n",
    "#         func = lambda qaid : count_wv_dis_weight(sent_words[qaid][tp], sent_words[d.get_value(qaid, 'QuestionID')][tp], tp)\n",
    "#         features[k][tp] = d['QAID'].apply(func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Found 35188 unique tokens.\n",
      "23 165\n"
     ]
    }
   ],
   "source": [
    "encode = lambda s : codecs.encode(s, 'UTF-8')\n",
    "ques_utf8 = {}\n",
    "sent_utf8 = {}\n",
    "for k in fn:\n",
    "    for qid, q in questions[k].items():\n",
    "        ques_utf8[qid] = encode(q)\n",
    "    d = data[k]\n",
    "    for qaid in d.index:\n",
    "        sent_utf8[qaid] = encode(d.get_value(qaid, 'Sentence'))\n",
    "\n",
    "texts = []\n",
    "texts.extend(ques_utf8.values())\n",
    "texts.extend(sent_utf8.values())\n",
    "tokenizer = Tokenizer(nb_words=None)\n",
    "tokenizer.fit_on_texts(texts)\n",
    "ques_seq = {qid:tokenizer.texts_to_sequences([tx])[0] for qid, tx in ques_utf8.items()}\n",
    "sent_seq = {qaid:tokenizer.texts_to_sequences([tx])[0] for qaid, tx in sent_utf8.items()}\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "print max([len(seq) for seq in ques_seq.values()]), max([len(seq) for seq in sent_seq.values()])\n",
    "# MAX_SENT_SEQUENCE_LENGTH = max([max([len(seq) for seq in sents_seq[k]]) for k in fn])\n",
    "ques_len = Series([len(q) for q in ques_seq.values()])\n",
    "sent_len = Series([len(s) for s in sent_seq.values()])\n",
    "MAX_QUES_SEQUENCE_LENGTH, MAX_SENT_SEQUENCE_LENGTH = 20, 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "word2vec = pd.read_csv(word2vec_fn, sep=' ', header=None, index_col=0, nrows=None, quoting=csv.QUOTE_NONE)\n",
    "embedding_index = {}\n",
    "for i, w in enumerate(word2vec.index):\n",
    "    embedding_index[w] = word2vec.values[i]\n",
    "\n",
    "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embedding_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "words_bag = set(word2vec.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "words_idf_series = {tp:Series(words_idf[tp].values(), index=words_idf[tp].keys()) for tp in words_idf}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def words_of_sentence_to_vector(words, tp, use_weight=True):\n",
    "    shp = (len(words), EMBEDDING_DIM)\n",
    "    try:\n",
    "        if use_weight:\n",
    "            weight = np.repeat(words_idf_series[tp].loc[words], EMBEDDING_DIM).values.reshape(shp)\n",
    "            return (word2vec.loc[words] * weight).mean().values    \n",
    "        else:\n",
    "            return (word2vec.loc[words]).mean().values\n",
    "    except: # none of words are in word2vec\n",
    "        return np.zeros(shp)\n",
    "def get_sent_vetor_df(q_words):\n",
    "    q_vecs = dict(zip(q_words.keys(), map(lambda words : words_of_sentence_to_vector(words, 'words', use_weight=False), q_words.values())))\n",
    "    q_vecs = DataFrame(q_vecs).T\n",
    "    q_vecs_wgt = dict(zip(q_words.keys(), map(lambda words : words_of_sentence_to_vector(words, 'words', use_weight=True), q_words.values())))\n",
    "    q_vecs_wgt = DataFrame(q_vecs_wgt).T\n",
    "    #q_vecs_wgt.columns = map(lambda col : col)\n",
    "    q_vecs_merged = pd.concat([q_vecs, q_vecs_wgt], axis=1)\n",
    "    return q_vecs_merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## calculate features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = {key:DataFrame({}, index=data[key].index) for key in fn}\n",
    "features_trn = features['trn']\n",
    "features_dev = features['dev']\n",
    "features_test = features['test']\n",
    "tps_wgt = map(lambda v : v + '_wgt', tps)\n",
    "tps_wv = map(lambda v : v + '_wv', tps)\n",
    "tps_wv_wgt = map(lambda v : v + '_wgt', tps_wv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# words, stems\n",
    "for k in fn:\n",
    "    d = data[k]\n",
    "    for tp in tps:\n",
    "        func = lambda qaid : count_num_cooccur(sent_words[qaid][tp], sent_words[d.get_value(qaid, 'QuestionID')][tp])\n",
    "        features[k][tp] = d['QAID'].apply(func)\n",
    "        func = lambda qaid : count_num_cooccur_weight(sent_words[qaid][tp], sent_words[d.get_value(qaid, 'QuestionID')][tp], tp)\n",
    "        features[k][tp + '_wgt'] = d['QAID'].apply(func)\n",
    "        func = lambda qaid : count_wv_dis(sent_words[qaid][tp], sent_words[d.get_value(qaid, 'QuestionID')][tp])\n",
    "        features[k][tp + '_wv'] = d['QAID'].apply(func)\n",
    "        func = lambda qaid : count_wv_dis_weight(sent_words[qaid][tp], sent_words[d.get_value(qaid, 'QuestionID')][tp], tp)\n",
    "        features[k][tp + '_wv_wgt'] = d['QAID'].apply(func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# question type\n",
    "for k in fn:\n",
    "    f, d = features[k], data[k]\n",
    "    for col in ['token1', 'qtype']:\n",
    "        f[col] = data[k][col]\n",
    "    f['qtype_time'] = (d['qtype'] == 'time') + 0\n",
    "    f['qtype_number'] = (d['qtype'] == 'number') + 0\n",
    "    f['year'] = map(lambda sent : int(re.search('[0-9]{4}', sent) != None), d.Sentence)\n",
    "    f['number'] = map(lambda sent : int(re.search('[0-9]+', sent) != None), d.Sentence)\n",
    "    f['number'] = f['number'] - f['year']\n",
    "    f['num_unknown'] = map(lambda words : len(set(words) - words_bag), #np.sum(np.logical_not(np.in1d(words, word2vec.index)))\n",
    "                           map(lambda qaid : sent_words[qaid]['words'], d.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# qa_vectors = {}\n",
    "# for k in fn:\n",
    "#     q_words = {qid:sent_words[qid]['words'] for qid in set(data[k]['QuestionID'])}\n",
    "#     q_vec_df = get_sent_vetor_df(q_words)\n",
    "#     q_vec_df = q_vec_df.loc[data[k]['QuestionID']]\n",
    "#     q_vec_df.index = data[k].index\n",
    "#     a_words = {qaid:sent_words[qaid]['words'] for qaid in data[k].index}\n",
    "#     a_vec_df = get_sent_vetor_df(a_words)\n",
    "#     qa_vectors[k] = pd.concat([q_vec_df, a_vec_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dummies = {}\n",
    "features_all = pd.concat(features.values(), axis=0)\n",
    "dummies_all = pd.get_dummies(features_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((23841, 31), (23841, 42))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_all.shape, dummies_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test 0 2351 (2351, 42) (2351, 11)\n",
      "dev 2351 3481 (1130, 42) (1130, 12)\n",
      "trn 3481 23841 (20360, 42) (20360, 12)\n"
     ]
    }
   ],
   "source": [
    "idx = 0\n",
    "k_all = fn.keys()\n",
    "for i in range(len(k_all)):\n",
    "    k = k_all[i]\n",
    "    new_idx = idx + data[k].shape[0]\n",
    "    dummies[k] = dummies_all.iloc[idx:new_idx]\n",
    "    print k, idx, new_idx, dummies[k].shape, data[k].shape\n",
    "    idx = new_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from learning2rank.rank import RankNet, ListNet\n",
    "# reload(RankNet)\n",
    "# reload(ListNet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "indiv = [1560, 'entropy', 3, 5, 5] # (0.68945635493254542,) # 0.68482672530291577,) [48, 'gini', 3, 2, 4] #(0.68573375478137377,) [1734, 'entropy', 5, 3, 4] # (0.68961935330982949,)  # 49, 'entropy', 13, 3, 5]\n",
    "models['rf'] = model_rf = RandomForestClassifier(n_estimators=indiv[0], criterion=indiv[1], \n",
    "                                                 max_depth=indiv[2], min_samples_split=indiv[3], \n",
    "                                                 min_samples_leaf=indiv[4], n_jobs=-1) # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#models['ada'] = model_ada = AdaBoostClassifier(base_estimator=None, n_estimators=100, learning_rate=.5, algorithm='SAMME.R', random_state=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models['lr_l1'] = model_lr_l1 = LogisticRegression(penalty='l1', n_jobs=-1)\n",
    "models['lr_l2'] = model_lr_l2 = LogisticRegression(penalty='l2', n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#models['svm'] = model_svm = SVC(probability=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rk_models = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rk_models['ranknet'] = rk_model_ranknet = RankNet.RankNet(silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#rk_models['listnet'] = rk_model_listnet = ListNet.ListNet(silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my_models = {}\n",
    "pair_ranker = PairWiseRanker(data_trn.QuestionID, dummies['trn'], data_trn.Label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from ParamGA import * \n",
    "# g = RFGA(data_trn.QuestionID, dummies['trn'], data_trn.Label, data_dev.QuestionID, dummies['dev'], data_dev.Label, pair_wise=False)\n",
    "# g.run(NPOP=30, NGEN=20, CXPB=0.5, MUTPB=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "indiv =[327, 'gini', 5, 5, 1] #  (0.69239713108760736,)  # [746, 'gini', 7, 6, 4] #[746, 'gini', 7, 6, 4] # 0.679 \n",
    "# (0.69028072897120518,) [352, 'entropy', 5, 4, 5] # (0.69001617870665499,) [1977, 'gini', 6, 4, 5]\n",
    "my_models['my_rf'] = my_models_rf = RandomForestClassifier(n_estimators=indiv[0], criterion=indiv[1], \n",
    "                                                 max_depth=indiv[2], min_samples_split=indiv[3], \n",
    "                                                 min_samples_leaf=indiv[4], n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for mn, md in models.items():\n",
    "    md.fit(dummies['trn'].values, data_trn.Label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pkuas/chenqy/anaconda2/lib/python2.7/site-packages/numpy/lib/shape_base.py:422: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future\n",
      "  sub_arys.append(_nx.swapaxes(sary[st:end], axis, 0))\n",
      "  0%|          | 5/5000 [00:00<01:40, 49.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load dataset\n",
      "('The number of data, train:', 19342, 'validate:', 1018)\n",
      "prepare initialized model!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [00:46<00:00, 106.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save the model\n",
      "save the optimizer\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGHCAYAAABrpPKuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzsnXl4VOXZh+83IQtLCCSEsAiIKIIrm1VUrLhArYqitopS\n675Uq6JWrbXaT6vYuuBe0VZRURQEdy1uoFJFEAKigCiCIDuZEMIkkO39/njmJJPJLGfOnDMzgfe+\nrlxJZs6ceTOZOed3nuX3KK01BoPBYDAYDOlCRqoXYDAYDAaDwRCMEScGg8FgMBjSCiNODAaDwWAw\npBVGnBgMBoPBYEgrjDgxGAwGg8GQVhhxYjAYDAaDIa0w4sRgMBgMBkNaYcSJwWAwGAyGtMKIE4PB\nYDAYDGmFEScGg8FgMBjSCiNODAZDTJRSf1BK1Sulvkj1WgwGw+6PMrN1DAZDLJRSc4CuwN7Aflrr\nH1O7IoPBsDtjIicGgyEqSqnewJHA9cBW4LzUrig8Sqk2qV6DwWBwByNODAZDLM4DfMA7wKuEESdK\nuFYp9bVSqkoptVkp9Z5SalDIdmOVUl8qpfxKKZ9S6hOl1IlB99crpW4Ps//VSqlngn7/fWDbY5RS\nTyilNgFrA/f1DNy2XClVqZTaqpSaqpTqFWa/+UqpCUqpVUqpnUqptUqp55RSBUqptkqpHUqpCWEe\n110pVauUujmuV9JgMNiiVaoXYDAY0p5zgela61ql1BTgCqXUYK31gqBtngF+jwiYp5FjyzDgCGAh\ngFLqDuAO4H/AX4Fq4HBgOPBBjDVEyj8/AWwG/g9oG7jtsMDzTgF+RlJRfwBmKaUO0FrvDKynLTAH\n2B/4D1ACdAJGAXtprb9WSr0GnK2Uul43zYGfG/g+Oca6DQaDA4w4MRgMEVFKDQb6AVcBaK3nKKXW\nIdGTBYFthiPC5CGt9fVBD58QtJ8+iCCZrrX+TdA2jyW4xK3A8SHC4W2t9fSQv+MtYC5wJvBi4Oab\ngAOA0VrrN4M2vyfo5+cRIXIi8H7Q7ecBn2qt1yW4foPBEAaT1jEYDNE4D9gIzA667RXgHKWUCvx+\nJlAP3BllP6MBFWObeNHA0yHCBK31LutnpVQrpVQB8COwDQhOM50BLA4RJqF8CGwgKJWllDoIOAR4\nIeG/wGAwhMWIE4PBEBalVAZwNjAL2Ecp1ScQAZkHdAGOD2y6D7Bea70tyu72QQTMMpeXuTr0BqVU\nrlLqTqXUGmAXEl3ZDOQHviz6AN9E23lA+LwInK6Uyg3cfB5QhdTfGAwGDzDixGAwROI4pH34HOD7\noK9XkKhFMrt2MiPcXhXmtseAPwMvA79BUjInIEW9To55zwN5wOmB38cAb2mtKxzsy2Aw2MDUnBgM\nhkiMBTYhxaQq5L4zgdFKqSuAlcAIpVSHKNGTlYgwOAD4OspzlgEdgm9QSmUhIskuZwKTtNY3Be0j\nJ3S/gTUdFGtnWutvlVIlwHmBepueBGpwDAaDN5jIicFgaEYghTEaiRC8prWeEfyFRCfaI50t05Fj\nyR1Rdvk6Em25PahWJRwrgWNCbrucyJGTcNTR/Nh2TZh9TAcOVUqdZmOfLwAjgeuQNNF/41iPwWCI\nExM5MRgM4TgNSWVEKhadC2wBztNan66UegG4RinVFzlxZyCtxB9rrZ/QWq9USt0N3AZ8ppSagdSD\nHAas01r/JbDffwNPKqVeRdqLDwVGBJ4rlEgi523gd0qp7cBSYChSH7M1ZLv7gLOAaUqpZ5Huo0Lg\nVOByrfWSoG1fAv6JpHae0FrXRXhug8HgAkacGAyGcJwLVCLdKs3QWmul1DvAuUqpjsAFwGLgYuQk\nXg58BXwe9Jg7lFI/An8E/h7Y/9dITYfF04gvycVIpOJTpGbkI5p7nUTyPrkGqA38DbmIl8kJwMzg\nx2it/UqpoxGPlNHA+Ujh7IeIP0rw37tZKfU+cBLG28Rg8BwzW8dgMBhsEIj2HKS17pvqtRgMuztp\nU3OilLoqYCFdpZSaq5Q6LMq2Ryml5gRsqSuVUsuUUteFbGPZW9cFvtcrpSq9/0sMBsPuhlKqK3Ay\nTaM8BoPBI9IiraOUOht4ALgM8VAYB8xUSvXVWofmiQH8wKNISNgPHA08pZTaobX+d9B25UBfGnPT\nJkxkMBhso5TaGzm+XILY7T+VyvUYDHsKaZHWUUrNBb7UWl8b+F0hQ7we0Vr/0+Y+pgM7tNa/D/z+\ne2CC1rrAo2UbDIbdnMBx5FnE7O0GrfVrqV2RwbBnkPK0TsDDYDBS8AY0uDJ+iFTZ29nHwMC2s0Pu\naheYZrpGKfW6UuoAd1ZtMBj2BLTWz2mtM7TW+xhhYjAkj5SLE2QKaCZi9hTMJsQiOyKB8eY7kVTQ\n41rrZ4Pu/g64CPFhOA/5Wz9XSnVza+EGg8FgMBjcJy1qThLgaKAdMh79H0qpH7TWrwBoreciXgwA\nKKW+QOZ6XE4EsyilVCHSvrga2Onpyg0Gg8Fg2L3IRawAZmqtSxPZUTqIk62Io2NxyO3FyDTUiGit\nfwr8+K1SqgvwN2TuR7htawMW1PtG2eVIGsepGwwGg8FgiJ/zEONCx6RcnGita5RSCxAHxzehoSD2\neOCROHaVCeREujMwYfVg4J0o+1gNMHnyZPr37x/HUxsSYdy4cUyYMCHVy9ijMK958jGvefIxr3ly\nWbZsGWPHjoUw08LjJeXiJMCDwKSASLFaidsAkwCUUuOBbkGdOH8A1gDLA4//JXAD8JC1Q6XUX5G0\nzg/IwK+bkIFdwa3GoewE6N+/P4MGDXLpTzPEIj8/37zeSca85snHvObJx7zmKSPhsoi0ECda66lK\nqU7AnUg6ZxEwUmttzdPoAvQIekgGMB7JbdUiw8L+pLUO9iDoiHgSdEEmnS4Ahmqtl2MwGAwGgyFt\nSQtxAqC1fgJ4IsJ9F4b8/hgyFTXa/q4HrndtgQaDwWAwGJJCOrQSGwwGg8FgMDRgxIkh5YwZMybV\nS9jjMK958jGvefIxr3nLJS3s69MFpdQgYMGCBQtMEZXBYDAYDHGwcOFCBg8eDDBYa70wkX2ZyInB\nYDAYDIa0wogTg8FgMBgMaYURJwaDwWAwGNIKI04MBoPBYDCkFUacGAwGg8FgSCuMODEYDAaDwZBW\nGHFiMBgMBoMhrTDixGAwGAwGQ1phxInBYDAYDIa0wogTg8FgMBgMaYURJwaDwWAwGNIKI04MBoPB\nYDCkFUacGAwGg8FgSCuMODEYDAaDY6qqYPHiVK/CsLthxInBYDAYHPPSSzBwIMydm+qVGHYnjDgx\nGAwGg2M2bgSt4eKLYdeuVK/GsLtgxInBYDAYHOPzQYcO8P33cM89qV6NYXfBiBODwWAwOMbng/79\n4c9/hvHjYcmSVK/IsDtgxInBYDAYHOPzQUEB3Hor7LefpHfq6lK9KkNLx4gTg8FgMDimtFTESU4O\n/Pvf8NVX8PDDqV6VoaVjxInBYDAYHGNFTgCGDoVrroHbboMff0ztugwtGyNODAaDweCYYHEC8Pe/\nQ+fOcOml0sVjMDjBiBODwWAwOEJrESeFhY23tWsHTz0FH38MzzyTurUZWjZGnBgMBoPBEX4/1NQ0\njZwAjBgBF1wAN9wA69enZGmGFo4RJwaDwWBwhM8n30PFCcADD0BuLlx1lUnvGOLHiBODwWAwOCKa\nOCkogMceg9dfh+nTk7suQ8vHiBODwWAwOKK0VL6HEycAZ54Jp58OV1/dKGQMBjsYcWIwGAwGR0SL\nnAAoBY8/Djt3Sv2JwWAXI04MBoPB4AifDzIyID8/8jbdukn9yaRJMG9e0pZmaOEYcWIweER9PWzZ\nkupVOOP992HdulSvwpDu+HzQsaMIlGhccIG0GM+enYxVGSystFtLJG3EiVLqKqXUKqVUlVJqrlLq\nsCjbHqWUmqOU2qqUqlRKLVNKXRdmu98E7qtSSi1WSp3k7V9hMDTy7LPQpw9UVaV6JfHx7bdw0kly\ntWswRCPUgC0SmZkwaJBY2xuSw48/Qteu8O67qV6JM9JCnCilzgYeAO4ABgKLgZlKqU4RHuIHHgWG\nAf2Au4C/K6UuCdrnkcBLwNPAAOAN4HWl1AFe/R0GQzDvvQcVFbBwYapXEh833CBRn5KSVK/EkO7Y\nFScAQ4YkT5xs3w5jxshncE/lxRfFg2batFSvxBlpIU6AccBErfXzWuvlwBVAJXBRuI211ou01q9o\nrZdprddorV8CZiJixeIa4D2t9YNa6++01rcDC4Grvf1TDAY5uVsh7C++SOlS4uK992DmTDjySBEn\nxp/CEI14xcmqVclJNSxeDC+/DL/+tXQLrV7t/XOmE1qLOGnVCt5+u2VOiU65OFFKZQGDgY+s27TW\nGvgQGGpzHwMD284OunloYB/BzLS7T4MhEb75Rg7CeXkwd26qV2OPmhqJmhx7rAxuKy+Xk4nBEAlr\nIrEdhgyR7wsWeLceC6uL6IknJFrTvz/cdZd0De0JlJTAd9/BrbfC1q3w5ZepXlH8pFycAJ2ATGBT\nyO2bgC7RHqiUWquU2gnMAx7XWj8bdHcXJ/s0GNxg9mzIzoaLLpLISUuIQDz1FCxfDhMmwMCBcptJ\n7RiiEU/kpE8f6epJRmrHEieXXirv6WuvhTvvhIMOgnfe8f75U82LL8rwxVtvhaIiePPNVK8oftJB\nnCTC0UjU5QpgXKB2xWBIObNmyfj44cNltsjataleUXTKyuD22+HCC2HAAOjSRb6MODFEI3ToXzQy\nMpJXFOvzSdSyVSvpErr3XliyBHr3hlNOgdNO232jgnV1MGUKnH025OTAySe3THHSKtULALYCdUBx\nyO3FwMZoD9Ra/xT48VulVBfgb8Argds2OtknwLhx48gPadwfM2YMY8aMifVQg4H6evjkE7laO+II\nuW3uXOjZM7XrisZdd8GuXTLu3mLgQCNODJGxJhLbjZyApHZeftm7NVmUlTVfV79+0iL/6qtw/fVw\nwAHw5z/DTTfJDKDdhdmzYcMGOO88+X3UKPGY+eEH2Hdf955nypQpTJkypclt5eXl7j2B1jrlX8Bc\n4OGg3xWwFvhTHPu4Hfgx6PeXgTdCtvkf8ESUfQwC9IIFC7TB4JSFC7UGrWfPlt9799b6uutSu6Zo\nrFihdVaW1nff3fT2W2/VumvX1KzJkP74/fI+nzzZ/mNeeUUes3Gjd+vSWusrr9R64MDI91dUaH3z\nzfK+P+wwrdev93Y98fDII1pPmOD88RddpHWfPlrX18vvFRVa5+Ro/eCD7qwvGgsWLNCABgbpBHVB\nuqR1HgQuVUqdr5TqBzwJtAEmASilxiulnrM2Vkr9QSl1ilJq38DXxcANwAtB+3wY+JVS6nql1P5K\nqb8hKaDHkvMnGfZUZs+WKzErajJ0aHp37PzpT+KHMG5c09sHDpQrsE2hlVsGA7Gt68ORrKLYWBEd\nK9XzxRdiNnjYYekRJdy1C+64A265RT578bJzp0SGzjtPRgeA/K3HH9/yUjtpIU601lOBG4E7gRLg\nEGCk1try1+wC9Ah6SAYwPrDtfOBKJMpyR9A+vwDOBS4DFgFnAKdprZd6+9cY9nRmzZJW3Jwc+X3o\nUDnw7dqV2nWFY9YseOMN+Mc/oHXrpvcNGiTf0+GgbUg/Yg39C0fv3uIo63Xdid100+DBMH++iPOj\nj4YZM7xdVyzeeUdSUpmZ8M9/Onv89u1w7rlNbz/1VPjsM9l3SyEtxAmA1voJrfXeWuvWWuuhWuuv\ngu67UGt9XNDvj2mtD9Za52mtO2qth2itnwqzz+la636BfR6itZ6ZrL/HsGdSVweffirtuBZHHAHV\n1elnxlZXJ9GSoUOleC6U3r2lu8KIE0M4nEROlJLoSaojJ8F06yY1YiefLFOUx49PXXfdCy+IYLrp\nJnjySdgYs0KyKS++KI/ff/+mt59yinzeW5IpXdqIE4Nhd2DRIvEHGT688bZDD5WoRLqldp59Vsyq\nJkxoDAEHo5R07qSbqDKkB5Y4sdutY5EMp9h4C3XbtJFC3dtvl/bb889PvidKaalEPn73O7jmGrEi\nuP9++48vK5PHW4Wwwey1l4iWlpTaMeLEYHCRWbNEiPziF423ZWXJATmdxElFhRitnXsuHH545O1M\nx44hEj6fCNhoE4nDMWSItNevX+/NuiB+cQLS6vx//wcvvSSW78cfD5s3e7O+cEydKp1+Y8ZI6uva\na+Ff/7K/hunTobYWzjkn/P2jRknkpLravTV7iREnBoOLzJoFRx0lVz3BHHFEejnFjh8vEZ7x46Nv\nN3AgrFwp2xoMwfh80KGD1EfEg9dFsbW18n6NV5xYjBkjaZ6VK+UiY8kSd9cXiRdegJEjxTwN4Lrr\n5LW1O4DzpZfguOOkfiYcp54q9SiffurOer3GiBODwSVqa6XoLDilYzF0KPz8s3ylmtWr4cEH4cYb\nY3uvWE6xixd7vixDC8NJdAKgRw9xLfUqtbNtm3x3Kk5Aoonz5on4OvJIEQ5epnl++EEiq7/7XeNt\nBQXwxz/C44+LBX001q2TLsFwKR2LAQMkvfPWW64s2XOMODEYXGLhQkmXhBMnVltxOqR2brlFDnw3\n3xx72379pOvIpHa8p7xcBG5Lwak4sYpivRInTgp1w9GzJ8yZAyNGSA1KcbE4KM+c6f7/6YUXoH17\nca4Nxmrvf/DB6I+fMkWitaNHR95GKUntvPlmyxinYcSJweASs2ZB27aNYetgunaFXr1Sn9r54Qd4\n5RVxgm3XLvb2WVlw8MFGnHiN1hKleuihVK/EPvEM/QvFEidenCQtcdKxY+L7atdOajmWLpU0y//+\nB7/6lXT4/OEPEimtr0/sObSGyZPhrLOat/N36gRXXQWPPtr4d4XjxRclbROr/mfUKImcfvNNYmtO\nBkacGAwuMWuWeCVkZYW/Px3M2KwZP8ccY/8x6VIUe/31cMklqV6FN6xdK7NekjF3xi3imasTypAh\nUujpRZrTrchJMP37S7Hsd99Jrczvfw9vvy2fo169JEXqNPX5+efw449NUzrB3HCDCKAJE8Lfv3Sp\ndAlGS+lYHHusCK6W0LVjxInB4AI1NRICDpfSsRg6VA5sqTRjs/Lx8VxVDhokB8BUm8h9/bW0P//0\nU+xtWxrz58v3ZctSu454cJrWgcboohdizDIac1OcWCgln4f77pMIxGefSSrm+efl9s8/j3+fL7wg\nKaRIFwydO8OVV8Ijj4Q3UXvpJamNOemk2M+VkyNFty2h7sSIE4PBBb76Cvz+6OLEMmNbtCh56wrF\nOrjF0/45cKDk2FMdCvb75QryiSdSuw4vsMTJd9+JWVZLIBFx0q2bpDq9ECc+n4yPCE2RuE1GhkRK\nH3tMClIHD4YrrpALFbvs2iUtxOedJ/uLxJ/+JMeOhx9uervWIk5+85tGR+pYjBoFX34Zv8FbsjHi\nxGBwgVmzZES7ZfkejgED5KCZytROWVnjKHm7HHywHDhTndrx+2UdTz8NlZWpXYvbzJsnJ/pdu+SK\nvCWQiDgB74piE12XE7KyxNH122/jqxuy7OojpXQsiotF+Dz0UNO2/i++kHSgnZSOxa9/LZ+jt9+2\n/5hUYMSJweACs2fDsGHRT/rZ2XJ1lUpxsm2bhIDjoU0b6dpJtVNsZaUUDZaXSwHh7kJ9vaT7LPOs\nlpDaqaqSLzfEidtFsakQJyAXJtdcA3/7m/3Uo2VX379/7G1vuknamR95pPG2F1+U9uBhw+yvs1Mn\naY9O99SOEScGQ4JUV0sVf7SUjkWqzdjKypx1MaRDUazfDwceKGHpRx5pGe2QdlixQsyxTj9dihVb\ngjhxo65j8GAREm5HilIlTgDuvFM+X1dfHfv9GWxXb4euXeGyy6Qwdvt2SR9NnSqmcdFSQuEYNQo+\n+CC9I5BGnBgMCTJvnnzI7YiToUNhzRpvrbujsW2bc3Hy9deprYfw+6VV+5prJHz+8cepW4ubzJsn\n3w87TCJUy5endj12sCYSO+3WAREn4H5qJ5XiJC9PhPPbb8Prr0ffNtiu3i433SSfg8cfh/ffF3O2\neFI6FqNGSeTro4/if2yyMOLEYEiQ2bOlwHTAgJA7pk+X8aBBZ/ShQ+V7qlI7ZWXxp3VAxEllpVzl\npwKtG8XJscdKHUxweLslM38+9O0r/5f+/VtG5MSNdt0uXSQlsTuJExAjtFNOEXfXiorI24Xa1dth\nr72knf6BB+CppySSeMgh8a9x//1hv/3Su6XYiBODIUFmzZI2wGYzRu67T87mQWYO3bqJfXeqUjtO\n0zqW8EpVamfXLrnKbNNG2jmvuUZy5j/+mJr1uMn8+RI1gUZxku4pK7e8RLwoik21OFFKOnjKymTK\ncTjC2dXb5eabJa3z5psSNQk3UdwOo0ZJhMeuiZzW8jksL5dOn9WrJcq3aJEcz2bPdtZKHQkjTgyG\nBNi1Sz6QzVI6JSXSrwfNwg2pNGNzUhALcrDv1St14sTvl+9t28r3884TkfXYY6lZj1tUV8trak2x\n7t9f/kebNqV2XbGwxImT91IwQ4ZIMXCiLqvBpFqcgHxW/vY3ie6FKySfPFlSQKF29Xbo2RMuukh+\njiclFMqoUSIyoolDrWUI4siRcvGVmyv/865doXdveb8OHCjHtOHDJVrkFkacGAwJ8OWXUkF/7LEh\nd0ycKJ/grCz4/vsmdw0dKgeEVIwudxo5gdQWxVqFe5Y4ad1aigP/8x/YsSM1a3KDJUvkfRAcOYH0\nT+34fJLKjKclPRxDhsiV+MqV7qyrvl7W5oZ1faJcd52kXS6/vGmtltaS0glnV2+Xe++Fd9+Fvfd2\nvr4jjxQRFy61o7UU6x59tBzbNm2SNubnn5damTfflJqXTz+VyN+SJXKYe/dd5+sJxYgTgyEBZs2S\nA+GhhwbdWFEhPX6XXgr77NMscnLEERJxSYUZm9OCWJBWyZKS1KQcQiMnILNN/H547rnkr8ct5s+X\nE7yVNttnH/k93Yti3YpOuF0UW1EhAiXVkROQ65KJE+Vve/LJxtstu/rzz3e+b7uOsNFo1Uo8T4Jb\niuvqZPbWgAFSNwMiUkpKJJX6u9+J4dupp8KJJ0oL85AhcNBBsO++4sfiFkacGAwJMGsW/PKXIa18\nL70kl/qXXCJVZyGRk4EDxfMk2XUnO3fKl9NQ/MCBclKy5vMkk3DipEcPOOMMGYrmZlogmcybJ8W9\n1hV0Vpa8ZdI9chL30L9vv4XjjoN77mniItapk1z9uyVOvJirkwhDh0qE79ZbGzv0YtnVJ5NRo6QL\nb8UKiUL27y9+O126SA3JnDkiYJzWtSSCEScGg0OqqqR2pElKR2v417/g5JPl7Nm3b7PISU5O/GZs\n8+cnPlPGyVydYAYOlO+pMGOzxEmbNk1vv+YasXz/4IPkr8kNgothLVpCx05cQ/8++giOOkrCBXfe\nKWfmv/wFtmwBGutO3FoXpI84AUnB5ObCuHH27eqTxciRIogPPVSupQ4+WN6TM2fKRVeDKFmzRkKU\nVg95EkiDl8dgaJnMnSv1Ak2KYefNk/GkV1whv++3n/hLhwzcOOII++Jk7lwJn959d2LrtYyznEZO\nunWDoqLU1J2Ei5yAnPMGDmw+c6QlsGOHDFRsqeLElgCYNAl+9St5w3/9tXwWLrtM/mG9esF11zF8\n37WuFcV6OfTPKR07woMPiii55hp7dvXJon17qY05+2wJbk2f3jiUsQk33wwXXCAhlVGjJPfjsYOb\nEScGg0NmzZKrx4MOCrpx4kQ56I4cKb/37SuJ3FWrmjx26FCJhGzYEP051q4V59BduxouNB1jHbid\nRk6USl1RbGhBbPCarr0W3nsvdR4sTlm4UE7IVqeORf/+Mkhu+/bUrMsOMcWJ1nDHHXDhhXJSe+st\nORN27Sot9j/9JI5izz/PFff3YcKOS1j1wQ+urAvSS5wAnHsunHCCeJPYtatPFv/8p2jIAw6IsEFZ\nGbz2Gtxyi9jTbt4suZ/iYvnffvCBJ+6MRpwYDA6ZNUtSOg3h2bIyePlluTK0TE/220++h+nYgeh1\nJ36/tBrm5MCIEYlHVBNN60DqxEmkyAnIVV9RkdSetCTmz5dak9CTQr9+8j2di2KjipPqavj97yWF\nM368nJGzsppuU1jYMIRm5213cwpv0/uk/aU3NoGiJp9PPnp5eY534QlKyTTtNm2kTr5F8fLLMpb8\n2mvFl3/uXDme3XijVPeOGCHucOPGuRryM+LEYHBAZaW0ETepN3nhBUnfWCYEAN27yxko5LJ+r73k\nrkipnfp6uSj57jtp2+vXr/Gq0CmJpnVAxMnPP4ttdjLx++Wkk53d/L7cXMmiTZrUdGJr2rB8uUQM\nQvpl58+XDqjQdtwWLU7KyiRq+MorclK75Zbo1ZR5ebS5408cv89qph/7GHz4odSjJLiuVBRwxmK/\n/SRSevnlqV5JnDzzjLQGdenSeNu++0pk7LvvJJX9299KI8DYsa49rREnBoMDPv9cdEhDvYnWktI5\n/fSmH+KMDDkqhck5DB0aOXJy113w6qti1nTooXLAdSNykp3t3FsBGotikx09sazrI510rrhCOpEm\nTUrqsuzx7LPidvXGG01unjeveUoH5O/s2TN960527ZL/RzNxsnq1FAF9/bUUwZ59tu19HvKLXB6u\nvlL6VBOouE4HA7ZotG+f6hXEyTffSCvVhReGv18pKZp6+GHJRT7+uGtPbcSJwRAnlZUwbZqkEhpC\n8nPmSHWjVQgbTJh2Ymg0YwuplWXaNIl4//3vMqcDJAru8yXmMWLN1UnkqnLffWVybirESWinTjDd\nusl57dFHUzucsBn19eJ5A+JaFWDrVilDCi2GtUjnolgrgtekW2fePDj8cEnpfPGFuHfFwZAh8p6q\nO3iAhIyqqhyvLZ3FSYvj2Wel39syPYlGq1ZS+OwSRpwYDDHQWqKXDz0kEeuCAkmjN5lr8eSTcuYO\nN5o4TDsxyOe4qkqaeywWLpR0/TnniDeCRXHONjKqqxIqkE/EHdYiI0MMmpItTiorw9ebBHPttZI5\nee+95KzJFp98IleUZ50lPwdOupavR7jICbQMcdIgAhYskPxmnz4iTPr2jXufQ4bI/3h1hwGiLr/9\n1vHajDj+9TavAAAgAElEQVQJsHMnnHmm8/xgTY2EbseODZ9P9RgjTgyGMPj9MhTrqqvkmNuvn3TT\naS01fsuWSXsgIJfBr74qyeRw5gX77SdFfiFXg4MGSZ2gldrZuFEKYA88UNK8wRGOEyb8mrv4a0Kp\nHadzdUJJRVGsldaJxuGHy8k+mW3FP/8M//1vlA0mTxbb1zvukJPFZ58BEmgoKJC7wtGvnwitVIw4\niEUzcfLaa1KB+tFHEk50gJUu/Hz7QfIZcvgGM+IkiIULYcYM6YpywrvvSmdOpJSOxxhxYvAErZun\nK1oC9fXiQVBQIBbN//1vo8WzzyeR+XHj5OTRIB6sQocLLgi/U+tK8oemrZK5uSJQvvhCzlujR8vz\nv/FGSF1IdTX533/FvvyQUFGsG5ETkBPJihXJnWljR5yA/As+/DB5771//1veJ2FbwquqRLSOHSuK\ns3v3htTO/PkSLYiUYuvfXwIIYbKBKaeZOFm8WN7ICRQztW8P++8PXy5pIx8uh7Md0mWuTlpgCby3\n3nJmR/3MM/J/PeQQd9dlEyNODJ7w5JPynk730e+hfPmlXOzecoukcn74QSbfnnJKhJNjfb0Uwp51\nluRmwxGhnRgaJxRfdpkcj19/XeonmrBsGRm1NRSzKeHIiVviROum6SivsStOevSQ78nqJtq+Xbos\nn3oqzJ1vvy0bWPm/ESNg5ky0FnESKaUDjT4Y6dixY4mThvfS4sUhw6WcMWRIIN01YEBC4sRETgIs\nWiQH4YMPjr8DatMmGaoT3HmYZIw4MXjC6tVyYE3HK79oTJ0qXZ+33y4Bj5jFox9/LAomXCGsRVGR\njHCNUHeyapV0IT/7bIQCycCBugsbE46cuJHWOeAASUclM7UTqyDWwtKHiRrW2cWKHj35ZJgUzOTJ\nokCsyNnIkfDNN6yfv45NmyIXw4K8ZQoL07PuxOeTLE5WFo3DllwSJ4sWBYpiFy+O2zJWayNOmlBS\nIo5vf/+7HKc++sj+YydPlt79MWO8W18MjDgxeIJlmjV7dkqXERf19RKFP/PMRg+1mEycKGfraN0J\nSkXs2Dn6aDFZu+02KYINS0CcFLOJ0q3OQ1FuRU6ys8UVN9nixE7kxCp5SJY48fslW7Nxo6T3G9i6\nVXL2wb4PJ5wASrHheRkEFE2cQPoWxTaZq/P11/LdJXGyaxes6jBAXtgQX5hYVFXJ4404QfKaS5ZI\nmPPUU0Uk33qrvVC21pLSOf30lL6YRpwYPMHqKpk1K7XriIe5c6XA8be/tfmADRskD3PFFbFDLBE6\ndrp3l2mld90V5bGLFkFODm2oYseGCpuLa45bkROQY17hx9PE+zoJ2OnWgeSLkx075LUYPjzEoXbq\nVDnIB3t9FBbCkCG0+ngme+0lEbpo9OuXnuKkyUTixYtFXTvo0AllwACphZ1bNUBuiFP9pqt1fUpY\ntkxCeQMHyrHpnnukCvvNN2M/dv58sUVIUSGshREnBk8Ijpy0lLoTK6Vz1FE2H/DMMxLbtjPFK0Lk\nBGzMKFm0SCb/ATU/b7K5uKbU1Yl7qlvFggMHwvmr70T/5xl3dhgDu5GTvDyJ7CRTnLRrJ67en38e\n5B82ebIMvOvcuekDRo6k9w8fcPiQ2GYs/ftLatSNgXhu0iR1smiR1DSE2tw6oF07EWRf/FAkqj3O\nupN0HPqXMixhZ0W0jj8ejjtOQrSx3lDPPiuv/4knervGGKSNOFFKXaWUWqWUqlJKzVVKRQx6KqVG\nK6XeV0ptVkqVK6U+V0qNCNnm90qpeqVUXeB7vVLK2zGKhgb8fjlRbNwohaXpTn29mJ/95jc2R5nX\n1cHTT0suxk44om9fKTKLd5rbmjWSjwkMEtQbnYkT62ndEidHFiznIL6hblNyVIBdcaKURE+SmdZp\n104GtfboIcXTrFwpVc5hrLzrTxhBfk0pJ3eLHRXo319SFQmMmvGEJuLEpWJYi733luilk6JYEzkJ\noqREfJeChwzdfbc4vr78cuTHVVXBlClitmQ7t+0NaSFOlFJnAw8AdwADgcXATKVUhPYHjgHeB04C\nBgGzgLeUUqGfknKgS9BXL/dXbwhHZaX4MmVmtoy6k88/l/SK7ZTOzJkyWTVaIWwwVtg73gph6wD9\nq18BkLF5Y3yPD+DGXJ1gDlw2DYBW5T5pV/EYuwWxkFxxsmOHiKZWreDKK2W8SOXTLzYqlhBWFBzB\ndvI4pmpmzH1bHTvpltppECc1NWKW5qI46dZNPoctUpykkzVxSUmjeYzFEUdI/cntt0futX/9dQmx\nRrJFSCJpIU6AccBErfXzWuvlwBVAJRC2j0lrPU5rfb/WeoHWeqXW+i/A98CpzTfVW7TWmwNfSTpk\nGfx+mah92GEto+5k6lSJZFrTgmPy/PNyUI5V1WhhtROHqTuJyqJF0oJy4IHUZGSTXeYscuLGROJg\nct58lbLMQFVkokN/bGA3cgLJFyft2snPl1wCaM3O/0yWquowampeSRYfcTy9lscWJz17yi7SVpx8\n953UNbgoTrp3F0NdBgyQmq5N9t/vljhxS4DHxXvviXGRVXz088/xPb6+Xmo9brtNXs+bb3a+lvp6\nOW6EihOQzp2VKyV1E45nnpEqfet4lUJSLk6UUlnAYKChz0lrrYEPAVunCqWUAvKA0EbLdkqp1Uqp\nNUqp15VSB4R5uMEDrCvd4cPTv+6krk66dGyndEAq4YcNsz+oJj9f6g+ciJMBA0ApdrQtJrfcmThx\nNXKyYgV8/TVz9gsUzHmsBOrrJdp81Oy7JTQdg1SJk6Ii+PMJ8ynY+j11Y8JPZ50/HxZ3HkGr+V/E\nTPFlZIgxWTqKk8JCGo1uXDTp6tZN9EjtQYGi2DjMdHw+eX8nPRuhtQzD6tdPBMoNN0iO74gj4L77\n4Mcfwz+uuloisFdeKdv/4hfwr39Jy9Hkyc4PmqtXy3srnDg55BBpD77zTnF+DGbNGmk3TqG3STAp\nFydAJyATCD3qbkJSMXb4E9AWmBp023dI5GUUcB7yt36ulAq1uDJ4gNVdMXy4OCCn2wE2mP/9Ty7S\nfvMbmw+oqxNvk3g7FKIUxUbEEidAVfti8vyJpXVciZxMmwZt2/L94YFCYI+VgOX633PFB3Iwj0Gy\na06CIzpX5k1mPV15a0eYGUuIONk+dKSkwmyEFNOtY6e6GioqApGTxYuhVy9XQxXdu4sY3dR2H6mX\niKNjJ2UeJx9/LJ0w998vEZTNm8W4yDJM6tNHPsN33SUV01OmSK1ap06Srv3vfyWfPGuWKLP77pPc\n1qpVztZjvWbhxAnA//2fFAP+619Nb3/uObmitH0g9JbES6xTjFLqXOCvwCitdYMvpNZ6LjA3aLsv\ngGXA5UhtS0TGjRtHfn5+k9vGjBnDmBQa0rQ0rIP2kUdKQ8vs2UETfNOMadNgr73iGKi5dq0cpeMN\nffbtKwVpdtm2Ta6CAuKkumMx+es3oXX8k4WttI4rI9tffRVOOQXVq6f87rEdq9X51dpfCvWxfemT\nJU5qa+Xi04qcUFND549f5qWu5/PMvzI5/cym21dXy3nj3Pv2gW/6iNA67bSoz9G/f5NhximnSUeM\ny8Ww0OiOvH5jBt0PPTSuupOUWdffc4/YvI8I9GR06CDF0GPHypv3vfdg+nRpu7/9dtlm0CC48Ubx\nEjn44KYf6KOPlt8/+yzy8KVolJSIMCouDn//fvtJm/A990guMi9PFOGkSSJMGt7Q0ZkyZQpTpkxp\nclt5eXn8641AOoiTrUAdEPpKFgNRLxOVUucATwFnaa2jXoZorWuVUiXAvrEWNGHCBAYNGhRrM0MU\nKitFhLdtK9HKWbPgD39I9aqaY6V0xoyJI6VjpWacRE5eew3b6sIKaQfESX1RF4r1Eioq4hcZZWWS\nWUo45P3DD3LCuO022pXmU00WrTZt8TQEa4mTnB2lsL0q+saIOCktlf+tlyF+a10Nx/L334ctW+hw\n91g++otYRQQL8iVLRKAcdhjSfWUjCtS/v/wtW7dGno6QTJoUnS5aJMMuXaRBnFhFsR9+GNfakh45\nmTtXIievvhr+M922rYy2OOssUbJffCGRlJ49I++zY0dxOfz0U+maiZdwxbCh3H671M099BD89a/y\nXD/+GLkWJQzhLtgXLlzI4MGD419zGFKe1tFa1wALgOOt2wI1JMcDn0d6nFJqDPAf4BytdbS5oNb2\nGcDBQLgRXQYX0bppuPvYYyVykm5+DQBz5kiE03aXDog4ycqSkHY89O0rIQy7kYaA+Rr77y+/dyl2\nPF/HLXdYpk0T1XnSSRR2UmylE1VrvQ1TiAjQZFWUyh8SY1RvUVGjlbn36wpK60yeDAcdxAk3HEpx\nMTz+eNPt58+Xrp4BAxBxsnJlTBfUdOvYsV7TorqNkr4ICGe3KCqS16ihKPa77xpfaBtrS7o4GT9e\ncm+jR8fe1iqYjSZMLIYNa5hgHTd2xEmPHnK1eP/98sI9+6y0Hgf8lNKBlIuTAA8ClyqlzldK9QOe\nBNoAkwCUUuOVUs9ZGwdSOc8BNwDzlVLFga/2Qdv8VSl1olKqt1JqIPAi0BP4d9L+qj2UnTvl5GAd\ntIcPl/Px0qXePWdJiXTBxcvUqXKsOPzwOB60YoV8kOO9LI8yADAsIQZXWd1FnPhK4y+Uc80ddto0\nOPlkaNOGwkLYQhHVP3srTioroQ2VZNQEREmMnE2yXGKtuTrt2iEFiK+/DmPHkp2juPxySeEHR7nn\nzZN/Z+vWiGJv1Spm9MR6m6WdOFkfiOq5nNbJyJCMREPkRGvbqdCki5MlS8Rx9ZZb4gi72mTYMDlO\nbIyzxmzTJimgsyMa//xnCS/+9a8S+bnggvjzxR6SFuJEaz0VuBG4EygBDgFGBrX+dgF6BD3kUqSI\n9nFgfdDXQ0HbdERSPkuBd4B2wNBAq7LBQyzrequTcuhQCTR42VL80EMSOf3qK/uPCe7SieszuWKF\nM7vuffdtfLwdgophAXJ6daE1O9n+c5xGbog4SThysnKlqMBAwZwlTrw2YvP7oZCgcFGM9tKUiJPX\nXhNVfu65gGQ7du0SgWIxf35Q53n79lKQFaOgJCdHyg7STZy0X7VY/vDevV1/jgavkwMPFGVmsyi2\niThZvx6OOSZyp4wb3HuvXNkE/ueuYkUw5syJ73GximGD6dwZrrsOnnhCqs7PPz++5/KYtBAnAFrr\nJ7TWe2utW2uth2qtvwq670Kt9XFBvw/XWmeG+booaJvrtda9A/vrprU+VWv9dbL/rj2R0HB3mzZS\nbOqlGdvmzSI2fve7xu6OWHz6qTwurpQOyBWNE3HSpo1U3tqJnFRXi8FVkDhp10fKsipXxd9OvG2b\nC5GTV1+Vy/5f/xoQcbKVTp6rgGbiZPPmqNtb4sTjOt2mNSeTJ0s0pIdcQ3XrJlYnjz0m6cwdOyRy\n+ItfBO1g5EipV4hkiBUgnQYA+nzy97b6drG0pbodMSDI6yQ3V4p2bBbFNhEnL78saZE//cn19QEi\n1F9+GW66KTCe2WW6dxdV+umn8T2upESEr13ReOONcmA48cSG9266kDbixLD70CwXT6PfiVd1J1u2\nyIXSqlUyfNMOU6dK2YhdHzVALodXr3ZuUhRhAGAzli2Tk1aQOGndWzrrq9fE307sSuRk2jQRJoF/\nrBU5yfSllzixvC6SFTnJq1gv/hAhdvVXXy069IMPpIO0vj7kvTZihPTlfvFF1OdJJ3HSMPTPg04d\ni4bICdh2iq2pkf9HgziZPl0iAzNmeHNV9M9/SoWyl54gTupOSkoaJyjaoUMHEchPPx3/+jzGiBOD\n64SmdUAuKn2++Dpp42HLFvksjx8vKZ5YKaTaWjl+xZ3S+fFHOcs4ncLat6+9yIl1QA4yuFJdJHJS\nt95Z5CQhcbJqFSxY0MQDITsbyrOLyC5PojjJyoqZ1snIkPNGssRJh/emyItxZtPe4aOOkvPEo49K\nSqd165B2+kGDZKExUjv9+4s/ls26UE/x+aBLh50ykdAjcdIQOQF5Ab/+OqY1fJMW5w0bZB7FP/8p\nIdtx49y1ll+3Ttpur78+UEDkEcOGiQiMpz03kjNsNAYOtFekm2SMODG4TrjIydChkj/3ou5Ea7mY\nLiqCa6+FX/5SaruifaY//VROXnGndJy2EVtYRmyx3B8XLWo+uCs/n2qVjdocvzhJuCD21VclzH7y\nyU1u3tmuiNwdWz21APb7oXOmTwpIe/WKGTmB5HidWO/z3Fcny8ySkBdYKYmevPuuROkGDw4Z3puR\nIeH0GEWxVsdOOgzQ9PlgYPa3crL3MHJSVhZIzw4YID/EEPRNWpxfe01e6FGjYMIE+SxNmuTeAh98\nUETJlVe6t89wHHOMfK4+j9i02pSKCnmd4hUnaYoRJwbXCSdOcnO9qzvx+6UWsXNnOd4/95wc3K67\nLvJjpk6VCahDhsT5ZCtWSNK9i13z4hD69pXQUkPcOgIhxbAAKIUvuwuttsaX1tHahcjJtGlw0knN\nDJqq84vI1HWNLm8eUFkJXbIC+YTi4rQRJzt2wMCsb1CLF4WdQAxSK9mxo3TqhE0fjhghEakoBTJW\nJ3k6pHZ8PjhELxbldfDBnjyH5XWyYQONn4EYqZ0m4mTGDDjuOHnhjzhC/gl/+Uv8E8HDUVoKEyfC\nH//okqNhFPbdV97vdutOLF8kI04MhvCES+uA1J188on7dSfWScgqhOzVCx55RC6WwrUXWymd3/7W\nQeecVQzrtOXOzgBArcOLE2B76/jn61RVSX2t48jJTz9JXiKMrXVdgfetMX4/FLcqlSKXzp1tDYNL\nljgZnfWWnKROOinsNq1bw8UXy88RxYnWUc3G8vPlhJ0u4mT/XYvlfWx3EmOcdO8u39etQ9RGz54x\nO3YscVJIqVwBnXFG45333ivCZPz4xBf36KPy/7r22sT3FQul4qs7KSmR9KIVamvhGHFicJ1wkROQ\nupOyMkkhu4l1IW2JExBjxdNOg8sua36hPXu2XKjGndIBERWJTOzs3VuqNaOJkzVrJBIRRpxU5hXT\nZkd84iThuTqvvio5uVNOaXaX6pwccdIpI0ic2IicJKPmxO+HvTI3yskzOzvidtdcI8L8+OPD3Nmt\nm0QgbKR20kWc9N7uXTEshLjEgq2i2IbIyZw35ern9NMb7+zRQ7p2HnzQ+bwakLTJI4/IQSVZdr3H\nHCMXBqFD+sJRUiLOsl50D6UAI04MruP3S8o39Hh9xBGS3nG77iQ0cgJy0fHUU/LzZZc1LYmYOlW6\n9BxNKHDqcWKRnS35pGg5dOtAHEac7OzQhfzK+NI6VsbFceRk2jQZUBZc/xIgszhwkPZanBAQJ2mW\n1umUURrT+WuvvaQhonPnCBuMGCFFsVHqdtJGnJRqum3xVpy0by8XNk2KYktKor4+Pp9EarPemiGz\naULnytx0kwiKm25yvrCJE+WffsMNzvcRL8OGSdhz3rzY29pxhm1BGHFicB1rrk4oOTniO+WVOAm9\nmOncWY4nb7zRaIZVUyMp6bi7dECunDZsSEycQOx24kWL5I+xLiGDqCsspmNNEiMna9bAl19GnFSa\n272QepTn4qSDDphYWJGTGAW4RUUSHfOwTpcdO6AQF2xJR46UMMG330bcpH9/GWsUwxLFU2prof32\nteRWbfNUnCgVpp14y5aobqk+H/TssF1EXkjXFCBq5957JQoYr3cISOTigQckJLvXXvE/3ikHHyxq\nLdaaLV8kI04MhsiEjpEP5thj5XPW0NlXW5vw823ZIlGBcJH10aPleHLNNVI6MWuW1LQ5SulY0Y5E\nxYnVsRMJq94kjHrSxV3oXL+R+jr7Z10rcuJInEyfLi/sqaeGvbugKBMfBZ6Kk8pK6FAXlNapqYlZ\ngFtUJJu5OCS1GTt2BERTYWFiOzr6aAkpRknt9Osnf4+XhqexKCuDQ/HGtj6UZuIEoqZ2fD4Y1epd\nOUlHmnNz3nlS+HPddfG3Fk+aJKI4kciLEzIzpSc9Vt3Jt9/KG8SIE4MhMtHEyfDhcsJYMnurjOtu\n27axytwhVhtxJB5+WMTLBReIqWOfPg4/w1a0I5GaExBxs3JlZGEWoRgWILNbMTlUs32t/bOuFTlx\nlNaZNk2u7CN0JlhGbDXrvY2c5NcGpXXAtkusl6kdvz8gmhKNnLRuLf3vUcRJOgwA9PlgAIuoyevo\nefSgidfJ3ntLVXCUolifD06qmi7td5E8OzIyxASppEQm8tqlthb+8Q+5okn0s++EY46RduJoF3KL\nFsnFTJAvUkvHiBOD60RK6wAcNqiOq7Mmsv+pfeWqvE2bhD0ItmyJks9HjmuTJkkh7KRJDrt0QKId\nRUWJW6327StXOWvWNL9v2zZxoI2gnnJ6ysl5+/f2UztlZXJhnpsb5zrXrhX30ggpHWgUJ9XrvfOK\nr9pRR9vqssbICaSFONmxA/JqXJo2N3KkhBQjzF7o0kXex6kWJ4eymF39DvV8QFyTyIlSMYtid2yp\n4gjfu+FTOsEceSScc47YSFdUxF5IdbWYua1eLYPyUsGwYfJmi3YRV1Iix5WQVv+WjBEnBteJGDn5\n6ityjh3KozVXMKfwNHGVuuACmDIlofTOli3RIycgtgfXXis1CI5SOpB4p45FtHZi6wAUIXJiWdj7\nV9ovinU8V2fGDEnpjBoVcZNkDP9T28vJQDcVJ2kw/K+yIkg0JcqIETIaIUJtgVKpL4q1xInyOKUD\njeKkoWYohjjZf/VMcusqm7YQR+Lee+VDce+9kbfZtAnuukuiNn/5i0xyTFVUYsgQKdiLVneymxXD\nghEnBg9oJk58PnFT/MUvYNcunrt0Dmdtf5bags5iXrVpU1Sfh1jESutY3HefTC22M008LIl26lj0\n6CEHm3B1J4sWSYgjwvPk7SuRk11r4oucOAr2TJsmJ838/IibWOLESxWQXRGwri8okD+kVauYkRNL\nL3gpTjIqAqLJjcjJAQdIEfSXX0bcpH9/cY1PFdvX76APK8k+zHtx0r27RGAbaoYGDJCK4AjRjiM3\nzmBDp4PsfT579ZKBdw88IBGRYBYskCK1nj3FF2XUKJm58eSTCf09CZGTA4cfHrnupL7emW19mmPE\nicF1GtI69fXwzDNicfnSS5LvXbCA/S44iu3bAynkQYPkqDt5suPnsxM5AWn/HzzY4ZNo7Z44ycyU\nwpdwkZNFi6RCv4nPeSMde7VnJznUrLMvThy5w65bB//7H5x1VtTNkjH8L9df2vhkStnyOmnVSjSD\nl+Kk1fZgW9IEUUrESZQKXkuceNmBFI2Mb5eQgSbrMKfq3j5hvU60hiVLmm9cXc1x/jf58VAbUROL\nm2+W/9vNN0uK9ZVXpPB0yBBxirz7bvj5ZxElBx6Y8N+TMMccA3PmhP/nr1wpaR/HV13piREnBtfx\n+6EHa6UL4eKLxSNj+XJpmWnVisMOE/EyezZyUB47VuZhWJPU4iRWzYkrbN0qZ3k3xAlEHgAYpRgW\noE1bxSa6RG2rDMXRXJ0ZM0TNnXZa1M3y88GnOpGzfYtnZ83WVUHiBNLGJbaJaHKDvLyoFuv9+kng\noKFQNMm0/n4xNbQKmV7oDU1cYkGeMysrbFFs/UezyNflbDwqRr1JMO3aSWRk6lSJkpxzjkQoXntN\nTvY33uiO6HSLYcPkzRxuwJL1mpjIicEQHb8fRq79t4RDZ8+GF16Arl0b7s/KEt3S4Hdy7rkSbnnt\nNUfPVVVlL3KSEG61EVvst1/zyInlVRBFnCgFpVnFZG7xOHLy6qsylC6GqlEKqtoV0apmp2djc9tW\nBaV1IG2M2HIrXYycgHRERSnSTHXHTsefFrM6t5+cxD3GOlw0RE6ysyWCEabupObl6fxAn/hn/fzu\ndyJKTjlFbKs//licZTMzE1u8FwwdKt1G4epOSkpEzXl+EEwuRpwYXKeyEtrXb5Pc7i9/GXab4cMl\nhVpbixSdHXOMo9ROOOt6T7CERJ8+7uyvb18xXtm1q/G2ZcskxBwjPFue24XsMg8jJ7W1MHeu1JvY\noKaDd9WntbXQvs5HTXabxnYjmxb2XoqTmhrIq3VZnMSInPTuLbogVeKky8ZFrG7vfb0JyL+6oCBk\nPma4oti6Olq9/TrTOZOCwjg7iDIypBj/6ac9G2LoGnl5kgIPV3eyGxbDghEnBg/w+6Fd/fawducW\nxx4rWZwFCwI3jB0rRbEbNsT1XNbJx/O0zooVUsgaqUc6XvbbT2pygl21LK+CGAfKHW2LabPdw4LY\nVaskimMz1+7l8D+/X4a5VecFpU7SIK1jrasuK9e990SMyElmpmjalIiT+nr22raEDZ2TI04gxOsE\nRJwsWdK0s2/OHDJ9W5jBGQl3+Kc9kYYA7obFsGDEicED/H5oU7c96kjxwYMl7duQ2jnrLKlinDIl\nrucKN1fHE9wqhrWw9hWc2lm0SMakRxF1ADvzi2lXGV9aJ67IydKl8t1mbUHD8L+t7nudWCKgNj9I\nnKRBWsfvhwJ81LR3sS4hRuQEJLUTxeXeO1aupHWdH1+P5ImTJl4nIOJk166mdRczZrCzU3fmc1ha\nlYh4wrBhEm0N9kfasEGEuhEnBkNsKiuhTW1FVHGSlSWftdmzAzd07CgW6XGmdqxzlOdDQt0WJ126\niDoLLoqNUQxrUV3QhQ67NtkqQK2tlYvxuK4qly6VStegOqFoZHX1bvhfZWUgQtEhJHJSXt40JRYG\na76OF+zYIeKkzk1xEiNyAlKrNXeut7b8YQn47+zok8LIieWvYqV26uthxgx+GnwGmozdX5wcfbR8\nD46e7KbFsGDEicFltJarytya6JETkNTOnDlBw8zGjpUPWxyXhlu2yHk0ysT6xKmvF48FN8WJUk2L\nYrW2LU5052KydXXM+TLQeBKLS5wsWyaX6DZdQPM751CR0d6ztE4BPnRBiDiBmM9XVCTF0l7U6crQ\nv1LqC1zq1AFbkZPTT5fPyzvvuPe0tli8mE2qmOwexbG3dYlmkZMOHaTwxjohf/UV/PwzS/c/g6ys\nyLz2n0gAACAASURBVCMzdhuKiuRzGSpOOnSQ+r7dDCNODK6yc6ecZ3N2xRYnxxwjJ44G64KTTpKz\naBzRk6S0Ea9bJ2c5t+dqBLcTr1kjYsOGOFFdAicIG+3EjubqLF0aV7toYSFspZOnNScq+LI4DVxi\nrciJcvNyPS8vZuSkRw+ZXTdjhntPawe9aDElekBSoxPduknWor4+6MbgotgZM6CoiG8LhlFQ4Lmj\nfnoQWndSUhJxSGhLx4gTg6tUVsr3nJ2xxUnv3vJ97drADTk5cPbZ8OKLIUekyNg1YEsIK7rhZuQE\nmkZOrAOuDXGS1UMs7OvWx647scSJ7chJfb1ETuIUJ5vqi6jf7J04ySgKqTmBlM7XsSI6qpPLaZ3q\n6pjpqtGj4b33Io7h8YT6RYtZzKFJFSfdu8vw4Cb/ZkucaC2zuU47jdJtmbt/Ssdi2DC5eLDylbtp\npw4YcWJwGSuEnmVDnBQVSQ1sk7zy2LGiVmKNCA9g17o+IVaskIXuvbe7++3bV+LWO3bIAbeoyFad\nR24vOTlXrootTqzMj+3Iydq1ojDjFCdbKKLWg8nEljjJ7BwkTqx/eArFiZXWadXZ5bQOxIyejB4t\n/6IPPnDvqaPi85H58xoWc6hrfnN2aOYSCyJOSkvhv/+VVOuZZ+JzafZii2DYMPk+Z47kbH/80YgT\ng8EOfj8o6mlVGb0gFsRmoGvXEHFy5JESUnnhBVvPl5S0zooVsM8+UsXrJlaa6IcfGutNbIRn8/fK\no5LWVK2yn9axHTmxOnUsxy8beDn8b2f5LtrhJ7tr0FkxJ0cKjWKkdTp5V6fbkNbJKnY5cgIxxUm/\nfvLlwLPQGV9/DZCSyAmEEScAt98u74HjjtuzxEmvXuJo+9lnjdFWI04MhthUVkI7Ajb0MVpiQQ5A\nTQ4+lp39tGlSwBKDpKR1vv/e/ZQONO7z++9tF8MCFBQqNlFMzc/2IicZGbb+FcLSpeLb0bOnzQc0\nihPlhQnbZjE6ayYCbLQT5+TI3+2FOKmsqKMD28hwM61j/ZNiFMWCRE/efDOhYd72WbyYuqwcvmP/\npIqAzp3lvdvk4qVHD1EiX30l3X3Z2XuWOIHGupOSEnGr69cv1SvyBCNODK7i90N7AgfXGJETCNMu\nCHDeeXKAfvvtmI9PWlrHC3FSUCBf8+bJdFS74qQANlFM/QZ7NSf5+XKQt4XVqWP7AUHD/7a537er\nt4p1fZOaE0i5S2zt1m0ykdjNPIfNyAnAGWfIsO9wbuaus3gx27ofSB2tkioCMjOl477ZxYv1OTlD\nBv2Vle2B4mThQkntRBkS2tIx4sTgKn4/5BE4uDoVJ/vvLy0JMVI7SZmrU1MjeV23O3Us+vaVOTYQ\nlzjZSBfUZntpnbg9TuJI6Vjr2UIRWZXbYxZzxk1phOF6KXaJrd/qsnU9xBU5GTxYgghJSe0sXszG\n4kPJzYXWrZPwfEGEPT4MGiTRvZEjAfa8yMkxx0il8Btv7LYpHTDixOAylZUuRE5AUjvvvhvVRSsp\n1vWrVsmBwIvICYjoWb1awrM2n6N1ayjNLCar1F5ax3YxrNZxtxGDpE8qcrxxic0oiyBOUuwSq8o8\nECdxRE6UEs+T11/3bBi0UFsL337Lmg7JrTexaOZ1AnDTTTKkr00btBZxsttb1wfTr58UVNXW2r6g\naYkYcWJwFSdpnfLyMEZZ55wjR91p0yI+NinW9V61EVtY+40zPFvRppjccpcjJxs3yj8jTnECUNvB\nm+rTzPJS6lHNFVaK0zoRRVMiWC5iNiInIHUnP/8s5Ree8d13sGsX37cdkNROHYuw4qSoCA4/HJDj\nRk3NHhY5UarRLdZETgwGe/j90CEjPnECYaInnTtL2DaKIVtSJhKvWCEhZKuv0W2sdFGcV0D+vC60\n9W+OedkcV+TEQaeORX2hN327Wdt9VGR2aD7G3hInMfxwvBInmeUeRE4yMmSkgY3ICUjpQWGhx6md\nQEfINxmHpEQARIysBvB58G9oEQwfLrbYhxyS6pV4hiNxopQa7vZCDLsHlZVQlB0QJza7dSBKaufz\nz2HlyrCPtU46ns7V+f57ERBxFIjGhRU5iVOcVHcsplV9TWOvcATiipwsXSoHvH32iWstEDT8z2Ul\nkF1RSkV2mEv24mIJa8ew8PdKnGTv8LErw4MiDBvzdSxatYJRozwWJ4sXQ8+e/OzvmLK0ztatkUuZ\n9lhxcsUVUhTr1kTsNMTpEfe/SqmVSqnblFI9XF2RoUXj90Nh9nb50NhIU1gBibDi5LTT5EryxRfD\nPtaaq5OTk8CCY7FihXfFsCAplLPPhpNPjuthtZ3EJTaWhf22bXGIk2XLpBjZQfV/285t5WTtshLI\nrSxlR04YcWIVGtkwYquocL9ON2dHKf5w60oUG/N1ghk9GpYvl3+dJ3zyCQwalLKiU+viJdLbfI8V\nJ9nZcOCBqV6FpzgVJ92Bx4CzgB+VUjOVUr9VSjkev6aUukoptUopVaWUmquUOizKtqOVUu8rpTYr\npcqVUp8rpUaE2e43SqllgX0uVkqd5HR9Bnv4/VCQud22sUbbtiIwwoqTNm3gzDMltRMmfdGi24gt\ncnLg5ZfjHtzVMF8nRsdKWVmcaR0HKR2Awk4KX6b7Y4DbVJZSmRtFnKRovk5ulY/K1h6cEeOInACc\neKJ8hjyJnnz9tbS5jx1LaWlqBEDUixf2YHGyB+BInGitt2qtJ2itBwCHAyuAJ4D1SqlHlFJxzdVW\nSp0NPADcAQwEFgMzlVKRAvbHAO8DJwGDgFnAW8HPq5Q6EngJeBoYALwBvK6Uir/az2CbykromBnb\nuj6YZkZswYwdK6mVDz9sdpfnBmx+v1QceilOHJLZLfbwP60dpHUcFMNCo9eJ2yqgzS4fO9uEOfOk\neL5O252l7GyT+shJbq7My/REnEycKEYjo0alLHIS1sI+CJ9P6kPz85O3JkNySDiRrrVeCIxHIint\ngIuABUqpz5RSduNO44CJWuvntdbLgSuAysC+wj3nOK31/VrrBVrrlVrrvwDfA6cGbXYN8J7W+kGt\n9Xda69uBhcDVTv5Ogz38fsjPjG1dH0zUorfhw0UcjBghtRCXXy4dPKWl3lvX//CDfE9DcZLXtR1+\n2kSNHPj90gVtK3KydaucwRMQJxvr3Bcn7atL2dUujAjIz5dxAikSJ+2qfexql/rICUhq56uvggZo\nuoHfLxHLiy6iPjMrZUZnHTuKAIsWOenY0buSMEPqcGwtp5TKAk5DBMSJwFfIiX8KUAT8HZgGRD3a\nBfYzGLjHuk1rrZVSHwJDba5FAXmAL+jmoUg0JpiZgTUbPMLvh3zij5wsXx7hzsxMOfJ+/LFETz78\nEJ56CpTi/taDWL3vCfDhCdJal5vbGC5Yt04ut9ata/qlNbz0kr31ff+9fE9DcWJZ2Pdav4nMCNvE\nNVfHKlpwmtYphM31najbtCbiepzQvraUmvZhxIlSttqJvRIneTU+avM8KLfLyxNvnTg4+WTRaa+/\nDn/8o0vreOUVEUmXXkp5uXxsUtFKrFSEduIAe5wB2x6EI3GilHoUGAMo4AXgJq31N0Gb+JVSNwKR\ngvXBdAIygdBLwE3A/jaX9CegLTA16LYuEfbZxeY+DQ6orIQ8B+Lko4+ibJCXJ8WxpwV05dq18NFH\nLL/6Q05YOQlO/IcIk27dYMOG5rPkO3eWJ+neXcTNhAlwxx2xF7ZihRz5UnFUjkFhoVjYd127kUj9\nIlYjiy1xsnSpCEGHxb+FhbCcIuo3LXBPnGhNh7pSavMjvP42XGLbtpWGGrfFSYe6Uso7ePC+cBA5\nyc+H44+HGTNcFCcTJ0or/9574ws0y6VKBERL+xpxsvviNHJyAPBHYIbWOlId/FbA85ZjpdS5wF+B\nUVprV6rxxo0bR35IEnPMmDGMGTPGjd3v1vj90K5+O7Tf2/ZjuncXTVFfbzM826MHXHABY6+6gL/f\npRl34jciOjZsaBQh3brJ965dpbLd4oYb4IEH4KqrYvcge92pkwCWhX3dusgnZytyYiuts2wZ7Luv\n49anhuF/W11UARUVZFFLfYcIZ58UucRWV8tE4rKOHpwV46w5sRg9Gq68UrJzCbfWL1okhbCBQpZU\nF5126xY9rWPESWqYMmUKU6ZMaXJbeXm5a/t3JE601sfb2KYW+MTG7rYCdUBxyO3FQNQ+SaXUOcBT\nwFla61khd290sk+ACRMmMGjQoFib7Z4kWGXq90PbuvgjJ7W1cp7pYjOu5fdLlKZzsRJ31YMPtvfA\nW26RtNC998L990ff1utOnQQoLIRvKYZN8yNuE1daJ4FOHWs9WygSc7K6uuamaQ7QpT4UoAuiRE4i\neOAEU+RyE5F/ex0d2Yby4qyYlxd35AQkqHjFFfDWW3DhhQmuYeJEEfWnnAKkhzgJeME1o6zMO39E\nQ3TCXbAvXLiQwYMHu7J/pyZsf1ZKNfsIKKUuUkrdHM++tNY1wAKgQfAEakiOBz6PsoYxwH+Ac7TW\n/w2zyRfB+wxwYuB2Qzh8PthrLxnH7ZDKSmhTa7+VGGIYsUXAsXV9UZFETx57TDpxopHG4sSKnGRu\njay1rbSOrchJAp06EBQ50bpxWF+C1GyU/ahOztM64H7kpHKdqL5mk5LdwEFaBySIdOSRLnTt7Ngh\nvkIXX9zgd2P9O1OZ1olVEGvY/XBa43w5sDTM7d8inTbx8iBwqVLqfKVUP+BJoA0wCUApNV4p9Zy1\ncSCV8xxwAzBfKVUc+Aq+XH8Y+JVS6nql1P5Kqb8hhbePOVjfnsGGDRKz/uknx7vw+6F1TfyRE0iS\nOAG4/noxd7vrrsjb+HxyVE5jcbKJYnLKI1u4l5WJVUx2LPeh8nJ58RMQJ/n54Mtwd/jfzvVyVszs\nHEEEpCits3O9hBIyizyKnOzaJZ/DODnjDHj/fdEXjpkyRXZwySUNN/l88h5KlRlpt26ypHCazaR1\ndl+cipMuQLijwhaga7w701pPBW4E7gRKgEOAkVpr65DSBQgujb8UKaJ9HCm6tb4eCtrnF8C5wGXA\nIuAM4DStdThRZYDGPEAMS/Bo+P2QWx2fOCkqkiyAE3HiqJW4fXv485/hP/9pbBcOJY07dUBOFttz\ni8moq22Mu4dg2x3WapVKQJwoBdX57rbG1GwQcdKqOErkZPt22Lkz6n5crznZKK93dhePWonBUfRk\n9GjRNe+9l8DzP/WUGKcEmQL6fBIZUyqB/SZANK8TI052X5yKk7XAUWFuPwp7HTrN0Fo/obXeW2vd\nWms9VGv9VdB9F2qtjwv6fbjWOjPM10Uh+5yute4X2OchWuuZTta2x2Cd5BIQJ7X+XWTW1cQlTjIz\nJcUd0YgtDAnP1fnDH6TA5fbbw99vTSPed1+HT+A9VfmBAp0IqQ3b7rBLl8qZZ3+7zXER6OTuZOKa\nTaVUk0VuYdvwG8RhYe+mOLHSTTndPDJhA0dFsb17w6GHJpDaWbhQ2vYvv7zJzakWAJEiqzt3ShrZ\niJPdE6fi5GngIaXUhUqpXoGvi4AJgfsMLZEEIydaQ6bf/kTiYGJNHw1l82Z5CsdzdVq3FmEyZYoM\nNwtlxQq5ZGvXzuETeE9tYXQLe9vusEuXwt57Jxy3zyrqQJ3KdE0J1G/xUUohbdpGuGSPwyXW55Oi\na1fWtVVEfG5XD4odEoicgERP3nnHUVZICmG7d4df/7rJzakWJ5EiJ9bhyoiT3ROn4uQ+pBj1CeDH\nwNejwCNa6/Eurc2QbKxPe4xJt5HYuTPgcQKeixNXrOsvvFAiI7fd1vy+NC6Gtagvim5hv21bHG3E\nCaR0LAo6ZbA9u5Nr4kRvLaWUQtpGCJzYjZxYAR2X6nSp3+qjkta0K3J5IjEkFDkBqTvZvl08C+Oi\nokLMCS+5pNngx1SLkzZt5H0cKk5S3UVk8Bans3W01vpmxAn2COBQoEBrfaebizMkmQTTOpWV0D6J\n4iRh6/qsLLjzTnj7bfg8pDHs++/TXpy0LW5HVWZbdyInCbQRWxQWBopiXRInyhdDnFjqNMnD/6x1\neVIgmmDk5KCDoE8fB6mdl16SD/DFFze7K9XiBMJ7nRhxsnuT0EQCrfUOrfV8rfU3UczYDC2FBNM6\nfn/yxIlrE4nPPhsOOQRuvbVx8rHWLSJyUlAAWzOLE4ucVFbC6tWuRE4KC2Gzi8P/MspK8VEQWZxk\nZ4v6SrKFfUa5j22qwA0rl+YkGDlRSlI7b7whdjO20FpSOr/+tRgchpCqicTBhLOwN+Jk98axOFFK\nDVFK/VMp9bJSakbwl5sLNCQRN8VJHD4nIOJk2zY5V9rBtYnEGRlw993wySfwwQdy24YN8sekuTgR\nC/suiUVOvvtOTk4uiRM3h/9lbvdFj5xASubrtCr3Ud7KozOiVePkMHICojE2bWoclxSTBQugpKRZ\nIayF1a2TSsJdvFjixPic7J44NWE7BzFI6w+MBrKAA4HjAPf8aw3Jxfq0O6w5STStA/ajJ65OJD75\nZBg6tDF6YnXqpKl1vUVBAayvK44oTmy1Ei8NdNb365fwegoLYX11Edoln5PsilJ8FEb3abEhTqwB\nxm6Jk6yKUiqyPDpbZ2SIQElAnFjm1iUlNh8wcaJETE46qdld9fXpk9YJFznJy5P/rWH3w2nk5FZg\nnNb6VKAauBbohwzeW+PS2gx2efxxGW+eKC5ETvKoQGdmSjdMHDgRJ65ETkBi4ePHyxXkjBkiTjIy\nYJ99XHoCbygshHV1xdRvaJ7WqamR/0fMtM7SpfLih8yScrqezRShN7mjAnJ2lFKRXRjdX8OGS6xS\nUhTrljjJ9fvw53h4tnY4X8ciP1/qTmyJk+3bpWPtkkvCjhyoqBCBkmpxYg3/szKvkB6iyeAdTsVJ\nH+CdwM/VQFuttUZaiS9zY2EGmzz3HFx9NUyalPi+fD65aisvj+g6Gg0rraPbtY/bsclqF7QjTior\n5blcEycAv/wljBghnTvLl4tpRExr1dRiWdjrjc1Pzrbn6rjUqWOtZyudUL6tTc8iTqitJbdqG/7c\nGBGKFLjE5lb5qGzt4VnRoYV9MAMH2hQnL74obXZhCmEhfeo6unUTwR0clDPW9bs3TsVJGWAVFawD\nDgr83AGxnTckg1mz4NJLJUoRwSU0LsrKJFpQX+/IA9tK6+g4Uzogmqh9e3tGbAlZ10fjnntEmPz7\n32lfb/L/7Z15eFxl2f8/d/YmTZpkWlJTKC2lG4strSyViuwIIuIKBWVTEQR5LfCivmwCCvIT6IsL\nL4goYKWKCyIiFFtBhBaRFlBIaStNF6SlzdKkzdJsz++P55xkMpnlzMyZmZP0/lxXrjRnzpw8OZ05\n8z3fewM356SGvIahLew9z9XxqVLHXc8OxiHd3VbgpoOjrjoSiQAPYR3wV5yUdTbSWZrBJIw0nRMY\nECdxNaKbCHv66QPWZQRBEicw+PrQ3Jz7dSmZI1Vx8jx2iB7Ar4G7ReR+YAmw3I+FKQlYs8Y2NTj2\nWLjiivSbOBgzIE4gpbwT1zmRMcmLE/BesZNW6/p4zJ0Ln/qUvWsdBuLEna8jvb1D/v89OSddXbZ9\nv0/OiStOgPSVgPOp2FmWQAS44iSB0+ebOOnpoay7ha7RwXdOWlqgvj7OTi+/bBsQxkiEhdwP/XOJ\nFvbVsM7IJlVxcjnwS+ff38EO7qsBfgtE9wcV/9i+3SZxTpgAv/61tbbTFSe7d9sWmpMn259TyDtp\na4Mxknlx4t4o++6cgB0GmJfn2wd2JgmFbFgHGJJ34f73xRUn69fbetMgihPn9dxV7iGs09ubUEz7\nJk6cE9tTEdycE/CYFPvjH8PEiTacGQPXOcl1tU5NjY0UhzsnKk5GNkmLExEpAE4HegGMMX3GmO8a\nY84wxlxljEmt1EPxRkcHfPzjNoby5JM2+y0UsspgTxqtZtyLu+ucpCBO2tuhOr8VSbKM2CVZ5yQj\n4mTmTPjnP+G88zJwcH+pqrLOCTCk14n73xk3rONW6vgU1ikuhvbSLIuTbM/XcdbVMyaDn9Y+OCc1\nNXZeVUxx0tkJv/xlzERYl6YmWw0Tt5w7CxQW2r9JxcneQ9LixBjTA9wLlPi/HCUufX32Q/P11+GJ\nJwYmh7rv0HTcE/fTLE3npDIvuYnE4SQjTtKaq5OIgw+GkuC/vAsKoLMi+nyd5mb7mRN3NFBdnf3U\nTnl6YhTcW2yfxElfpYecE/DUJbaxMaU878FkIwnDB+cEbGhn9eoYD774or2b+PjH4x7DFQC5mkgc\nTmSXWBUnI5tUwzovA7P9XIjigf/5H/jtb22r6cMPH9jufiCkkxTrPtcVJ6nmnEh64uTddxN/gPha\nRjzMKQmVsadodNSwTmVlgg8VHyt1XCrHFrC7uHpwWUUqNDXRnj+a4vIEFVNJDP/zEP3xtC4gs5+K\nPjgnkKBiZ/lyK+wOOSTGDpYgCYDwXic9PTanJihrU/wnVXFyD3CXiFwuIvNE5P3hX34uUHG4/364\n/Xa4804488zBj7nixA/npKbGergphnXK2ZWWOOnpSXzT7Vvr+hFAKAQtJUNb2HvqDltX57s4CYWg\ntdCHpiKNjezMT9AdFuxrragoe11infdY3tgMOyc+iJM5c+zLIup0g+XL4fjjbX5VHIIkTtybFxi4\nPAVlbYr/pCpOfglMBr4PvAi8Brwa9l3xk2eegUsvha98Bb72taGP+ylOKivtV4phnfK+9JwTSBza\nUedkgOpqaCwc2sI+4Vydnh7but6nfBOXUAga831I8GhspDnPgzgR8dSIzX29pN28tqmJNkoZVZXB\nsJ+PYR2I4p7s3AmvvAInnJDwGP/5T3Dea+FhnaCUOCuZI1VxMjnK1wFh3xW/eOMN+PSnbUb93XdH\n9+ld/z4dcdLUZEVFfr49XophnbI0xInXRmy+tq4f5oRCsEOGtrBP6JzU19tS4gw4JzuMP+KkiWpv\nk389NGLzyznp3Z5gUrIfVFTY5PaurrQOM2mSfSsPyTv5619t7DSBOOnuhpUr4aij0lqGb9TW2v/m\n7m4VJ3sDBak8yRizye+FKFEwxiasTZ4Mv/qVzYCMhiso0nVO3Hd6VVVKzklnWy+lvbtTFic1NfZP\nSdSITcM6A1RXw1ZTA9tWDNq+c2eCC7dbqZMBcbK1dxzsiJWJ6ZHGRhrMPt5EgIdGbNXVNoKRrjjp\nfq+JJqrjJxqni1vttmtXWjW8IjB7dhTnZPlye01x88tisGqVveE47riUl+ArEybYS+K2bSpO9gZS\nEiciErfO0hjzcGrLUQbx3nuwYQM89ljiKb+hUHoJseG32imGdfpana6yKYqT/HwYP17DOslQXQ1b\nuoaGdcL76UWlrs6WoY8f7+t6QiH4zx4fnJOmJrb3zvQuTtxhjTHIy3NcnXSdk4YsiBP3/ZOmOAGb\nd/L730dsXL7cU0jnuedstZfbMyXXhHeJ1YnEI5+UxAlwd8TPhdi29V1AO6DixA82bLDfvQygC4XS\nD+uEi5PNKcxvdOPkKfY5gcTlxB0d9m5OwzqWUAj+3VkDbdttOYrTsyJhWMet1PG5RjQUgld6xmF2\n7CCdI5vGRt7rCfE+L+KkpgZeeCHhbn70OjENNqwzIRvOiU95J3fdFZaD9O67Vphef33C5z77LHzo\nQ7EN22zj5qS54qS4OOn5osowIqWcE2NMVcTXaGA68AKwwNcV7s24vacT2K9A+uIkPKyTYs5Jfptz\nMU3ROYHE4iSjDdiGIdXVsLl7vM0hCPv/T5gQm4FKHQibr+OqyFRpTCK3I4vzdaTJOicZzzkB38qJ\nAV57zdnwl7/Y78cfH/d53d1W7wUlpAP2tVVYaK8PQeq/omSGVBNih2CMWQ98g6GuipIq9fW2QZYX\nJ8JP5yTFnJNsiJOMtq4fhrjD/4D+0E5fn/3vi+mc9PVZ58TnSh13Pf0t7FMtjenoQDo6vCfE7rOP\n/SDv6Ii7mx/iJK8lizknPjgn06fbfoL9eSfLl8Ohhya0Hl95xbYGOPbYtJfgGyIDvU506N/Ixzdx\n4tAD1Pp8zL2X+npvrgk4NaU+OiepiJN2505PnZOs4Q7/A/obWuzebfVHTHGyZYv95MmQc9KA03E2\nVSXgvI6Tck4goXsy1of2KwUt1tHJWs5JmhQUwKxZjjgxxnO+ybPPWo3kOi9Bwb0+BKn/ipIZUk2I\nPSNyE/A+7EDAF9NdlOKwYYN3ceJ3QuyuXbYXRhIB56I9/jgnzc32JjhaPFnFyWCiOScJ5+pkqFLH\nXU/aw/+c17FncRLeJdYd6RCFtJ2T7m4K21szH9ZxlY8P4gSswHjhBewE6i1bPCfDHnNMcPJNXFzn\npKhIxclIJ1Xn5PcRX78DvgX8E7jIl5UpyTknbljHmOR/T2QcwP1Ua2nxfAhjoLjTn4RYiO2ebN9u\nDz8MRt9khepq6KCU7lHl/c6JK05iOid1dVBaCvvt5/t6xoyBprw0xUmGnBNXnKTyFgH63UTP4aZU\nycuzXZp9COuAFSdr1kDXU8ttwvSHPxx3/64uO3onSCEdF7dLrDonI59UE2LzIr7yjTHjjTHnGGO2\n+r3IvZLubnuXk4w46elJ7W6rpcVescP7nEBSoZ3OTiinle7isrhTThORqBGblhEPxu2/114+0IjN\n/W+L6Zy4+SYJWpenggiUhUrsvJ80xYlnh8J9QXjoEtvVlYYh4ayrvSSUiVM3GJ/m64AVJ729sPvx\n5XDkkQlvHv7xj+Dlm7i4XWJVnIx8Mv0WU1JlyxbraHgpI4b0WthH3mq7n2pJiJP2dqiglZ7S1EM6\nMLhcMBraHXYwbv+91tLxQ8I6cZ2TDIR0XEIh2F2SRgylsRGTl8dOKr2Jk8JC+0mV6S6xTripszQL\nn4o+tbAHm/9akNfHqL8/6zmkU1ERvHwTsOKkpcUKFBUnI5uUxImI/FZE/jvK9mtE5NfpL0tJqowY\nBt6pqYiTyI5GKYiTtjYrTvrKUg/pgL0ml5erc5IMoRA0Fw0M/3P/28aMibKzMVacZKBSJ3w9p56c\nHgAAIABJREFUO4vSECdNTXSVVWHI8x4+8VBO7Jc46RqdhU9FH52TkhL4xAGvM6qt0XMy7DHHpGWA\nZgz35qWtTcXJSCdV5+QY4E9Rtj/lPKaky4YN1iOfONHb/q5zkkpSrHurHV6tE77dA65z0jc6PecE\n4lfsaOv6oVRXQ0P+YOdk9GhrKAzh3XftrWeGnZPGvPSck85S+3r2nHiaxPC/lMWJI/x7KoaXcwLw\nyYpldOaNSjgoZ88eWLEimCEdGAj7goqTkU6q4mQ0tmw4km4g/U8nxTon++5r09K94GdYZ8wYK4yS\ndE7K2QXlmRUn6pwMJRSCbaZmkDiJGtLp64PLL7fK5cgjM7qeHX1jU+9z0thIe2mIoqIkqkU8DP8b\nm2aFM01NdBaUUVRenOIBksBH5wTgqPbl/I0P0ZMff+3/+IetlAtS87VwVJzsPaQqTv4FnBVl+9lA\nXerLUfpJplIH7C1mUVHqYZ28vIFEubw8e3FMIazDmMyLE805GUx1NbzbW2NPTm9v7O6w110Hjz8O\nS5b4PlMncj1be9JzTtqKkyzX9RDWKSy05yUdcbK7MMMN2Fz8dE66uthv49/4c98JvPVW/F2ffdbe\nm8ya5c+v9hs37As6V2ekk2oV+y3A70RkCuD0Q+YEbOv6z/ixsL2e+vrk8gJEUm/E5t5qh5cgJNmI\nzQ3r5FV6TOCNw4QJdqp7JB0dtsGYOieDCYVgY6fTwr6hgebmmqEX7p//HG67Db73PTj99Iyv5510\nhv81NrKr6MDkxUmCsA6k2euksZGWggw3YHPx0zl56SXyO9tZzgkc+iocckjsXd3+JkHMN3GprYW1\na9U5GemkWkr8BHAmcCBwD3AnsC9wojEmcgamkgrJOieQegv7aHGAJOfruM5JfpU/zsm779rP2nC0\nAVt0qquhvn2gS+wQ52TlSvjiF+HCC+GqqzK+nlAINneMs7ktXV3JH6CpidaCUHK9RGoc5yjyRRNB\nWuKkqYmd+RluwOZSXu6fOFm+HKqq2HXA7IE29lFw802CGtJxcZNiVZyMbFIuJTbGPGmMOdoYU2aM\nGWuMOd4YE+V+1xsicpmI1ItIh4i8JCKHx9l3vIj8QkTWikiviNwVZZ/zRaTPebzP+WpPdX1ZZfdu\na1GnIk5SSYiN1jQgyfk6rjgp8EmcdHcPTVlwP1Q0rDOYUAjWtw50iR2kNTdtgjPPhCOOgP/7v6xM\nSguFYHs683UaG9mZ77EBm8s++1hhkuD1n644ac70XB0XP8M6y5fDcccxa04+q1fH3u3vf7f9ioKa\nDOtSW2udnTQaUSvDgFRLiQ8XkSEZdSJypIh8IIXjnYV1X24EDgNeB5aKyNgYTykGtmPDS6/F2Aeg\nBRgf9hW7t3WQ2LjRfs+1c5JCWCe/Mr1SYojdiE2dk+hUV8OW7gFx0t/sd/duOOMM2w32d7+zM+az\nQFot7I2xIkBSyDkBz11iU6KxkQYzzMI6u3db1XHiicyZY6cTx+qQ+9xz9m3//ven/2szyYQJOpF4\nbyBV5+RHRB/wN8F5LFkWAvcZYx42xrwFXAK0E6MVvjFmkzFmoTFmMRDv9sIYY3YYY7Y7X2mO/coS\nyfY4ccmhOGnbbaigFfEpIRaGNmLTicTRqa6GTkbRV14B27bR3AyVFX3wuc/ZkvQnnsjqSUtLnLS0\nQG+v99b1LjWD5wvFYty41IuIaGpiR28WnZPOTmshpsPzz9vO0SecwGGH2dPrXl4iee4529k+yPkm\nAJddZlOolJFNquLkIKI7Fq86j3lGRAqBucByd5sxxgDLgHkprs9ltIhsFJHNIvJ7Eclccwc/qa+3\nd7nve19yz0s1ITZaWCfJnJM9LZ0U0uOL11pTY3Nzozkno0frXJ1I3Cryrurx/WGdj664Fv7wB1uZ\nEy8DMkPr6Z9MnKwScMIyO/pSCOtAZp2Tpibe68lSzolfk4mXL7ctCaZO7e/4Gi3vpLMz2P1Nwtlv\nPzjllFyvQsk0qYqTPdgwSSTvI3r/k3iMBfKByFue92L8Dq+sxTovZwDnYv/WFSISzfEJFvX1MGlS\n8rNP/HROksw56d3pXER9ECcFBbbSNZo40XyTobi6sqOiht7/bOMznQ/zgWXfzUplTjRCIdhFOb0F\nRckrAef1+15PkuKkvNwKeg/ipK3NVn4lRXc3tLaydU+WwjpuvWy6eSfLltmusCLU1NiQabS8k7//\n3SbEDgdxouwdpCpOngFuE5H+BtkiUgncCvzZj4WlizHmJWPMYmPMP40xfwM+CewAvpzjpSVmw4bk\nQzpgPxVaWqyNmwxNTWmHdfp2OhdRn7LUovU60e6w0XGdk91lNfDCC9zPl9h8woVw5ZU5WU9xMZSV\nCe1lKdgUjjjZ1p1ktY5IZrvEOi5ig8lSWMcP52T7dvjnPwe1rD/ssOjOybPP2ktA0PNNlL2HVPuc\nXA08D2wSEfelPhvrdnw+yWM1AL1ATcT2GmBbiusbgjGmx1nrgYn2XbhwIWMiBpMsWLCABQsW+LWc\n+NTXw/z5yT/P/ZRqbvb+Kd7dbZPmooV1Ojrs7ZSHRErTknlxot1ho1NRYU22nSXj2e+dTbzIh8j7\n5v8xMYcZg6EQ7NozjvJUxUlXNQclGz7x0CU2XJx4nQwB9IebPE9KThc/nJNnn7XfI8TJT34ydFc3\n3yTj05aVEcOSJUtYsmTJoG0tLS2+HT8lcWKM+Y+IvB8bLpkFdAA/A5YYY5LK4DLGdIvIKmwTtz8A\niIg4P38/lfVFQ0TygEOBJxPtu2jRIubMmePXr04OY6w4+XyyGo/BLey9foq77kg058R9vCZSNw4l\nb7f/4uT55wdv27EDDj7Yl8OPKPLyrLbcVD2HKZMP5lP1v+XZmuxU5sQiFIKdO8ZRm4o4KS6mob00\neRGQyeF/jmhqJIvVOpCec7J8OcyYMajn+2GH2fmQW7cOpLR1dsJLL8Htt6exXmWvI9oN++rVq5k7\nd64vx0+nz0kb8ALwBNZF2QmcKiJnpHC4u4Avich5IjIDuBcoBR4EEJHbROSh8CeIyCwRmY2d8zPO\n+Xlm2OPXi8hJIjJZRA4DfgFMBKLcNwSIxkbrZByQQqfVVCYTR04kdnF/9hjakV3qnOSS6mp4bvKF\nPPeDf9HAuJy39k55+F9TE4RCtLVLauIkQVgn5fk6Yc7JsMk5Wb58yBRi954rPLSzcqXmmyjBIyXn\nREQOAB7DOhEGEOe7S1LFaMaYR52eJjdjwzmvAaeElf6OB/aLeNqrYb9zDnAOsAlwP9WrgB87z20G\nVgHznFLl4JJqGTGkNvwvciKxS7hz4oH8NuciWp5+nxOw4qSpyUaWRo2y2zTnJDZuLvTOFhvKiTpb\nJ8vr2d43DnbEaUkajcZGK07WJjGR2KWmZqjdFsGoUfa4qTonWQvruAooVedk40abuxYhTvbf3953\nvPoqnHaa3fbcc/btf+ihKa9WUXwn1ZyTu4F6bOilHjgSqMY2Urs6lQMaY+7BtsKP9tiFUbbFdX2M\nMVcCuckITIcNG+z3VMSJKzCS6RIbOZHYxf1081hOXNjRSo8UUOBTna/rRL/7LkyZYq1nnasTm+pq\n+9/e3GwH3CWVTJoBQqEUh/81NmKqqunqSuFv8BDWgRTLiZua6Bk1mu6Oouw4J/n5VkWlKk6WL7fx\nvgg7RARmzx7snGi+iRJEUn05zgNuMMY0AH1ArzHmBeCb+JgnsldSX2/HgqbiyxcVWecilbBOtPb1\n4Nk5KexoZU9xhW9tGyMbsWnr+vj0OyfOXJ1cd88MheCdzrH29dXb6/2JjY30jLEOYEphnd27bbvi\nOKQqTrpG2/dIVsQJpNfCfvlyG8OJch0Jr9jp6LD5JhrSUYJGquIkH3AlfQMD3WI3AdPTXdReTSoD\n/8JJttdJc7MVNW7sxKWszN69eRUnnbusOPEJV5y4eSfaHTY+4c5JrvNNwL4MN7WPs/NukmjmR2Mj\n3RUpihM3cTsTjdgaG+ksS3FdqZJqC3tj4C9/gRNPjPrwnDnWoN250+abdHUFf9ifsveRqjh5A1ul\nA/B34BoRORq4Adjgx8L2WurrU0uGdUm2S6z7aRZ5qy2SVK+Tkq5Wukv8EycVFfYO1RUnOlcnPuHO\nSVDEyTtdKZTGNDWxZ3QazglkRpw0NdFRUo3IUB2fMVJ1Tt580yYGR+SbuLidYl97zYZ0QiGtglOC\nR6ri5Nthz70BmAz8DTgNuMKHde29ZNs5ida63iWJFvajulvpKfV3TGh4xY6Kk/iEOye5ToaFNObr\nNDbSMcq+HlMWJx4asaUiTtqKbTJs1nIzUnVO3njDfv9A9Bms06dbgfXqq7YViuabKEEkpZekMWap\nMeZ3zr//bYyZgW1Dv48x5i9+LnCvorfXjrhPV5wkmxAb61bbYwt7Y2BUTys9ZZkVJ6NHZ/GudZgR\nCtnGwJs3B8c5SVqcOC3i20dZ5yTphFhXuWYorLOrKMmW+umSqnOybZt9o0Q0knTJz7edYF980bat\n15COEkR808vGmCZnYJ+SKv/5j71AZzvnJNanmcewzp49UEErpsyfMmKXcHGiZcTxcc2vt98OjnPS\nTBV9efnelYAjqttKUgzrFBQ4NcyJxUlLi8218ExTEy0FWepx4pKqc7Jtmx1OFScres4cePxxe7nR\nZFgliKiZFyTS6XHi4ndYx4M4aWtzxIlPDdhcIp0TFSexCZ9cEBTnxJBHV3koaXHSWphG4mkSXWKT\nGpjc1MROybI4Scc5GR9/Zuphh1mnbexYOGh4zGpX9jJUnAQJV5xMmpT6MVJNiI2Gx5wTV5z41R3W\nZcIEW0psjE4kTkS4vgyCczJmjA0ftJeO9S5OnNdtS0Ea4qSmxv/hf11dsGsXTZKl1vUu6Tgnbm/6\nGLhJscceq/kmSjDRl2WQqK+3F5V0EitCIduxLEGvh358yDlpb7fiJG+Mv+KkttZ+LjQ0aFgnEeHi\nJAjOiYhdx66Scd4tCkec7Myzf0xKb4NMzNdxBHqjyVJ3WJfy8tTEydatCZ2TQw6xhz/llBTXpigZ\nJtUOsUom2LAhvZAODPj7TU3eMgp9Cuvsyy52VvnvnIAN7WhYJz7l5TbloqcnGOIEnOF/fePYP0nn\npIlqSktTvKPfZx9YsybuLm47lG1eZ5474aYdvSMnrFNSAuvX63tKCS7qnASJdMuIIbn5Oh0d1mVJ\nlBCbIM+5rbWX0bSRX50ZcfLuuypOEiEyoDGDENYB+1JsSGb4X2MjVFSwq7Mw9fb7Hob/lZXZsJPb\nfdjTuoBt3TkI63R22qxVr/T02POdQJyAFWka0lGCir40g4Qf4iSZycSx5uq4VFbaC2OCEFFXo7We\ni0L+ipPx4+3Fc8MG625rzkl8XF0aJOfEDv9LIiE2FKK9PY0urPvvb3/fsmVxd4s29TruuoBtXTkI\n60ByoZ0dO+zNhAdxoihBRsVJUOjstLdy6XSHheSck1gTiV08ztfparDWc9FYf8VJQYG9u3vtNfuz\nOifxCaJz8m73uIEPzEQ0NkJ1NW1taYiTs8+2iRRnngn/+EfM3Wprk3BOHHGytbMq+84JJCdO3FhV\ngoRYRQk6Kk6CwqZN9nu6zsmYMdZu8Ms5gYTipKfJipPisf72OQF7h6vixBtBdE627Bln3TcvnYYb\nGyEUSk+cFBXBb35ju4ydeiq89VbU3ZJyThobobyc5rYsTSR2cZ2TZPJOtm6139U5UYY5Kk6CwgZn\nJFG64iQvb6CXeSJiTSR28ShOepvtxbPQ57AO2A8Rtxu3hnXi4/43xmgMmnVCIXi+4whrgT3wQOIn\n+CFOwD75j3+0H9AnnWTb5kaQtHOSrqOTCuk4J/pmUYY5Kk6CQn29vYjvu2/6x/LaiM2rc5Lgrrdv\npxUn4nMpMVhxsmeP/bc6J/EJheznWX5+rldiCYXg9dbJmC98EW67LXHll5Nz0taWQuv6SKqrYelS\n+546+eQheS9uD52+Pg/HamrCVFezezfBd062bbMnvqgoM2tSlCyh4iQo1NfDxIn+fLJ4bcTW3Gw/\nBWJdyDzmnPS1Ond2Pjdhg4GKnbIynauTiOOOg898JterGCAUsh/+O796vc2puuOO+E9wnJO0EmLD\nmTABnnnGvs5PO22QA1Fba6NNnvoVNjbSVxWiry/L4iRV50RDOsoIQMVJUKivTz8Z1sWrcxKvxwnY\nZgjFxQnFibh3dhm4ctfW2u/qmiTmox+Fn/wk16sYwM2BaSiqhSuugEWLYpf5GuNPQmwkU6fC00/D\nunXwiU/023DhZeoJaWqipzzFScnp4L6fkhUnmgyrjABUnAQFPxqwuSQT1kmUPemhEVve7lba8kZn\nJJ7gfohoCH34Mahw7Otfh8JC+Pa3o+/c3m6Fgx85J5Ecdhg88YQdw3vuudDb2y96PSXFNjWxZ7QV\nJ1l1TvLzrbOZbEKsOifKCEDFSVDwo8eJSyjkLSHWqzhJkHOSt7uV9nz/QzowIE7UORl+DBInVVVW\noNx338AMqXBcMZ0JcQJwzDHwq1/B738Pl17K+BqDiEfnpLGRzlL7x2RVnEDy83U0rKOMEFScBIGd\nO+1Xtp2TRGEd8DRfp6C9lY5C/8uIQcXJcGZIy50rrrAbb7xx6M6umPYrITYaZ5xhq4buv5+CG6+l\npsa7c9IxKgdhHUi+hb2KE2WEoOIkCLh3kn7lnLilxIlKEXwK6xR2tNJZlBnnpKLCfiBoWGf4UVxs\n/+/6xUlZGVx/PSxePFAf7uLulOmS3fPPhzvvhNtu49NlTyV2Trq6YPdu2opzENaB5JyT3bvtl4oT\nZQSg4iQIuOLET+ekrw9aWuLv55M4KdrTyp7izIgTEZumcNZZGTm8kmGGmHhf/KJ9nV977eAdnZ1M\ntY/VOrG48kqYPZtz2u5P7Jw4jk5rYY7COsk4J26ysSbEKiMAFSdBYMMGezUeO3bQ5r4+Gyp/+ukk\nj+e1hb2XsI6HnJOSPa10l2RGnAB87WswZ07GDq9kkCHipKgIbr4Z/vAHWLlyYHtjI+Tn0zVqDL29\nWQifXHABR2x/gvZNCeb+uOKkYBg4J9odVhlBqDgJAm4yrMigzf/5D/ztb3Dddd5Gk/TjipN4SbHG\neHNOPOScFHfvors0c+JEGb5ETX9asMC2l//mNwde2G4Zcbt9D2RcnJx7Logwb+OS+Ps5i9+ZV42I\nra7PKsk4J253WBUnyghAxUkQiFGps26d/b5qFTz3XBLH8+Kc7N5tx6t7cU4SiJPS7lZ6VZwoUYgq\nTvLy4Dvfgb/+1TZJg0Et4iEL4mTsWLa8/3Q+vftndHfH2c8R+I2EGD16yP1D5knGOdm2zZZrB2W4\nkqKkgYqTDNDdTf9F1hMxGrCtX2+7bx98MHzve0kczxUc8cRJotb1LpWVNnclTnJtWW8rfaNVnChD\niVk49tGPwtFHW/ekr2/QXB3IULVOBI1nXMBhvEbDstdi7+SKk74sTyR2KS9PTpyMH58DBaUo/qPi\nxGeMsS3EP/pRj0/o64ONG2M6J5MnwzXXwFNPDS1wiElpqfWf/RAnVVV2jXEukGV9rZhyFSfKUGK2\n3BGB734XXn3VThEOa10P2SnZLfzYqbzHPvDgg7F3ciYS7+oszH4ZMSQf1tFkWGWEoOLEZx59FB5/\n3IZiPOWJbNtm547EECfTpsHZZ9t+H4lGkwwiUa+TRBOJXRJMJjZ9hgpaB4aUKUoYcV+G8+fbmTfX\nXWcrTcKck2wIgQmTClnM56h++he2ZDgazjDCrA/9c0k2IVbzTZQRgooTH2lshK9+1Q4W3r3bY/fJ\nOGXE69dbcVJUBAsXwiOPeGwaBYm7xCYT1oGY4mTPzg4K6M3IRGJl+BMK2c70nZ0xdrj1VvtCf+WV\nrIuT6mp4pPACilsb4E9/ir6TkwuTM3FSXg4dHTY/LBHagE0ZQag48ZGrr7b5Jo88Yn9+6y0PT4oh\nTrq7bYXxtGn25y99yU7lvftuj4tJ5Jy44sQVH7FwH49RTty+zVrO+VUqTpShuM3z3CrXIcyaZat3\nILsJsdjIUvO+h/LO+Lnws59F3yksFyYnYZ1kJhOrOFFGECpOfGLZMhu6vuMOmDfPJs2vWePhifX1\ntjd7xG3Zxo32ZmnqVPtzRQVccokdTeIpBF1dnTisM2ZM4mF9rrMSwznp3K7iRInNgQfa7//+d5yd\nbr7ZvmH23TerCbFgp14vn3ghPPlk9InJQXBOIPGbvq/Prl/FiTJCUHHiA+3tcPHFcNxxcNFFUPDW\nG1w4/invzkmcMmLXOQH4r/+yDu+Pf+zhuF6cEy8lh2PG2O+xwjqN9o6uMKTiRBnKxIlWd6xfH2en\nAw+0OyxYQHu7rTQuLs7O+mpr4fGSs61I/8Uvhu6Qa3Hi1TlpbITeXk2IVUYMgREnInKZiNSLSIeI\nvCQih8fZd7yI/EJE1opIr4jcFWO/z4jIGueYr4vIqZlY+403Wtv6vvucKr5bbuGedz5G3ksrEj95\nw4aY+SajRg0MvgN7IT33XPjf/42dv9ePX+KksND62THESVeDvaMrGqviRBlKQQFMmTIgtmOy//5Q\nWNgfPslWNeyECbBme8gOBfzZz4Zmsec6rOPVOdHusMoIIxDiRETOAu4EbgQOA14HlorI2BhPKQa2\nA7cAUZsUiMgHgUeA+4HZwOPA70XkID/XvmoV3HUXfOtbAyEY6urIN71847WzoKEh/gHiOCcHHmjv\nIsO5+mqbFPvLXyZYWKKEWC+t613itLDvabQXzeJxKk6U6EydmsA5CSPbIqC21klcv/BCW6v/6quD\ndxguzol2h1VGGIEQJ8BC4D5jzMPGmLeAS4B24KJoOxtjNhljFhpjFgOxbimuAJ4yxtxljFlrjLkB\nWA1c7teiu7vhC1+wnbivvNLZ2NMDa9fyxunfoLC3k54Fn4/dwKy7G955J24ZcSQHH2yrL++4I0Gp\ncnW1vaDFsli8OicQt4V9b7M9/SX7qDhRojNtmgfnxCHb4mTCBGtK7P7gyTYkEp4Yu2ePXVAQck68\nipOamsyuR1GyRM7FiYgUAnOB5e42Y4wBlgHz0jj0POcY4SxN85iDuPNO+Ne/4Cc/sdEPAN5+G7q7\nKTztJM7lF+QvXwq33x79AJs3W+ESoztsNHEC8N//bX/v0qVxFpdovk5zc3LOSRxx0kUhZdVZShJQ\nhh1Tp1qDMG6beIe2tuwlw4J1TgDe3V4An/+8LbXbs8dudN87uQzruIooUVhn2zabHzZqVObXpChZ\nIOfiBBgL5AORqfLvAel4lOMzcMx+1q+3oZwrr4S5c8MeqKsDYMJJB/FnTuaNM661Tab++tehB4lR\nRtzRYXVLLHHy4Q/DBz6QoKV9ovk6TU3enZM44sS0tNJKRVY/UJThxdSpNlfTfbnHIxfOCTj9g84/\n374vnnjCbnTEianKoXNSUGDVmhfnRJNhlRFEEMTJsKOvz/YdmTABbrop4sE334SqKkZPqWHCBHj0\noG/BMcfYXg6RpYobNtikkokTB212yy77c1giELHuyV/+AqtXx1hkInGSTFgnTs4Ju6w4KSrydihl\n78MV2V7yTtrbs59zAk7eyUEHwRFHDLSzd8RJZ2k1xuRInIC3FvbaHVYZYRTkegFAA9ALRAZLa4Bt\naRx3W6rHXLhwIWPcElqHBQsWsMBpFvXTn1oj5M9/jmJB19XZxBARZsyAurX51io+7DBbarN06UBv\nkfp62062PyZkiVZGHMknP2kNl+99D5ZEm/oeL6zT12edEK9hnaoqeP31qA/Jrl3szqvQWWNKTGpr\nbbRh3brEM6ey7ZyUldloSH835wsvhMsvtx/2jrBvK7HvpZyJEy8t7LUBm5JllixZwpKID5+Wlhbf\njp9zcWKM6RaRVcAJwB8AREScn7+fxqFXRjnGSc72uCxatIg5c+ZEfWzrVlsxc8EFcOKJUXaoq4Mj\njwRgxgx49lms3frII3DSSfDtb9vaY4hZqbN+vTUrxsaqVcK6vVddZXuf3HYbTJoUsYPrikRzTlpa\nbDatD2GdvLZW2vM1GVaJTV6e94qdtrbsRydqa8PGQpx1Fnzta7B4cb/A31Vg3yc5yTkBb87Jtm0w\ne3Z21qMoDL5hd1m9ejVzB+U5pE5Qwjp3AV8SkfNEZAZwL1AKPAggIreJyEPhTxCRWSIyGxgNjHN+\nnhm2y93AR0TkShGZLiLfwibe/jCdhX71q7ZBVNQhfL29tmf9QbZaeeZMe0Hu6QGOP94mqdx0k20n\nC1acREmGXbfOXswTuREXXmh1w6JFUR4sKLC3hNHEide5Oi5xxElBeyvthSpOlPhMneqtYicXiacT\nJoQ5J1VV8IlP2NBOYyNUVLC7097DqXOiKNkjEOLEGPMocDVwM/Aq8H7gFGPMDmeX8cB+EU97FVgF\nzAHOwZYJPxl2zJXO9ouxvVA+CXzcGFOX6jo7O23Fwfe/PxA1GUR9vc30d8TJjBl2//5EwP/5H2u3\nnHuutWDi9DiJF9JxKS2Fyy6z1UJRi3JiNWLzOpHYpbLSXhyjDB8rbG+lU8WJkoBp07w7J9lOrh7k\nnIC1RevqbAjWmUgMAc456ey0Nw+aEKuMIAIhTgCMMfcYYyYZY0YZY+YZY14Je+xCY8zxEfvnGWPy\nI74OiNjnt8aYGc4x32+MiVd8m5CSEvj97+Gzn42xg1OpEy5OIGwAYH6+tYsLCuBTn4IdO+JOI/bC\n5ZfbFJIf/SjKg7HESbLOibtflHhiUWcrXcXl3o6j7LVMnWor0Do64u+X7YRYCGvE5nLiidZOWb48\n68MIo5LIOdEGbMoIJDDiZLggEifc8uab9kLilADU1tq7rUEzdvbZx2awvvyy/TlCnOzcCdu3excn\n48bBpZfavJMhd6axusSmEtZxFxdB8Z5WukrUOVHi476e3347/n65DOv0NzXMz4fzzrP/dhqwQY6d\nExUnyl6GihM/CavUAfttxgyGDgA85hj4znesgxKhQlyBEauMOBq33GKF0AUX2LSXfmLE5eNMAAAg\nAElEQVRNJm5qshfgCo+iIo44KelupXuUihMlPu7rOVHeSS7ESW2tbaQ86K1ywQX2+3AI66g4UUYg\nKk78pK6uP6TjElWcAFxzjW1dP27coM3uxTsZcVJWBg89BCtX2jk//cQL61RWep+u5oqTKL1ORnW3\n0lOq4kSJz7hxNj87Xt5JX19uwjqDGrG5TJsGZ54Jc+fS1pbdSclD8BLWyc+PX96nKMMMFSd+0dcH\na9ZEFSdr1kSZgyMSdQ7G+vX2BsirqeFy9NG2xPm662x0CYgvTryGdGBg30jnpKeHkr4O+karOFHi\nI5K4YsfNR8lFQixE5J0APPYYXH11f3fYnPXy8eKc1NQMnRKqKMMYfTX7xaZN9uoaRZw0N9vcVy+4\nZcSpcPPNdpLxeec5c0xccRKpjJKZSAxWKYkMFSfO3ZyKE8ULiSp22tvt92w7J+PH25f3EHHikLPW\n9S4VFfbaEqVaDtDusMqIRMWJX0RU6rjMdDqvRA3tRMFrGXE0Skrg4YdtM9dbb8WKk+5u+ssNXJJ1\nTvLy7AUyUpy4d3PJ2jzKXkki5yRXVTGFhTZPfVBYJ4ycDf1zSTSZWHucKCMQFSd+8eab9vZqv8Ht\nWKZMseFgL+LEmOTKiKMxd64N7Xz727C+0XFHIkM7yUwkdok2X8cRJzJGxYmSmGnT7HipWBGKXJbs\nDmrEFkEgnBNQcaLsVag48Qs3GTYiMF1cbJvAehEn27fbC3c64gTg2mvh0EPhukUxhv8lM5HYpaoq\npnOSN0b7nCiJccOV7mDLSHIpToY0Ygsj5+LEdU5iqToVJ8oIRMWJX0Sp1HGJWbETQSqVOtEoLLTV\nO69tiSFOkg3rQNQW9qbFXiwLqtU5URKTqJzYFSfZToiF+M5JzsM68ZwTY6w40e6wyghDxYkfGOOb\nOBGxoaB0OfRQuORaK07WrvQprBMhTrobrTgpDKk4URJTWWlLimMlxapzEoN4zklzs23Sos6JMsJQ\nceIHW7bYK2sccbJxY+LW3evXw/7728RWP7ji2nJ6pIBHftg0kBPb3W3vwFIJ60TknHQ1OOKkKpdX\nbmU4ES8pNlfVOmDFyfbtTpVbBDkXJ/GcE23ApoxQVJz4QYxKHZcZMwaSXeORThlxNPILBKqryW9u\n5BvfcDa67ocPYZ3upl20Uk5Zub6MFG/EKyfOdUKsGyGJpK0tx+LE/eUqTpS9CP1U8YO6Ohso33//\nqA+7AwDXrIl/mHTKiGNRsE+IM45u5Ic/tHPMkp5I7BJFnPQ2t9JKRW7j8cqwIp5z0tYGRUV2qkO2\nidmIDeuc5PQ1XlAAo0ZFD+uoOFFGKCpO/ODNN21DkxgdGqurbR+FeHknfX22isFvcUIoxKx9Gznu\nOPjiF6GvMcmhfy5RSoldcZKLBEZleDJtmn0ZRWtcnMvE06gt7B1yHtaB2C3st22zi8v5AhXFX1Sc\n+EGcZFiXREmxW7bAnj2ZESfS1Mj119u8l82vpyhOqqqgs9N+OfS1tLKLcnVOFM/Eq9hpa8tNpQ7Y\nfoVFRbGdk5x/9sdqYa/dYZURioqTdElQqeOSSJz4VUY8hOpqaGriiCOsO/z2P9II6wC0tAxsa9Ww\njpIcBx5ov0fLO8nF0D8XERvaiRQnuRpGOIR4zomKE2UEouIkXd59197ReBAna9fai1001q2z/Uli\npK2kjjNfp6wM5syBd/7VbG8RR41K7jiuOAnLO5FdGtZRkqOszIZQYjknuRQB0cqJOzrs/UdgnRMV\nJ8oIJQepZyOMBJU6LjNm2Avdli3RBcj69ba/ie/JgGGTiY8+Ghp+6vQ4SXbEqitOwvJO8na30sr7\ncn9XqQwrYlXs5FqcRGvEtnu3/Z5zcRLPOXEz7kcQmzdvpqGhIdfLUCIYO3YsEydOzMrvUnGSLnV1\ntjHJ5Mlxd3OvH2+9FV2c+F1G3E8oZN2O3l7mz89n06ImuvepojDZ47g5KmHOSX77LnZJBUVFvq1W\n2QuYOhVefnno9lyLk9paeOONwdtyWd48iPLy6Nm6I7A77ObNm5k5cybtbuMbJTCUlpayZs2arAgU\nFSfp8uabVnnk58fdbeJEq2HeegtOOWXo4+vWwZlnZmB9oZD1pZubOfrosbTSTItUMTbZ40QJ6xS2\nt9JZWJG0CaPs3UybBr/4hX1Zhr92cpkQC9FzTgLlnEQmrXV1QUPDiAvrNDQ00N7ezuLFi5npjnVX\ncs6aNWv43Oc+R0NDg4qTYYGHZFiw2mXatOhJsV1dtpLG90odGEh8bWqiZtpY9i1rZmtXdfLipKzM\n/hHh4qSzlc5ibV2vJMfUqVaIbN060F8EbOLpuHG5W9eECTbfO9zBCYw4KS8fGtbZvt1+H2HixGXm\nzJnMmTMn18tQcoQmxKaDx0odl5kzozdiq6+H3t4MiZPQ4OF/E8ua2NSSZBkx2Fvc8F4nxlC8p5Uu\nFSdKkriv88i8kyCEdWCwexKosE5kQqw2YFNGMCpO0uG99+yHtUdxEqucOGNlxDBEnIwraGZDc1XM\n6etxqaoacE7a28kzffSMKvdnncpewwEH2H6FkRU7uRYn0RqxBcY5iZYQq+JEGcGoOEkHj5U6LjNm\nDOiZcNats7H2cIvbNyLEyeieZhqp5qWXUjhWeAt7R930lKpzoiRHURFMmjQ8nJPAiJPychv36ukZ\n2LZtm3U099knd+tSlAyh4iQd6urslXbKFE+7uxU7a9cO3r5+vbW6M5JYWlxsr/iOOClobaK7rIoX\nXkjhWFHESW+ZihMleaLN2Ml1Quzo0dagiAzr5OeT+4o0dzKxq5bAJu2MG5ebYUSKkmFUnKTDm2/C\n9OmeLw5urD0ytJOxMmIXp0ssnZ1IZyf7TK/ixRdTOE54zokjTvpGqzhRkidar5NcOycwtBGb27o+\n5xVp5U74NDweqw3YlDisXbuWvLw8Hn300VwvJSVUnKRDEsmwMDC4OJo4yUgyrIvbiM0RFhNn27BO\nd3eSxwnPOXHj3xUqTpTkmTrVDrrs7bU/9/TYqrVci5PIRmyBmKsDA++z8LwTFSfDiry8vIRf+fn5\nPP/88779Tsm5qk4d9QPToa4OTjghqadEJsW2tdk7tayIkyY7V2fqkVW0/xReew0OPzyJ40QJ68gY\nFSdK8kydasXIli02/8Ttt5VrcVJbCxs2DPzc1hYQcRLLOXGHFSmBZ/HixYN+fuihh1i2bBmLFy/G\nGNO/3a/eLtOnT6ejo4OinMckU0PFSars2GEbICXhnIAVJ08/PfDzv/9tv2dcnDQ09Dsn04+qpqQE\nXnghfXGSN0ardZTkcV/v69ZZcRKUkt0JExiUj7V7d+7XBMR2TubPz816lKQ555xzBv28cuVKli1b\nxoIFCzw9v7Ozk5KSkqR+53AVJqBhndRJslLHZeZMK0i6uuzPGS0jdolwTopqqjjiCJLPO3FzToyB\n1lb2SDHFFcX+r1cZ8UycaAddunknQREnbpdY90Y2MGEd1zlxxYkxNiF2hLWuVyxLly4lLy+Pxx57\njK9//etMmDCB0aNH09XVRUNDAwsXLuSQQw5h9OjRVFZW8rGPfYw69zPJIVrOydlnn824cePYsmUL\np59+OuXl5dTU1HDttddm+09MiDonqVJXZxNhk1QVM2bYOPvbb1uhsn69zVd1K34zgpsQ6yazVlUx\nfz488MDQFuJxqaqyyQHt7dDaym7Kc/5hogxPCgpskVukOMn1hOvaWtizx75dQqEAihM3rLNrl50k\nqjknI5rrr7+esrIyvv71r9PW1kZ+fj5r167l6aef5tOf/jT7778/W7du5d577+XYY4+lrq6OsWNj\n9/8WEbq7uznppJM49thjueOOO3j66af57ne/y7Rp0zj//POz+NfFR8VJqrz5pvWmC5MboRc+AHDm\nzCwkw8LghNiyMigq4uij4dZbrUjyHLYOn6/T2koLFTn/MFGGL+HlxEHJOQlvxBYKWdHkvuxzSkEB\njBo14JxoAzbAvm6iNbb0mxkzciOcjTG8+OKLFIRVhB5++OGsiWg1vmDBAg4++GAeeughrrrqqrjH\n3LVrFzfccANXXnklAF/+8pc55JBDeOCBB1ScjAiSrNRx2Wcfe7Fz31AZLyMGe5Vtb7d+tTNdeN48\n65i88EJq4sS0tNJiKnL+YaIMX6ZNg8cft/8OUlgH7Fvl/e+3zsm+++Z2Tf2Et7BXcQLY6+jcuZn/\nPatWQS7G/Fx00UWDhAkMziPp7e2lpaWFyspKJk+ezOrVqz0d9+KLLx708/z58/njH/+Y/oJ9RMVJ\nqtTVwZe/nPTTRAZX7KxbB6ed5vPaInFjRm+/3S9OqqrgkENs3skFF3g8jitOmpvp29lKKypOlNSZ\nOtXOleruDo44cVM43F4ngQnrwOAW9ipOAHstXbUqO78nF0yaNGnItr6+Pu644w7uu+8+Nm3aRF9f\nH2BDNgd6uNOsrKxkdMSLuqqqiubI1uU5JjDiREQuA64GxgOvA181xvwjzv7HAncCBwObge8YYx4K\ne/x84GeAAdysik5jTPrmXGOj7UOfgnMC9oVeV2fj2o2NWQrrwECCi8PRR8NzzyVxHEfYsHMnvTt3\n0aphHSUNpk2z+Vf19cERJ4WF1t10e50EoTFcP+HOydatUFICY8bkdk05prQ0N45Gthg1atSQbTfc\ncAO33norl1xyCccddxxVVVXk5eVx6aWX9guVeOTn50fdHl7OHAQCIU5E5Cys0LgYeBlYCCwVkWnG\nmIYo+08C/gjcA5wDnAj8RETeNcb8OWzXFmAaA+LEn7PvxvsOPjilp8+YAb/73UC8PePixBUkb789\nqNX+/Plw7722yjhODtUAYWGdvpZWWqllbFAu3Mqwww1nrls3IE6iXIuzTngjtkA7J+PHB6B1rZJt\nfvvb33Laaadxzz33DNre1NTEFI+jVIYDQSklXgjcZ4x52BjzFnAJ0A5cFGP/S4ENxphrjDFrjTE/\nAn7jHCccY4zZYYzZ7nzt8GW1dXV24EaKySIzZtgbILcRYMb7KLnOSXv7gPvBQIuEFSs8HqekxM7q\n2bkTWjSso6RHba29812/3r40R42y04pzTXgL+0CJk8ick708pDPSidXdNT8/f4jL8fOf/5xGZ37a\nSCHnlwIRKQTmAsvdbcae+WXAvBhPO8p5PJylUfYfLSIbRWSziPxeRFKLw0Ty5ptWURSn1uPDbQD4\nhz/YC2HGL36VlQN3WGFhnYkTbbJfUkMAq6ps1c+uVnZRrmEdJWXy8uzbyHVOgiJ0Xeekr8+KpqCs\nK6pzooxYYoVZTj/9dJ5++mkuvvhi7r//fi6//HKuvPLKqPkpw5mcixNgLJAPvBex/T1s/kk0xsfY\nv0JEXMWwFuu8nAGci/1bV4hIbdorTrFSx2XyZBvbXrEiCyEdsC6P65iEOSciNu8kKXHidInN263O\niZI+7gDAIIkT1zlxy5sD5ZyoOBlRxJt9E+uxb33rW1xxxRU8+eSTXHnlldTV1fHMM88wfvz4Ic+J\ndoxYxw3aHJ5A5JxkAmPMS8BL7s8ishJYA3wZuDHecxcuXMiYiESzBQsWDLQZrquDCy9MeW2FhfaO\ncc2aLIkTGGjEFiZOwIZ2rrzS9nPyFO93xEl+m4oTJX2mToXFi23lWFBeSxMm2Hx3d1JDoMRJeEKs\ndocd1vzgBz/gBz/4QdTHTjnlFHrdqZgRlJSUsGjRIhYtWjRo+4qI+Pz06dOHHGPJkiVRj3nbbbdx\n2223eV16/7Eij9fS0pLUMeIRBHHSAPQCNRHba4BtMZ6zLcb+rcaYPdGeYIzpEZFXgYQZHosWLWJO\nrBTwnTut55uGcwI272TNmiz0OHEJhWzf/LCwDlhx0t0Nr7wCH/qQh+NUVsL27eR3dWq1jpI206bZ\n4X8NDcERJ7W1tnPy22/bn4Oyrv6wTm+vne2lzomSQwbdsDusXr2auT41nsl5WMcY0w2sAvrH+4r1\nl04AYqVqrgzf3+FkZ3tURCQPOBTYms56063UcXHr5rPmnLhJsRHOyaGH2hsyz6GdqirYvBlAnRMl\nbVxx/s9/5r51vYvbiM2tpgucc7Jjh02IUXGijGByLk4c7gK+JCLnicgM4F6gFHgQQERuE5GHwva/\nFzhARG4Xkeki8hXg085xcJ5zvYicJCKTReQw4BfAROAnaa20rs5m8qWpKlxxklXnBIY4J/n5tlus\nZ3FSWWlvdYG2vAqG8dBLJQC4b6M1a4LjULgt7Neutd8DI04qKmwijFtKpOJEGcEEQpwYYx7FNmC7\nGXgVeD9wSljp73hgv7D9NwIfxfY3eQ1bQvwFY0x4BU8V8GOgDngSGA3Mc0qVU+fNN+GAA9JuyPDx\nj8P3vgfTp6d1GO/EcE7AhnZWrLA3YwlxJxMDXSUV2mZBSYuxY20fsZ6e4IiTUMjmhQXSOYGBaYkq\nTpQRTCDECYAx5h5jzCRjzChjzDxjzCthj11ojDk+Yv/njTFznf2nGmN+HvH4lcaYyc7jtcaYjxlj\n/pn2QtOs1HEZMwauvjqLfR1cxySGONm50/5pCQmbgtZTWuHT4pS9FZEB9zAo4iQvz4Z2XHESlHVR\n4bzf3IXVRKbdKcrIITDiZNjgkzjJOrW1UFQUdcTqEUfY8I6n0E6YuOkrK/dxgcreihvaCYwIwL5d\n3ITYwDkn69bZm40U+ywpynBAxUmy/PCHcM45uV5F8pxzDrz0klUhEZSV2fkUnsRJmLjpLVPnREkf\n1zkJSkIs2LyTnh4b3glMXpXrnKxfryEdZcSj4iRZzjjDlrgMN0aNgsMOi/nw/Pl2QnFCHHHSh5BX\nHqBbXWXYElTnBIK1pkHOiYoTZYSj4kQBbKfYjRvhnXcS7OiIk87CckpH68tHSZ+g5ZzAQMVOYEI6\nMOCc7Nyp4kQZ8einiwJYcQIe3BMn56QtXxuwKf4QRHHiOieBEiflYTle2h1WGeGoOFEAeyN24IEe\n8k4c56QtXxuwKf5QWQk//Sl88pO5XskArnMSqNd4QYGdDA7qnCgjniC0r1cCgqe8E2fm0C5R50Tx\njzRGVWWEQDonYEM7nZ0qTpQRjzonSj9HHw2vvz4wWywqhYVQVsYuUx6su0pF8ZHAihM3tKPiZNiR\nl5eX8Cs/P5/nn3/e19+7ZcsWbrrpJuo8NbIKDuqcKP3Mn2+7xP71r/Cxj8XZsaqKlmYN6ygjl/Jy\n+xW417ibFKviZNixePHiQT8/9NBDLFu2jMWLF2OM6d8+c+ZMX3/v5s2buemmm5g5cyYHDaMeXSpO\nlH6mT7fVxj/+cQJxUlnJzgYN6ygjm9raAIoT1znRhNhhxzkR/bFWrlzJsmXLhkz29Ztw4TOc0LCO\n0o8IXHYZPPkk1NfH2fGWW7hXLg3ehVtRfOTWW+HLX871KiKoqLCh1ShjKJSRRWdnJ9deey1Tpkyh\npKSESZMmcd1119Hd3T1ovz/96U8cffTRVFZWUl5ezsyZM7npppsAWLp0Kccccwwiwtlnn90fOnr0\n0Udz8SclhTonyiAWLLAzf+69F26/Pfo+5uNn8sIeOEfFiTKCCVL1UD/l5XamTtaGcim5oK+vj1NP\nPZXVq1dzySWXMHXqVF599VVuv/12NmzYwCOPPALAa6+9xplnnsnhhx/Od77zHYqKili3bh0rVqwA\nYNasWVx//fXccsstXH755Rx11FEAzJs3L2d/m1dUnCiDKC21lRMPPAA33TRQuRjOnj02N0XDOoqS\nZSZPhsbGXK9CyTA//elPefHFF1m5ciVz587t3z59+nQWLlzINddcw+zZs1m6dCl9fX38+c9/pjTK\nBXn8+PGcfPLJ3HLLLcyfP5/Pfvaz2fwz0kLFiTKESy+FRYvgV7+C888f+nhbm/2uYR1FyTI33wy9\nvbleRXBob4e33sr875kxI6t3Y7/5zW+YNWsWkyZNojFMjB5//PEYY3j22WeZPXs2lZWVGGN47LHH\nOPfcc7O2vmyg4kQZwtSpcMop8KMfRRcn7e32u4oTRcky+flRh3futbz1FoQ5Cxlj1So7HTVLrF+/\nno0bNzJu3Lghj4kI27dvB+Dzn/88Dz74IOeddx5XXXUVJ554Ip/61Kf4xCc+kbW1ZgoVJ0pULrvM\nzjj8xz/g8MMHP+Y6JxrWURQlp8yYYYVDNn5PFunr62Pu3LncfvvtUatt9t9/fwBKS0tZsWIFy5cv\n509/+hNPP/00jzzyCKeddhp//OMfs7pmv1FxokTltNNg//2te/Lgg4Mf07COoiiBoLQ0q45Gtpgy\nZQqbNm3iuOOOS7iviHDiiSdy4oknctddd3HjjTfy7W9/mxUrVvDBD34QEcnCiv1HU76VqOTn29yT\nX/4SGhoGP6ZhHUVRlMzx2c9+lg0bNvDzn/98yGPt7e10dHQA0NTUNOTxWbNmAbBnzx4AypwL9c6d\nOzO13IygzokSky98AW680Q5lu+aage0a1lEURckcX/jCF/j1r3/NhRdeyDPPPMO8efPo7u6mrq6O\nX//617zwwgscdNBBXHvttaxevZqPfOQjTJw4ka1bt3LPPfdwwAEHcOSRRwK2wqesrIwf/vCHFBYW\nUlpaygc/+EH222+/HP+V8VFxosRk7Fg46yz4v/+Dq64ayMPTsI6iKEr6xAq55Ofn89RTT3HHHXew\nePFifvOb3zB69GimTJnCNddcw6RJkwD41Kc+xdatW3nggQdobGxk3LhxnHzyydx00039pcUlJSU8\n/PDDXHfddVxyySX09PSwZMmSwIsTGa6tbTOBiMwBVq1atYo5IzCOmQovvwxHHglPPAGnn263/fzn\ncN55djhqcXFu16coyshi9erVzJ07F70OBwsv/y/uPsBcY8zqdH6f5pwocTniCPjAB2xirEtbm21Q\nWVSUu3UpiqIoIxcVJ0pCvvIVePpp+Pe/7c9tbTakM0yTwBVFUZSAo+JEScjZZ0N1tc09AVuto/km\niqIoSqZQcaIkZNQouOgiW7XT3m6dE63UURRFUTKFihPFE5deCi0tsGTJQFhHURRFUTKBihPFEwcc\nAKeeahNjVZwoiqIomUTFieKZyy6DV1+Fv/5VwzqKoihK5lBxonjmIx+xDsqGDeqcKIqiKJlDO8Qq\nnsnLs7kn//3fKk4URcksa9asyfUSlDCy/f+h4kRJiosuguuv17COoiiZYezYsZSWlvK5z30u10tR\nIigtLWXs2LFZ+V0qTpSkqK6GBx6AyZNzvRJFUUYiEydOZM2aNTREjkNXcs7YsWOZOHFiVn6XihMl\nac45J9crUBRlJDNx4sSsfQgqwSQwCbEicpmI1ItIh4i8JCKHJ9j/WBFZJSKdIrJORM6Pss9nRGSN\nc8zXReTUzP0FSqosWbIk10vY69Bznn30nGcfPefDl0CIExE5C7gTuBE4DHgdWCoiUYNbIjIJ+COw\nHJgF3A38REROCtvng8AjwP3AbOBx4PciclDG/hAlJfQCkn30nGcfPefZR8/58CUQ4gRYCNxnjHnY\nGPMWcAnQDlwUY/9LgQ3GmGuMMWuNMT8CfuMcx+UK4CljzF3OPjcAq4HLM/dnKIqiKIqSLjkXJyJS\nCMzFuiAAGGMMsAyYF+NpRzmPh7M0Yv95HvZRFEVRFCVg5FycAGOBfOC9iO3vAeNjPGd8jP0rRKQ4\nwT6xjqkoiqIoSgDQap3BlIA2/8k2LS0trF69OtfL2KvQc5599JxnHz3n2SXss7Mk3WMFQZw0AL1A\nTcT2GmBbjOdsi7F/qzFmT4J9Yh0TYBKgzX9ywNy5c3O9hL0OPefZR8959tFznhMmASvSOUDOxYkx\npltEVgEnAH8AEBFxfv5+jKetBCLLgk92tofvE3mMkyL2iWQpcC6wEej09hcoiqIoioJ1TCZhP0vT\nQmzuaW4Rkc8CD2KrdF7GVt18GphhjNkhIrcBtcaY8539JwH/Au4BfooVIf8LnGaMWebsMw94Dvgm\n8CSwAPgGMMcYU5elP01RFEVRlCTJuXMCYIx51OlpcjM29PIacIoxZoezy3hgv7D9N4rIR4FF2JLh\nd4AvuMLE2WeliJwDfMf5Wg98XIWJoiiKogSbQDgniqIoiqIoLkEoJVYURVEURelHxYmiKIqiKIFC\nxYlDsoMHFe+IyIdE5A8i8h8R6RORM6Lsc7OIvCsi7SLyZxE5MBdrHSmIyDdF5GURaRWR90TkMRGZ\nFmU/Pe8+ISKXOANGW5yvFSLykYh99HxnCBH5hnN9uStiu55zHxGRG53zHP5VF7FP2udcxQnJDx5U\nkqYMm+T8FWBIkpOIfB078+hi4AigDXv+i7K5yBHGh4AfAEcCJwKFwDMiMsrdQc+772wBvg7MwY7k\n+AvwuIjMBD3fmcS5mbwYe+0O367nPDO8gS1eGe98zXcf8O2cG2P2+i/gJeDusJ8FWwF0Ta7XNtK+\ngD7gjIht7wILw36uADqAz+Z6vSPlCzsmog+Yr+c9q+e9EbhQz3dGz/FoYC1wPPAscFfYY3rO/T/f\nNwKr4zzuyznf652TFAcPKj4hIpOxyjv8/LcCf0fPv59UYl2rJtDznmlEJE9EzgZKgRV6vjPKj4An\njDF/Cd+o5zyjTHXC9G+LyGIR2Q/8PeeB6HOSY+INHpye/eXsdYzHfmjqkMYM4XRc/l/gBTPQ50fP\newYQkUOwXahLgF3AJ4wxa52mkHq+fcYRgLOBD0R5WF/jmeEl4AKsW/U+4FvA885r37dzruJEUUY+\n9wAHAUfneiF7AW8Bs4Ax2C7XD4vIMbld0shERPbFiu4TjTHduV7P3oIxJrw1/Rsi8jKwCfgs9vXv\nC3t9WIfUBg8q/rENm+Oj5z8DiMgPgdOAY40xW8Me0vOeAYwxPcaYDcaYV40x12ITNP8LPd+ZYC4w\nDlgtIt0i0g18GPgvEenC3q3rOc8wxpgWYB1wID6+zvd6ceIobnfwIDBo8GBaUxWVxBhj6rEv2vDz\nX4GtMtHznwaOMPk4cJwxZnP4Y3res0YeUKznOyMsAw7FhnVmOV+vAIuBWcaYDW+UeqcAAAS9SURB\nVOg5zzgiMhorTN7183WuYR3LXcCDznRkd/BgKXYYoZImIlKGffGKs+kAEZkFNBljtmCt2etE5N/Y\nidC3YKulHs/BckcEInIPdtjlGUCbiLh3Mi3GGHfitp53HxGRW4GngM1AOXbC+YexE9NBz7evGGPa\ngMj+Gm1AozFmjbNJz7nPiMj3gCewoZwJwE1AN/BLZxdfzrmKEzwNHlTS4wPYEj/jfN3pbH8IuMgY\n8/9EpBS4D1tV8jfgVGNMVy4WO0K4BHuun4vYfiHwMICed9/ZB/uafh/QAvwTONmtItHznRUG9VHS\nc54R9gUeAULADuAF4ChjTCP4d8518J+iKIqiKIFir885URRFURQlWKg4URRFURQlUKg4URRFURQl\nUKg4URRFURQlUKg4URRFURQlUKg4URRFURQlUKg4URRFURQlUKg4URRFURQlUKg4URQlZUTkWRG5\nK9frCEdE+kTkjFyvQ1GU1NEOsYqipIyIVALdxpg2EakHFhljvp+l330jcKYx5rCI7fsAzc5QT0VR\nhiE6W0dRlJQxxuz0+5giUpiEsBhyd2WM2e7zkhRFyTIa1lEUJWWcsM4iEXkW2B9Y5IRVesP2mS8i\nz4tIu4hsEpG7ncFg7uP1InKdiDwkIi3YgWGIyHdFZK2ItInI2yJys4jkO4+dD9wIzHJ/n4ic5zw2\nKKwjIoeIyHLn9zeIyH3OpGz38Z+JyGMicpWIvOvs80P3dymKkn1UnCiKki4G+AR2LPr1wHjsZF5E\nZArwFPBr4BDgLOBo4AcRx7gKOw18NnbEOkArcB4wE7gC+CKw0HnsV9jp1m9iJ4m/z9k2CEcELQUa\ngbnAp4ETo/z+44ADgGOd33mB86UoSg7QsI6iKGljjNnpuCW7I8Iq3wAWG2NcMbBBRL4GPCcil4aN\nUV9ujFkUccxbw37cLCJ3YsXNHcaYThHZDfQYY3bEWdq5QDFwnjGmE1gjIpcDT4jI18Oe2wRcbmwS\n3joReRI4AXgg2XOhKEr6qDhRFCWTzAIOFZHPhW0T5/tkYK3z71WRTxSRs4CvAlOA0djrVUuSv38G\n8LojTFxexLrG0wFXnLxpBlcHbMU6PYqi5AAVJ4qiZJLR2BySuxkQJS6bw/7dFv6AiBwFLMaGiZ7B\nipIFwJUZWmdkAq5Bw96KkjNUnCiK4hddQGQS6WrgIGNMfZLH+iCw0RjzXXeDiEzy8PsiWQOcLyKj\njDEdzrb5QC8Dro2iKAFD7wwURfGLjcAxIlIrIiFn2+3AB0XkByIyS0QOFJGPi0hkQmok64GJInKW\niBwgIlcAZ0b5fZOd44ZEpCjKcX4BdAIPicjBInIc8H3g4QS5Koqi5BAVJ4qipEN4nsYNwCTgbWA7\ngDHmX8CHganA81gn5VvAf2IcA+d5TwCLsFU1rwJHATdH7PZb4GngWef3nR15PMctOQWoBl4GHgX+\njM1lURQloGiHWEVRFEVRAoU6J4qiKIqiBAoVJ4qiKIqiBAoVJ4qiKIqiBAoVJ4qiKIqiBAoVJ4qi\nKIqiBAoVJ4qiKIqiBAoVJ4qiKIqiBAoVJ4qiKIqiBAoVJ4qiKIqiBAoVJ4qiKIqiBAoVJ4qiKIqi\nBAoVJ4qiKIqiBIr/Dz+/P7AoHMaRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fce7c94a210>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# %%capture\n",
    "for mn in rk_models: # ['listnet']: # \n",
    "    args = {}\n",
    "    if mn == 'ranknet':\n",
    "        args = {} # {'batchsize':100, 'n_iter':5000, 'n_units1':512, 'n_units2':128, 'tv_ratio':0.95}\n",
    "    elif mn == 'listnet':\n",
    "        args = {'batchsize':100, 'n_epoch':1, 'n_units1':32, 'n_units2':16, 'tv_ratio':0.9}\n",
    "    rk_models[mn].fit(dummies['trn'].values, data_trn.Label, **args)\n",
    "\n",
    "# clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for mn in my_models:\n",
    "    pair_ranker.fit(my_models[mn], mn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Series(model_rf.feature_importances_, index=features_trn.columns).sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sfn = fn.keys() # specify a subset of fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "preds = {k:{mn:md.predict_proba(dummies[k]) for mn, md in models.items()} for k in sfn}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rk_preds = {k:{mn:md.predict(dummies[k].values) for mn, md in rk_models.items()} for k in sfn}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my_preds = {k:{mn:pair_ranker.predict(data[k]['QuestionID'], dummies[k], mn) for mn in my_models} for k in sfn}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ranks = {k:deepcopy(dummies[k]) for k in fn}\n",
    "dummy_rank_features = {k:deepcopy(dummies[k]) for k in fn}\n",
    "rank_trn, rank_dev, rank_test = ranks['trn'], ranks['dev'], ranks['test']\n",
    "all_tps = tps + tps_wgt + tps_wv + tps_wv_wgt + models.keys() + rk_models.keys() + my_models.keys()\n",
    "for k in sfn:\n",
    "    rk = ranks[k]\n",
    "    for col in ['QuestionID', 'SentenceID']:\n",
    "        rk[col] = data[k][col]\n",
    "    if k != 'test':\n",
    "        rk['Label'] = data[k]['Label']\n",
    "    for mn in models:\n",
    "        rk[mn] = preds[k][mn][:, 1] # larger is better\n",
    "    for mn in rk_models:\n",
    "        rk[mn] = rk_preds[k][mn][:, 0]\n",
    "    for mn in my_models:\n",
    "        rk[mn] = my_preds[k][mn]\n",
    "    grp = rk.groupby('QuestionID')\n",
    "    for tp in all_tps:\n",
    "        col = 'rank_' + tp\n",
    "        rk[col] = grp[tp].rank(method='min', ascending=False).astype(int)  \n",
    "        rk.sort_values(['QuestionID', col], inplace=True)\n",
    "        # add one Strictly Increasing Monotonically vector to avoid same ranks\n",
    "        b = np.ones(rk.shape[0]).cumsum()\n",
    "        rk[col] += b\n",
    "        grp = rk.groupby('QuestionID')\n",
    "        rk[col] = grp[col].rank(method='min', ascending=True).astype(int)  \n",
    "        dummy_rank_features[k][col] = rk[col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev\n",
      "\twords                [ 0.66772  0.67436  0.66772  0.67436]\n",
      "\tstems                [ 0.66973  0.67913  0.66973  0.67913]\n",
      "\twords_not_stop       [ 0.65747  0.66276  0.65747  0.66276]\n",
      "\tstems_not_stop       [ 0.67028  0.67904  0.67028  0.67904]\n",
      "\tlemma                [ 0.67349  0.68225  0.67349  0.68225]\n",
      "\tlemma_not_stop       [ 0.68214  0.6909   0.68214  0.6909 ]\n",
      "\twords_wgt            [ 0.65222  0.65798  0.65222  0.65798]\n",
      "\tstems_wgt            [ 0.6847   0.69353  0.6847   0.69353]\n",
      "\twords_not_stop_wgt   [ 0.66889  0.67507  0.66889  0.67507]\n",
      "\tstems_not_stop_wgt   [ 0.66975  0.67792  0.66975  0.67792]\n",
      "\tlemma_wgt            [ 0.68184  0.69041  0.68184  0.69041]\n",
      "\tlemma_not_stop_wgt   [ 0.68163  0.6902   0.68163  0.6902 ]\n",
      "\twords_wv             [ 0.59265  0.59182  0.59265  0.59182]\n",
      "\tstems_wv             [ 0.58874  0.5895   0.58874  0.5895 ]\n",
      "\twords_not_stop_wv    [ 0.62636  0.631    0.62636  0.631  ]\n",
      "\tstems_not_stop_wv    [ 0.54792  0.55225  0.54792  0.55225]\n",
      "\tlemma_wv             [ 0.58557  0.58398  0.58557  0.58398]\n",
      "\tlemma_not_stop_wv    [ 0.62133  0.62693  0.62133  0.62693]\n",
      "\twords_wv_wgt         [ 0.60172  0.60445  0.60172  0.60445]\n",
      "\tstems_wv_wgt         [ 0.56176  0.56882  0.56176  0.56882]\n",
      "\twords_not_stop_wv_wgt [ 0.62568  0.63001  0.62568  0.63001]\n",
      "\tstems_not_stop_wv_wgt [ 0.5719   0.57668  0.5719   0.57668]\n",
      "\tlemma_wv_wgt         [ 0.59694  0.60301  0.59694  0.60301]\n",
      "\tlemma_not_stop_wv_wgt [ 0.6061   0.60989  0.6061   0.60989]\n",
      "\trf                   [ 0.67367  0.67731  0.67367  0.67731]\n",
      "\tlr_l1                [ 0.65783  0.6616   0.65783  0.6616 ]\n",
      "\tlr_l2                [ 0.66073  0.6645   0.66073  0.6645 ]\n",
      "\tranknet              [ 0.62936  0.63237  0.62936  0.63237]\n",
      "\tmy_rf                [ 0.65566  0.65877  0.65566  0.65877]\n"
     ]
    }
   ],
   "source": [
    "for k in sfn:\n",
    "    if k in ['test', 'trn']:\n",
    "        continue\n",
    "    print k\n",
    "    rk = ranks[k]\n",
    "    for tp in all_tps:\n",
    "        col = 'rank_' + tp\n",
    "        rk.sort_values(['QuestionID', col], inplace=True)\n",
    "        grp = rk.Label.groupby(rk.QuestionID)\n",
    "        print '\\t', '{: <20}'.format(tp), np.round([MAP(grp), MRR(grp), MAP(grp, keep_no_ans=False), MRR(grp, keep_no_ans=False)], 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# #res = collections.defaultdict(dict)\n",
    "# print sfn, '\\n'\n",
    "# print ' ' * 15, '{: >15}\\t{: >15}'.format('MAP', 'MRR') \n",
    "# for tp in tps + tps_wgt + models.keys() + rk_models.keys() + my_models.keys(): # ['rf']: #\n",
    "#     print '{: >20}'.format(tp), \n",
    "#     for k in sfn:\n",
    "#         if k in ['test', 'trn']:\n",
    "#             continue\n",
    "#         col = 'rank_' + tp\n",
    "#         ranks[k][['QuestionID', 'SentenceID', col]].sort_values(['QuestionID', col]).to_csv('rank.txt', sep='\\t', header=False, index=False)\n",
    "#         out = commands.getoutput('python eval.py rank.txt %s' % fn[k])\n",
    "#         m = re.search('MAP: ([.0-9]+)\\nMRR ([.0-9]+)', out)\n",
    "#         print '\\t%f\\t%f' % (float(m.group(1)), float(m.group(2))),' ',\n",
    "#     print ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rank_and_evaluate(y, k):\n",
    "    '''\n",
    "    if k is 'test', then this func will only rank, not evaluate \n",
    "    '''\n",
    "    d = DataFrame({}, data[k].index)\n",
    "    d['y'] = y\n",
    "    cols = ['QuestionID', 'SentenceID']\n",
    "    if k != 'test':\n",
    "        cols.append('Label')\n",
    "    for col in cols:\n",
    "        d[col] = data[k][col]\n",
    "    \n",
    "    \n",
    "    grp = d.groupby('QuestionID')\n",
    "    d['rank'] = grp['y'].rank(method='min', ascending=False).astype(int)    \n",
    "    \n",
    "    d.sort_values(['QuestionID', 'rank'], inplace=True)\n",
    "    res = dict()\n",
    "    res['rank_df'] = d\n",
    "    if k == 'test':\n",
    "        return res\n",
    "    grp = d.Label.groupby(d.QuestionID)\n",
    "    res['my_MAP'] = MAP(grp, keep_no_ans=False)\n",
    "    res['my_MRR'] = MRR(grp, keep_no_ans=False)\n",
    "    d[['QuestionID', 'SentenceID', 'rank']].sort_values(['QuestionID', 'rank']).to_csv('rank.txt', sep='\\t', header=False, index=False)\n",
    "    out = commands.getoutput('python eval.py rank.txt %s' % fn[k])\n",
    "    m = re.search('MAP: ([.0-9]+)\\nMRR ([.0-9]+)', out)\n",
    "    res['MAP'] = float(m.group(1))\n",
    "    res['MRR'] = float(m.group(2))\n",
    "    return res    \n",
    "\n",
    "def save_final_rank(df, dst_fn='final_rank.txt', rank_col='rank', \n",
    "                    save_cols=['QuestionID', 'SentenceID', 'rank'], sort_by_cols=['QuestionID', 'rank'], re_rank=False):\n",
    "    if re_rank:\n",
    "        col = rank_col\n",
    "        df.sort_values(['QuestionID', col], inplace=True)\n",
    "        # add one Strictly Increasing Monotonically vector to avoid same ranks\n",
    "        b = np.ones(df.shape[0]).cumsum()\n",
    "        df[col] += b\n",
    "        grp = df.groupby('QuestionID')\n",
    "        df[col] = grp[col].rank(method='min', ascending=True).astype(int)  \n",
    "    df[save_cols].sort_values(sort_by_cols).to_csv(dst_fn, sep='\\t', header=False, index=False)    \n",
    "\n",
    "def resample(rt, Xs, Y):\n",
    "    if rt == None or rt == 0:\n",
    "        return Xs, Y\n",
    "    n = Y.shape[0]\n",
    "    n1 = Y.sum()\n",
    "    n0 = n - n1\n",
    "    cls_idx = np.arange(n)\n",
    "    pos_cls_idx = cls_idx[Y==1]\n",
    "    neg_cls_idx = cls_idx[Y==0]\n",
    "    pos_sample = np.random.choice(pos_cls_idx, size=int(n0 / float(rt)) - n1, )\n",
    "    new_cls_idx = np.concatenate([neg_cls_idx, pos_cls_idx, pos_sample])\n",
    "    np.random.shuffle(new_cls_idx)\n",
    "    print n0, n1, n, int(n0 / float(rt)) - n1, new_cls_idx.shape\n",
    "    new_Y = Y[new_cls_idx]\n",
    "    new_Xs = [X[new_cls_idx] for X in Xs]\n",
    "    return new_Xs, new_Y\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#ques_len.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#sent_len.hist()\n",
    "#sent_len[sent_len<200].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "sent_pad = dict(zip(sent_seq.keys(), pad_sequences(sent_seq.values(), maxlen=MAX_SENT_SEQUENCE_LENGTH, padding='post', truncating='post')))\n",
    "ques_pad = dict(zip(ques_seq.keys(), pad_sequences(ques_seq.values(), maxlen=MAX_QUES_SEQUENCE_LENGTH, padding='post', truncating='post')))\n",
    "ques_as_sent_pad = dict(zip(ques_seq.keys(), pad_sequences(ques_seq.values(), maxlen=MAX_SENT_SEQUENCE_LENGTH, padding='post', truncating='post')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sent_pad_df = DataFrame(sent_pad.values(), index=sent_pad.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ques_pad_df = DataFrame(ques_pad.values(), index=ques_pad.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ques_as_sent_pad_df = DataFrame(ques_as_sent_pad.values(), index=ques_as_sent_pad.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "paddata = collections.defaultdict(dict)\n",
    "for k in fn:\n",
    "    paddata[k] = {'ques':ques_pad_df.loc[data[k]['QuestionID']], \n",
    "                  'ques_as_sent':ques_as_sent_pad_df.loc[data[k]['QuestionID']],\n",
    "                  'sent':sent_pad_df.loc[data[k]['QAID']]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_output_class = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "label = {}\n",
    "for k in sfn:\n",
    "    if k == 'test':\n",
    "        continue\n",
    "    tmp = data[k]['Label'].values\n",
    "    if num_output_class == 1:\n",
    "        label[k] = tmp\n",
    "    elif num_output_class == 2:\n",
    "        label[k] = np.array([1 - tmp, tmp]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adapted CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "use_add_features = True\n",
    "num_output_class = 1\n",
    "features_used = dummy_rank_features\n",
    "add_features_dim = features_used['trn'].shape[1] if use_add_features else None\n",
    "X, Y = {}, {}\n",
    "for k in sfn:\n",
    "    X[k] = [paddata[k]['ques'].values, paddata[k]['sent'].values]\n",
    "    if k != 'test':\n",
    "        Y[k] = label[k]\n",
    "    if use_add_features:\n",
    "        X[k].append(features_used[k].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from MyCNN import *\n",
    "cnn = MyCNN(embedding_matrix, word_index, MAX_QUES_SEQUENCE_LENGTH, MAX_SENT_SEQUENCE_LENGTH, add_features_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cnn.init_model(num_filters=100, filter_size=5, pool_length=2, \n",
    "               denses=[100], dropouts=[0], activations=['relu'],\n",
    "               num_output_class=num_output_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20360 samples, validate on 1130 samples\n",
      "Epoch 1/2\n",
      "20360/20360 [==============================] - 13s - loss: 0.1842 - acc: 0.9474 - val_loss: 0.4068 - val_acc: 0.8761\n",
      "Epoch 2/2\n",
      "20360/20360 [==============================] - 13s - loss: 0.1741 - acc: 0.9483 - val_loss: 0.3135 - val_acc: 0.8770\n"
     ]
    }
   ],
   "source": [
    "#new_Xs, new_Y = resample(0, X['trn'], Y['trn'])\n",
    "cnn.fit(X['trn'], Y['trn'], validation_data=(X['dev'], Y['dev']), nb_epoch=2, batch_size=50) # data['trn'].shape[0]) # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev\n",
      "my_MAP \t0.681653212308\n",
      "my_MRR \t0.685950293093\n",
      "MAP \t0.681653212308\n",
      "MRR \t0.685950293093\n"
     ]
    }
   ],
   "source": [
    "res = {}\n",
    "for k in sfn:\n",
    "    if k in ['trn', 'test']:\n",
    "        continue\n",
    "    y = cnn.predict(X[k])\n",
    "    res[k] = rank_and_evaluate(y[:, 0], k)\n",
    "    print k\n",
    "    for m in ['my_MAP', 'my_MRR', 'MAP', 'MRR']:\n",
    "        print m, '\\t', res[k][m]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model.save('model/cnn_2d_%d_%f_%f.h5' % (add_features_dim, res['dev']['MRR'], res['testx']['MRR']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## adapted LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from MyLSTM import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare X and Y\n",
    "use_add_features = True\n",
    "features_used = dummy_rank_features # dummies \n",
    "add_features_dim = features_used['trn'].shape[1] if use_add_features else None\n",
    "X, Y = {}, {}\n",
    "for k in sfn:\n",
    "    X[k] = [paddata[k]['ques_as_sent'].values, paddata[k]['sent'].values]\n",
    "    if k != 'test':\n",
    "        Y[k] = label[k]\n",
    "    if use_add_features:\n",
    "        X[k].append(features_used[k].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = MyLSTM(embedding_matrix, word_index, MAX_SENT_SEQUENCE_LENGTH, add_features_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.init_model(lstm_output_dim=64, denses=[64], dropouts=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20360 samples, validate on 1130 samples\n",
      "Epoch 1/2\n",
      "20360/20360 [==============================] - 57s - loss: 0.1669 - acc: 0.9485 - val_loss: 0.3003 - val_acc: 0.8850\n",
      "Epoch 2/2\n",
      "20360/20360 [==============================] - 56s - loss: 0.1646 - acc: 0.9485 - val_loss: 0.3484 - val_acc: 0.8770\n"
     ]
    }
   ],
   "source": [
    "#np.random.seed(1337)\n",
    "model.fit(X['trn'], Y['trn'], validation_data=(X['dev'], Y['dev']), nb_epoch=2, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev\n",
      "my_MAP \t0.685823839594\n",
      "my_MRR \t0.691201126915\n",
      "MAP \t0.685823839594\n",
      "MRR \t0.691201126915\n"
     ]
    }
   ],
   "source": [
    "res = {}\n",
    "for k in sfn:\n",
    "    if k in ['trn', 'test']:\n",
    "        continue\n",
    "    y = model.predict(X[k])\n",
    "    res['y'] = y\n",
    "    res[k] = rank_and_evaluate(y, k)\n",
    "    print k\n",
    "    for m in ['my_MAP', 'my_MRR', 'MAP', 'MRR']:\n",
    "        print m, '\\t', res[k][m]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model.save('model/lstm_2d_%d_%f_%f.h5' % (add_features_dim, res['dev']['MRR'], res['testx']['MRR']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# from keras.models import load_model\n",
    "# model = load_model('model/lstm.1337.709692.663293.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## predict and save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "# predict on test dataset\n",
    "k = 'test'\n",
    "y = model.predict(X[k])\n",
    "tres = rank_and_evaluate(y, k)\n",
    "save_final_rank(tres['rank_df'], dst_fn='final_rank.txt', rank_col='rank', \n",
    "                    save_cols=['QuestionID', 'SentenceID', 'rank'], sort_by_cols=['QuestionID', 'rank'], re_rank=False)\n",
    "\n",
    "print 'Done'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}