Untitled11.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pyarrow.parquet as pq\n",
    "import os\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "import matplotlib.pyplot as plt\n",
    "#import tqdm\n",
    "from tqdm import tqdm_notebook\n",
    "%matplotlib inline \n",
    "\n",
    "EBINS=128\n",
    "MAX_ENERGY=2000\n",
    "ncomp_bg=8\n",
    "ncomp_bgs=10\n",
    "\n",
    "#MAX_ENERGY=2000\n",
    "#TSCALE_LIST07 = [0.064,0.125,0.25,0.5,1.0,4.0]\n",
    "TSCALE_LIST = [0.032,0.064,0.125,0.25,0.5,1.0,4.0]\n",
    "#BINS_PER_TSCALE = [512,256,192,128]\n",
    "BINS_PER_TSCALE = [128,128,128,128,128,128,128]\n",
    "#TWIN_PER_TSCALE7 = [1, 1, 1, 1, 3, 3]\n",
    "TWIN_PER_TSCALE = [1, 1, 1, 1, 1, 1, 3]\n",
    "TWIN = np.sum(TWIN_PER_TSCALE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_obj(fpath):\n",
    "    fd = open(fpath,\"rb\")\n",
    "    ret = pkl.load(fd)\n",
    "    fd.close()\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.read_csv(\"solution2.csv\")\n",
    "\n",
    "model_bg = load_obj(\"model_bg.pkl\")\n",
    "model_bgs = load_obj(\"model_bgs.pkl\")\n",
    "model_bgs1 = load_obj(\"model_bgs1.pkl\")\n",
    "classifier = load_obj(\"multiclass_tree.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b92a0266bf644bb8aada5f3c7b478e7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=15840), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hist_list = []\n",
    "for row in tqdm_notebook(submission.values):\n",
    "    runid = int(row[0])\n",
    "    source_id = int(row[1])\n",
    "    source_time = row[2]\n",
    "    \n",
    "    g_dat = pd.read_csv(os.path.join(\"testing\", \"%d.csv\" % runid), header=None)\n",
    "    g_dat = pd.concat([g_dat, g_dat])\n",
    "    g_dat = g_dat.reset_index()\n",
    "    \n",
    "    d0=g_dat[0]*1e-6\n",
    "    d1=np.cumsum(d0)\n",
    "    d2=g_dat[1]\n",
    "    \n",
    "    ebins = np.linspace(0,MAX_ENERGY,EBINS+1)\n",
    "    tmax = d1.values[-1]\n",
    "    \n",
    "    for (k, tscale) in enumerate(TSCALE_LIST):\n",
    "        invtscale=1/tscale\n",
    "        for j in range(TWIN_PER_TSCALE[k]):\n",
    "            if TWIN_PER_TSCALE[k] == 0:\n",
    "                continue\n",
    "            twinhalf = TWIN_PER_TSCALE[k]/2\n",
    "            dind = np.argwhere((d1 > source_time + (twinhalf - j - 1) * invtscale) & (d1 < source_time + (1 + twinhalf - j - 1) * invtscale)).flatten()\n",
    "            d3 = d2[dind]\n",
    "            hist = np.histogram(d3, bins=ebins)[0]\n",
    "            hist_list.append(hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "vlist_source_b1 = model_bg.transform(hist_list)\n",
    "vlist_source_s1 = model_bgs.transform(hist_list)\n",
    "\n",
    "fit_bg = np.dot(vlist_source_b1, model_bg.components_)\n",
    "diff_fit_bg = np.abs(fit_bg - hist_list)\n",
    "\n",
    "fit_bgs = np.dot(vlist_source_s1, model_bgs.components_)\n",
    "diff_fit_bgs = np.abs(fit_bgs - hist_list)\n",
    "\n",
    "BINS = 64\n",
    "div_fit_bg = np.linalg.norm(diff_fit_bg[:,:BINS],axis=1) / np.linalg.norm(diff_fit_bgs[:,:BINS],axis=1)\n",
    "\n",
    "hist_list1 = model_bgs1.transform(hist_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "div_fit_bg = div_fit_bg.reshape(-1, TWIN)\n",
    "hist_list1 = hist_list1.reshape(-1, (ncomp_bg+ncomp_bgs)*TWIN)\n",
    "hist_list1 = np.hstack([hist_list1, div_fit_bg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alexey/.local/lib/python3.5/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "pred_list1 = classifier.predict(hist_list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 6, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_list1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.hist(pred_list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "lst = []\n",
    "for (i,row) in enumerate(submission.values):\n",
    "    runid = int(row[0])\n",
    "    source_id = pred_list1[i]\n",
    "    source_time = row[2]\n",
    "    \n",
    "    #source_id = idx\n",
    "    #source_time = row[2]\n",
    "    \n",
    "    #idx = np.argmax(pred_list1[i])\n",
    "    #if idx == 0 or pred_list1[i][idx] > 0.7:\n",
    "     #   source_id = idx\n",
    "      #  if source_id == 0:\n",
    "       #     source_time = 0\n",
    "        #lst.append([int(runid), source_id, source_time])\n",
    "    #else:\n",
    "     #   lst.append([int(runid), int(0), 0.00])\n",
    "    \n",
    "    #idx = np.argmax(pred_list1[i])\n",
    "    #source_id = int(row[1])\n",
    "    #if pred_list1[i][idx] > 0.2:\n",
    "#        source_id = idx\n",
    "#        source_time = row[2]\n",
    "    if source_id == 0:\n",
    "        lst.append([int(runid), int(0), 0.00])\n",
    "    else:\n",
    "        lst.append([int(runid), source_id, source_time])\n",
    "dframe = pd.DataFrame(data=lst,columns=[\"RunID\",\"SourceID\",\"SourceTime\"])\n",
    "dframe[\"RunID\"] = dframe[\"RunID\"].astype(np.int64)\n",
    "dframe[\"SourceID\"] = dframe[\"SourceID\"].astype(np.int64)\n",
    "dframe.set_index([\"RunID\"],inplace=True)\n",
    "dframe.to_csv(\"solution4.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}