champs_scalar_coupling_h2o4gpu_dt.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introduction\n",
    "\n",
    "This notebook demonstrates how you can take advantage of [H2O4GPU](https://github.com/h2oai/h2o4gpu) and [Datatable](https://github.com/h2oai/datatable) to speed up model training.\n",
    "\n",
    "It uses Kaggle Competation Data from [Predicting Molecular Properties](https://www.kaggle.com/c/champs-scalar-coupling)\n",
    "\n",
    "A special weighting strategy is performed to be as close as possible to the actual challenge evaluation metric.\n",
    "\n",
    "The score on a 5 fold CV is and LB score is \n",
    "\n",
    "Please see readme file for installation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import h2o4gpu as sklearn\n",
    "from h2o4gpu.model_selection import GroupKFold\n",
    "from h2o4gpu.metrics import mean_absolute_error\n",
    "from h2o4gpu.preprocessing import LabelEncoder\n",
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from progressbar import progressbar\n",
    "\n",
    "import datatable as dt\n",
    "import gc\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_mean_log_mae(y_true, y_pred, mol_types, floor=1e-9):\n",
    "    \n",
    "    maes = []\n",
    "    for aType in sorted(mol_types.unique()):\n",
    "        #print(aType)\n",
    "        the_mean = np.mean(np.abs(y_true.loc[mol_types==aType] - y_pred.loc[mol_types==aType]))\n",
    "        #print(max(floor, the_mean))\n",
    "        maes.append(max(floor, the_mean))\n",
    "    print(np.mean(maes), np.log(maes).mean())\n",
    "    return np.log(maes).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../input/\"\n",
    "train = dt.fread(path + \"train.csv\")\n",
    "test = dt.fread(path + \"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool { background: #DDDD99; }\n",
       ".datatable .obj  { background: #565656; }\n",
       ".datatable .int  { background: #5D9E5D; }\n",
       ".datatable .real { background: #4040CC; }\n",
       ".datatable .str  { background: #CC4040; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n",
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>id</th><th>molecule_name</th><th>atom_index_0</th><th>atom_index_1</th><th>type</th><th>scalar_coupling_constant</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>0</td><td>dsgdb9nsd_000001</td><td>1</td><td>0</td><td>1JHC</td><td>84.8076</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>1</td><td>dsgdb9nsd_000001</td><td>1</td><td>2</td><td>2JHH</td><td>&minus;11.257</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>2</td><td>dsgdb9nsd_000001</td><td>1</td><td>3</td><td>2JHH</td><td>&minus;11.2548</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>3</td><td>dsgdb9nsd_000001</td><td>1</td><td>4</td><td>2JHH</td><td>&minus;11.2543</td></tr>\n",
       "    <tr><td class='row_index'>4</td><td>4</td><td>dsgdb9nsd_000001</td><td>2</td><td>0</td><td>1JHC</td><td>84.8074</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>5 rows &times; 6 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": []
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2358657, 6)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures = dt.fread(path + \"structures.csv\")\n",
    "structures.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>molecule_name</th><th>atom_index</th><th>atom</th><th>x</th><th>y</th><th>z</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>dsgdb9nsd_000002</td><td>0</td><td>N</td><td>&minus;0.0404261</td><td>1.02411</td><td>0.0625638</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>dsgdb9nsd_000002</td><td>1</td><td>H</td><td>0.0172575</td><td>0.0125452</td><td>&minus;0.0273772</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>dsgdb9nsd_000002</td><td>2</td><td>H</td><td>0.915789</td><td>1.35875</td><td>&minus;0.0287578</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>dsgdb9nsd_000002</td><td>3</td><td>H</td><td>&minus;0.520278</td><td>1.34353</td><td>&minus;0.775543</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>4 rows &times; 6 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": []
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures[dt.f.molecule_name == \"dsgdb9nsd_000002\", :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Electronegativity and Radius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Atomatic Radius\n",
      "{'H': 0.43, 'C': 0.8200000000000001, 'N': 0.8, 'O': 0.78, 'F': 0.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (2358657 of 2358657) |##############| Elapsed Time: 0:00:02 Time:  0:00:02\n",
      "100% (2358657 of 2358657) |##############| Elapsed Time: 0:00:02 Time:  0:00:02\n"
     ]
    }
   ],
   "source": [
    "# Copied From https://www.kaggle.com/adrianoavelar/bond-calculaltion-lb-0-82\n",
    "# Credits to https://www.kaggle.com/adrianoavelar\n",
    "\n",
    "atomic_radius = {'H':0.38, 'C':0.77, 'N':0.75, 'O':0.73, 'F':0.71} # Without fudge factor\n",
    "fudge_factor = 0.05\n",
    "atomic_radius = {k:v + fudge_factor for k,v in atomic_radius.items()}\n",
    "print(\"Atomatic Radius\")\n",
    "print(atomic_radius, flush=True)\n",
    "\n",
    "\n",
    "electronegativity = {'H':2.2, 'C':2.55, 'N':3.04, 'O':3.44, 'F':3.98}\n",
    "atoms = structures[:, 'atom'].to_numpy()[:, 0]\n",
    "atoms_en = [electronegativity[x] for x in progressbar(atoms)]\n",
    "atoms_rad = [atomic_radius[x] for x in progressbar(atoms)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "structures.cbind(dt.Frame({\"EN\": atoms_en, \"rad\": atoms_rad}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bond calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating the bonds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (27 of 27) |########################| Elapsed Time: 0:00:06 Time:  0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counting and condensing bonds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (2358657 of 2358657) |##############| Elapsed Time: 0:00:08 Time:  0:00:08\n",
      "100% (2358657 of 2358657) |##############| Elapsed Time: 0:00:10 Time:  0:00:10\n",
      "100% (2358657 of 2358657) |##############| Elapsed Time: 0:00:24 Time:  0:00:24\n",
      "100% (2358657 of 2358657) |##############| Elapsed Time: 0:00:02 Time:  0:00:02\n"
     ]
    }
   ],
   "source": [
    "# Copied From https://www.kaggle.com/adrianoavelar/bond-calculaltion-lb-0-82\n",
    "# Credits to https://www.kaggle.com/adrianoavelar\n",
    "\n",
    "i_atom = structures[:, 'atom_index'].to_numpy()[:, 0]\n",
    "p = structures[:, ['x', 'y', 'z']].to_numpy()\n",
    "p_compare = p\n",
    "m = structures[:, 'molecule_name'].to_numpy()[:, 0]\n",
    "m_compare = m\n",
    "r = structures[:, 'rad'].to_numpy()[:, 0]\n",
    "r_compare = r\n",
    "\n",
    "source_row = np.arange(structures.shape[0])\n",
    "max_atoms = 28\n",
    "\n",
    "bonds = np.zeros((structures.shape[0]+1, max_atoms+1), dtype=np.int8)\n",
    "bond_dists = np.zeros((structures.shape[0]+1, max_atoms+1), dtype=np.float32)\n",
    "\n",
    "print('Calculating the bonds', flush=True)\n",
    "\n",
    "for i in progressbar(range(max_atoms-1)):\n",
    "    p_compare = np.roll(p_compare, -1, axis=0)\n",
    "    m_compare = np.roll(m_compare, -1, axis=0)\n",
    "    r_compare = np.roll(r_compare, -1, axis=0)\n",
    "    \n",
    "    mask = np.where(m == m_compare, 1, 0) #Are we still comparing atoms in the same molecule?\n",
    "    dists = np.linalg.norm(p - p_compare, axis=1) * mask\n",
    "    r_bond = r + r_compare\n",
    "    \n",
    "    bond = np.where(np.logical_and(dists > 0.0001, dists < r_bond), 1, 0)\n",
    "    \n",
    "    source_row = source_row\n",
    "    target_row = source_row + i + 1 #Note: Will be out of bounds of bonds array for some values of i\n",
    "    target_row = np.where(np.logical_or(target_row > structures.shape[0], mask==0), structures.shape[0], target_row) #If invalid target, write to dummy row\n",
    "    \n",
    "    source_atom = i_atom\n",
    "    target_atom = i_atom + i + 1 #Note: Will be out of bounds of bonds array for some values of i\n",
    "    target_atom = np.where(np.logical_or(target_atom > max_atoms, mask==0), max_atoms, target_atom) #If invalid target, write to dummy col\n",
    "    \n",
    "    bonds[(source_row, target_atom)] = bond\n",
    "    bonds[(target_row, source_atom)] = bond\n",
    "    bond_dists[(source_row, target_atom)] = dists\n",
    "    bond_dists[(target_row, source_atom)] = dists\n",
    "\n",
    "bonds = np.delete(bonds, axis=0, obj=-1) #Delete dummy row\n",
    "bonds = np.delete(bonds, axis=1, obj=-1) #Delete dummy col\n",
    "bond_dists = np.delete(bond_dists, axis=0, obj=-1) #Delete dummy row\n",
    "bond_dists = np.delete(bond_dists, axis=1, obj=-1) #Delete dummy col\n",
    "\n",
    "print('Counting and condensing bonds', flush=True)\n",
    "\n",
    "bonds_numeric = [[i for i,x in enumerate(row) if x] for row in progressbar(bonds)]\n",
    "bond_lengths = [[dist for i,dist in enumerate(row) if i in bonds_numeric[j]] for j,row in enumerate(progressbar(bond_dists))]\n",
    "# Note that datatable does not support numpy types, so need to cast to python float\n",
    "bond_lengths_mean = [float(np.mean(x)) for x in progressbar(bond_lengths)]\n",
    "n_bonds = [len(x) for x in progressbar(bonds_numeric)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>molecule_name</th><th>atom_index</th><th>atom</th><th>x</th><th>y</th><th>z</th><th>EN</th><th>rad</th><th>n_bonds</th><th>bond_lengths_mean</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int8'>&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>dsgdb9nsd_000001</td><td>0</td><td>C</td><td>&minus;0.0126981</td><td>1.0858</td><td>0.008001</td><td>2.55</td><td>0.82</td><td>4</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>dsgdb9nsd_000001</td><td>1</td><td>H</td><td>0.00215042</td><td>&minus;0.00603132</td><td>0.00197612</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>dsgdb9nsd_000001</td><td>2</td><td>H</td><td>1.01173</td><td>1.46375</td><td>0.000276575</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>dsgdb9nsd_000001</td><td>3</td><td>H</td><td>&minus;0.540815</td><td>1.44753</td><td>&minus;0.876644</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>4</td><td>dsgdb9nsd_000001</td><td>4</td><td>H</td><td>&minus;0.523814</td><td>1.43793</td><td>0.906397</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>5</td><td>dsgdb9nsd_000002</td><td>0</td><td>N</td><td>&minus;0.0404261</td><td>1.02411</td><td>0.0625638</td><td>3.04</td><td>0.8</td><td>3</td><td>1.01719</td></tr>\n",
       "    <tr><td class='row_index'>6</td><td>dsgdb9nsd_000002</td><td>1</td><td>H</td><td>0.0172575</td><td>0.0125452</td><td>&minus;0.0273772</td><td>2.2</td><td>0.43</td><td>1</td><td>1.01719</td></tr>\n",
       "    <tr><td class='row_index'>7</td><td>dsgdb9nsd_000002</td><td>2</td><td>H</td><td>0.915789</td><td>1.35875</td><td>&minus;0.0287578</td><td>2.2</td><td>0.43</td><td>1</td><td>1.01719</td></tr>\n",
       "    <tr><td class='row_index'>8</td><td>dsgdb9nsd_000002</td><td>3</td><td>H</td><td>&minus;0.520278</td><td>1.34353</td><td>&minus;0.775543</td><td>2.2</td><td>0.43</td><td>1</td><td>1.01721</td></tr>\n",
       "    <tr><td class='row_index'>9</td><td>dsgdb9nsd_000003</td><td>0</td><td>O</td><td>&minus;0.0343605</td><td>0.97754</td><td>0.00760159</td><td>3.44</td><td>0.78</td><td>2</td><td>0.962107</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>10 rows &times; 10 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": []
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures.cbind(dt.Frame({'n_bonds':n_bonds, 'bond_lengths_mean': bond_lengths_mean}))\n",
    "structures.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge with train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>id</th><th>molecule_name</th><th>atom_index_0</th><th>atom_index_1</th><th>type</th><th>scalar_coupling_constant</th><th>atom_0</th><th>x_0</th><th>y_0</th><th>z_0</th><th class='vellipsis'>&hellip;</th><th>z_1</th><th>EN_1</th><th>rad_1</th><th>n_bonds_1</th><th>bond_lengths_mean_1</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td></td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int8'>&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>0</td><td>dsgdb9nsd_000001</td><td>1</td><td>0</td><td>1JHC</td><td>84.8076</td><td>H</td><td>0.00215042</td><td>&minus;0.00603132</td><td>0.00197612</td><td class=vellipsis>&hellip;</td><td>0.008001</td><td>2.55</td><td>0.82</td><td>4</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>1</td><td>dsgdb9nsd_000001</td><td>1</td><td>2</td><td>2JHH</td><td>&minus;11.257</td><td>H</td><td>0.00215042</td><td>&minus;0.00603132</td><td>0.00197612</td><td class=vellipsis>&hellip;</td><td>0.000276575</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>2</td><td>dsgdb9nsd_000001</td><td>1</td><td>3</td><td>2JHH</td><td>&minus;11.2548</td><td>H</td><td>0.00215042</td><td>&minus;0.00603132</td><td>0.00197612</td><td class=vellipsis>&hellip;</td><td>&minus;0.876644</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>3</td><td>dsgdb9nsd_000001</td><td>1</td><td>4</td><td>2JHH</td><td>&minus;11.2543</td><td>H</td><td>0.00215042</td><td>&minus;0.00603132</td><td>0.00197612</td><td class=vellipsis>&hellip;</td><td>0.906397</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>4</td><td>4</td><td>dsgdb9nsd_000001</td><td>2</td><td>0</td><td>1JHC</td><td>84.8074</td><td>H</td><td>1.01173</td><td>1.46375</td><td>0.000276575</td><td class=vellipsis>&hellip;</td><td>0.008001</td><td>2.55</td><td>0.82</td><td>4</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>5</td><td>5</td><td>dsgdb9nsd_000001</td><td>2</td><td>3</td><td>2JHH</td><td>&minus;11.2541</td><td>H</td><td>1.01173</td><td>1.46375</td><td>0.000276575</td><td class=vellipsis>&hellip;</td><td>&minus;0.876644</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>6</td><td>6</td><td>dsgdb9nsd_000001</td><td>2</td><td>4</td><td>2JHH</td><td>&minus;11.2548</td><td>H</td><td>1.01173</td><td>1.46375</td><td>0.000276575</td><td class=vellipsis>&hellip;</td><td>0.906397</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>7</td><td>7</td><td>dsgdb9nsd_000001</td><td>3</td><td>0</td><td>1JHC</td><td>84.8093</td><td>H</td><td>&minus;0.540815</td><td>1.44753</td><td>&minus;0.876644</td><td class=vellipsis>&hellip;</td><td>0.008001</td><td>2.55</td><td>0.82</td><td>4</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>8</td><td>8</td><td>dsgdb9nsd_000001</td><td>3</td><td>4</td><td>2JHH</td><td>&minus;11.2543</td><td>H</td><td>&minus;0.540815</td><td>1.44753</td><td>&minus;0.876644</td><td class=vellipsis>&hellip;</td><td>0.906397</td><td>2.2</td><td>0.43</td><td>1</td><td>1.09195</td></tr>\n",
       "    <tr><td class='row_index'>9</td><td>9</td><td>dsgdb9nsd_000001</td><td>4</td><td>0</td><td>1JHC</td><td>84.8095</td><td>H</td><td>&minus;0.523814</td><td>1.43793</td><td>0.906397</td><td class=vellipsis>&hellip;</td><td>0.008001</td><td>2.55</td><td>0.82</td><td>4</td><td>1.09195</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>10 rows &times; 22 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": []
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_names = structures.names\n",
    "\n",
    "structures.names = [_f + \"_0\" if _f != \"molecule_name\" else _f for _f in col_names]\n",
    "structures.key = [\"molecule_name\", \"atom_index_0\"]\n",
    "train = train[:, :, dt.join(structures)]\n",
    "test = test[:, :, dt.join(structures)]\n",
    "\n",
    "structures.names = [_f + \"_1\" if _f != \"molecule_name\" else _f for _f in col_names]\n",
    "structures.key = [\"molecule_name\", \"atom_index_1\"]\n",
    "train = train[:, :, dt.join(structures)]\n",
    "test = test[:, :, dt.join(structures)]\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>id</th><th>molecule_name</th><th>atom_index_0</th><th>atom_index_1</th><th>type</th><th>scalar_coupling_constant</th><th>atom_0</th><th>x_0</th><th>y_0</th><th>z_0</th><th class='vellipsis'>&hellip;</th><th>z_1</th><th>EN_1</th><th>rad_1</th><th>n_bonds_1</th><th>bond_lengths_mean_1</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td></td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int8'>&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>10</td><td>dsgdb9nsd_000002</td><td>1</td><td>0</td><td>1JHN</td><td>32.6889</td><td>H</td><td>0.0172575</td><td>0.0125452</td><td>&minus;0.0273772</td><td class=vellipsis>&hellip;</td><td>0.0625638</td><td>3.04</td><td>0.8</td><td>3</td><td>1.01719</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>11</td><td>dsgdb9nsd_000002</td><td>1</td><td>2</td><td>2JHH</td><td>&minus;11.1866</td><td>H</td><td>0.0172575</td><td>0.0125452</td><td>&minus;0.0273772</td><td class=vellipsis>&hellip;</td><td>&minus;0.0287578</td><td>2.2</td><td>0.43</td><td>1</td><td>1.01719</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>12</td><td>dsgdb9nsd_000002</td><td>1</td><td>3</td><td>2JHH</td><td>&minus;11.1757</td><td>H</td><td>0.0172575</td><td>0.0125452</td><td>&minus;0.0273772</td><td class=vellipsis>&hellip;</td><td>&minus;0.775543</td><td>2.2</td><td>0.43</td><td>1</td><td>1.01721</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>13</td><td>dsgdb9nsd_000002</td><td>2</td><td>0</td><td>1JHN</td><td>32.6891</td><td>H</td><td>0.915789</td><td>1.35875</td><td>&minus;0.0287578</td><td class=vellipsis>&hellip;</td><td>0.0625638</td><td>3.04</td><td>0.8</td><td>3</td><td>1.01719</td></tr>\n",
       "    <tr><td class='row_index'>4</td><td>14</td><td>dsgdb9nsd_000002</td><td>2</td><td>3</td><td>2JHH</td><td>&minus;11.1758</td><td>H</td><td>0.915789</td><td>1.35875</td><td>&minus;0.0287578</td><td class=vellipsis>&hellip;</td><td>&minus;0.775543</td><td>2.2</td><td>0.43</td><td>1</td><td>1.01721</td></tr>\n",
       "    <tr><td class='row_index'>5</td><td>15</td><td>dsgdb9nsd_000002</td><td>3</td><td>0</td><td>1JHN</td><td>32.6905</td><td>H</td><td>&minus;0.520278</td><td>1.34353</td><td>&minus;0.775543</td><td class=vellipsis>&hellip;</td><td>0.0625638</td><td>3.04</td><td>0.8</td><td>3</td><td>1.01719</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>6 rows &times; 22 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": []
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[dt.f.molecule_name == \"dsgdb9nsd_000002\", :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to compute value_counts ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>atom_0</th><th>C0</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>H</td><td>4658147</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>1 row &times; 2 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": []
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[:, dt.count(), dt.by(\"atom_0\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>atom_1</th><th>C0</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>C</td><td>3360469</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>H</td><td>968647</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>N</td><td>329031</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>3 rows &times; 2 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": []
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[:, dt.count(), dt.by(\"atom_1\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple Feature Engineering\n",
    "\n",
    "#### L1 distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [\"x\", \"y\", \"z\"]\n",
    "train.cbind(dt.Frame(\n",
    "    {\n",
    "        \"l1_distance\": np.sum(\n",
    "            np.abs(train[:, [f+\"_0\" for f in cols]].to_numpy() \n",
    "                   - train[:, [f+\"_1\" for f in cols]].to_numpy()), axis=1)\n",
    "    }\n",
    "))\n",
    "            \n",
    "test.cbind(dt.Frame(\n",
    "    {\n",
    "        \"l1_distance\": np.sum(\n",
    "            np.abs(test[:, [f+\"_0\" for f in cols]].to_numpy() \n",
    "                   - test[:, [f+\"_1\" for f in cols]].to_numpy()), axis=1)\n",
    "    }\n",
    "))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### L2 distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [\"x\", \"y\", \"z\"]\n",
    "train.cbind(dt.Frame(\n",
    "    {\n",
    "        \"l2_distance\": np.sum(\n",
    "            (train[:, [f+\"_0\" for f in cols]].to_numpy() \n",
    "             - train[:, [f+\"_1\" for f in cols]].to_numpy()) ** 2, axis=1)\n",
    "    }\n",
    "))\n",
    "            \n",
    "test.cbind(dt.Frame(\n",
    "    {\n",
    "        \"l2_distance\": np.sum(\n",
    "            (test[:, [f+\"_0\" for f in cols]].to_numpy() \n",
    "             - test[:, [f+\"_1\" for f in cols]].to_numpy()) ** 2, axis=1)\n",
    "    }\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Number of links in the molecules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_nb_links = train[:, dt.count(), dt.by(\"molecule_name\")]\n",
    "trn_nb_links.names = [\"molecule_name\", \"nb_links\"]\n",
    "trn_nb_links.key = \"molecule_name\"\n",
    "train = train[:, :, dt.join(trn_nb_links)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tst_nb_links = test[:, dt.count(), dt.by(\"molecule_name\")]\n",
    "tst_nb_links.names = [\"molecule_name\", \"nb_links\"]\n",
    "tst_nb_links.key = \"molecule_name\"\n",
    "test = test[:, :, dt.join(tst_nb_links)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding Categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['atom_index_0', 'atom_index_1', 'x_0', 'y_0', 'z_0', 'EN_0',\n",
       "       'rad_0', 'n_bonds_0', 'bond_lengths_mean_0', 'atom_1', 'x_1',\n",
       "       'y_1', 'z_1', 'EN_1', 'rad_1', 'n_bonds_1', 'bond_lengths_mean_1',\n",
       "       'l1_distance', 'l2_distance', 'nb_links'], dtype='<U19')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = [f for f in train.names if f not in [\"scalar_coupling_constant\", \"id\", \"molecule_name\", \"type\", \"predictions\", \"atom_0\"]]\n",
    "np.array(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['atom_1']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# One of the coolest thing about DataTable ;-)\n",
    "# You can select features on their type like train[:, [int, float]]\n",
    "cat_feats = [f for f in train[:, str].names if f not in [\"molecule_name\", \"type\", \"atom_0\"]]\n",
    "cat_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "lbl_encoders = {}\n",
    "for f in cat_feats:\n",
    "    lbl_encoders[f] = pd.Index(dt.unique(train[:, f]).to_numpy()[:, 0])\n",
    "    train.cbind(dt.Frame({f+\"_lbl_enc\": lbl_encoders[f].get_indexer(train[:, f].to_numpy()[:, 0])}))\n",
    "    test.cbind(dt.Frame({f+\"_lbl_enc\": lbl_encoders[f].get_indexer(test[:, f].to_numpy()[:, 0])}))\n",
    "    features.remove(f)\n",
    "    features.append(f+\"_lbl_enc\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build a general model\n",
    "\n",
    "We use weights to optimize for average MAE over the molecule types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_mean_log_mae_dt(y_true, y_pred, mol_types, floor=1e-9):\n",
    "    \n",
    "    maes = []\n",
    "    np_types = mol_types.to_numpy()[:, 0] \n",
    "    np_true = y_true.to_numpy()[:, 0]\n",
    "\n",
    "    for aType in sorted(sorted(dt.unique(mol_types).to_list()[0])):\n",
    "        the_mean = np.mean(np.abs(np_true[np_types==aType] - y_pred[np_types==aType]))\n",
    "        maes.append(max(floor, the_mean))\n",
    "        \n",
    "    print(\"Mean of Type MAE : %.4f | Eval Metric : %.4f\" % (np.mean(maes), np.log(maes).mean()))\n",
    "    \n",
    "    return np.log(maes).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-mae:10.813\n",
      "Will train until validation_0-mae hasn't improved in 100 rounds.\n",
      "[500]\tvalidation_0-mae:1.0762\n",
      "[1000]\tvalidation_0-mae:1.04392\n",
      "[1500]\tvalidation_0-mae:1.02914\n",
      "[2000]\tvalidation_0-mae:1.0203\n",
      "[2500]\tvalidation_0-mae:1.01421\n",
      "[3000]\tvalidation_0-mae:1.0102\n",
      "[3500]\tvalidation_0-mae:1.00676\n",
      "[4000]\tvalidation_0-mae:1.00429\n",
      "[4500]\tvalidation_0-mae:1.00196\n",
      "[4999]\tvalidation_0-mae:1.00024\n",
      "Mean of Type MAE : 1.0002 | Eval Metric : -0.1025\n",
      "MEAN LOG MAE SCORE : -0.102507\n",
      "[0]\tvalidation_0-mae:10.8014\n",
      "Will train until validation_0-mae hasn't improved in 100 rounds.\n",
      "[500]\tvalidation_0-mae:1.07225\n",
      "[1000]\tvalidation_0-mae:1.04\n",
      "[1500]\tvalidation_0-mae:1.02485\n",
      "[2000]\tvalidation_0-mae:1.01514\n",
      "[2500]\tvalidation_0-mae:1.00973\n",
      "[3000]\tvalidation_0-mae:1.00507\n",
      "[3500]\tvalidation_0-mae:1.00157\n",
      "[4000]\tvalidation_0-mae:0.998908\n",
      "[4500]\tvalidation_0-mae:0.996846\n",
      "[4999]\tvalidation_0-mae:0.994942\n",
      "Mean of Type MAE : 0.9949 | Eval Metric : -0.1072\n",
      "MEAN LOG MAE SCORE : -0.107164\n",
      "[0]\tvalidation_0-mae:10.8207\n",
      "Will train until validation_0-mae hasn't improved in 100 rounds.\n",
      "[500]\tvalidation_0-mae:1.07368\n",
      "[1000]\tvalidation_0-mae:1.04099\n",
      "[1500]\tvalidation_0-mae:1.02519\n",
      "[2000]\tvalidation_0-mae:1.01563\n",
      "[2500]\tvalidation_0-mae:1.00923\n",
      "[3000]\tvalidation_0-mae:1.0047\n",
      "[3500]\tvalidation_0-mae:1.00102\n",
      "[4000]\tvalidation_0-mae:0.998379\n",
      "[4500]\tvalidation_0-mae:0.996392\n",
      "[4999]\tvalidation_0-mae:0.99493\n",
      "Mean of Type MAE : 0.9949 | Eval Metric : -0.1083\n",
      "MEAN LOG MAE SCORE : -0.108283\n",
      "[0]\tvalidation_0-mae:10.8408\n",
      "Will train until validation_0-mae hasn't improved in 100 rounds.\n",
      "[500]\tvalidation_0-mae:1.07912\n",
      "[1000]\tvalidation_0-mae:1.04728\n",
      "[1500]\tvalidation_0-mae:1.03195\n",
      "[2000]\tvalidation_0-mae:1.02233\n",
      "[2500]\tvalidation_0-mae:1.016\n",
      "[3000]\tvalidation_0-mae:1.01165\n",
      "[3500]\tvalidation_0-mae:1.0082\n",
      "[4000]\tvalidation_0-mae:1.00551\n",
      "[4500]\tvalidation_0-mae:1.00308\n",
      "[4999]\tvalidation_0-mae:1.00107\n",
      "Mean of Type MAE : 1.0011 | Eval Metric : -0.0998\n",
      "MEAN LOG MAE SCORE : -0.099760\n",
      "[0]\tvalidation_0-mae:10.8166\n",
      "Will train until validation_0-mae hasn't improved in 100 rounds.\n",
      "[500]\tvalidation_0-mae:1.07356\n",
      "[1000]\tvalidation_0-mae:1.03998\n",
      "[1500]\tvalidation_0-mae:1.0252\n",
      "[2000]\tvalidation_0-mae:1.01628\n",
      "[2500]\tvalidation_0-mae:1.01022\n",
      "[3000]\tvalidation_0-mae:1.006\n",
      "[3500]\tvalidation_0-mae:1.00267\n",
      "[4000]\tvalidation_0-mae:1.00013\n",
      "[4500]\tvalidation_0-mae:0.997893\n",
      "[4999]\tvalidation_0-mae:0.996085\n",
      "Mean of Type MAE : 0.9961 | Eval Metric : -0.1062\n",
      "MEAN LOG MAE SCORE : -0.106227\n"
     ]
    }
   ],
   "source": [
    "target = \"scalar_coupling_constant\"\n",
    "oof_preds = np.zeros(train.shape[0])    \n",
    "models = []\n",
    "\n",
    "for trn_, val_ in GroupKFold(5).split(X=train[:, \"type\"].to_numpy(), y=None, groups=train[:, \"molecule_name\"].to_numpy()[:, 0]):\n",
    "    trn_type_sizes = train[trn_, :][:, dt.count(), dt.by(\"type\")]\n",
    "    val_type_sizes = train[val_, :][:, dt.count(), dt.by(\"type\")]\n",
    "    \n",
    "    trn_type_sizes[:, \"C0\"] = trn_type_sizes[:, trn_.shape[0] / (dt.f.C0 * trn_type_sizes.shape[0])]\n",
    "    val_type_sizes[:, \"C0\"] = val_type_sizes[:, val_.shape[0] / (dt.f.C0 * val_type_sizes.shape[0])]\n",
    "    \n",
    "    trn_type_sizes.key = [\"type\"]\n",
    "    trn_weights = train[trn_, \"type\"][:, :, dt.join(trn_type_sizes)][:, \"C0\"].to_numpy()[:,0]\n",
    "    val_type_sizes.key = [\"type\"]\n",
    "    val_weights = train[val_, \"type\"][:, :, dt.join(val_type_sizes)][:, \"C0\"].to_numpy()[:,0]\n",
    "    \n",
    "    model = xgb.XGBRegressor(\n",
    "        tree_method=\"gpu_hist\",\n",
    "        objective=\"reg:squarederror\",\n",
    "        n_estimators=5000, \n",
    "        learning_rate=.5,\n",
    "        max_depth=9,\n",
    "        sub_sample=.8,\n",
    "        colsample_bytree=.8,\n",
    "        reg_alpha=10,\n",
    "        reg_lambda=5,\n",
    "        n_jobs=-1,\n",
    "        eval_metric=\"mae\"\n",
    "    )\n",
    "    \n",
    "    # Need to materialize views before sending to XGBoost\n",
    "    trn_X = train[trn_, :]\n",
    "    trn_X.materialize()\n",
    "    val_X = train[val_, :]\n",
    "    val_X.materialize()\n",
    "    \n",
    "    model.fit(\n",
    "        trn_X[:, features], trn_X[:, target], sample_weight=trn_weights,\n",
    "        eval_set=[(val_X[:, features], val_X[:, target])],\n",
    "        sample_weight_eval_set=[val_weights],\n",
    "        early_stopping_rounds=100,\n",
    "        verbose=500,\n",
    "    )\n",
    "    \n",
    "    oof_preds[val_] = model.predict(val_X[:, features])\n",
    "    \n",
    "    models.append(model)\n",
    "        \n",
    "    print(\"MEAN LOG MAE SCORE : %.6f\" \n",
    "          % group_mean_log_mae_dt(\n",
    "            y_true=val_X[:, target], \n",
    "            y_pred=oof_preds[val_], \n",
    "            mol_types=val_X[:, \"type\"], \n",
    "            floor=1e-9\n",
    "        ))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of Type MAE : 0.9974 | Eval Metric : -0.1048\n",
      "MEAN LOG MAE SCORE : -0.104776\n"
     ]
    }
   ],
   "source": [
    "print(\"MEAN LOG MAE SCORE : %.6f\" \n",
    "  % group_mean_log_mae_dt(\n",
    "    y_true=train[:, target], \n",
    "    y_pred=oof_preds, \n",
    "    mol_types=train[:, \"type\"], \n",
    "    floor=1e-9\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict Test target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_preds = np.zeros(test.shape[0])\n",
    "test.materialize()\n",
    "\n",
    "for model in models:\n",
    "    sub_preds += model.predict(test[:, features]) / len(models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submit files to Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>id</th><th>scalar_coupling_constant</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='real' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>4658147</td><td>15.6552</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>4658148</td><td>192.717</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>4658149</td><td>0.168881</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>4658150</td><td>193.485</td></tr>\n",
       "    <tr><td class='row_index'>4</td><td>4658151</td><td>18.0486</td></tr>\n",
       "    <tr><td class='row_index'>5</td><td>4658152</td><td>91.3877</td></tr>\n",
       "    <tr><td class='row_index'>6</td><td>4658153</td><td>1.36821</td></tr>\n",
       "    <tr><td class='row_index'>7</td><td>4658154</td><td>&minus;6.93436</td></tr>\n",
       "    <tr><td class='row_index'>8</td><td>4658155</td><td>&minus;8.74346</td></tr>\n",
       "    <tr><td class='row_index'>9</td><td>4658156</td><td>92.1111</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>10 rows &times; 2 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": []
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_dt = test[:, \"id\"].copy()\n",
    "preds_dt.cbind(dt.Frame({\"scalar_coupling_constant\": sub_preds}))\n",
    "preds_dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_dt.to_csv(\"champs_scalar_counpling_h2o4gpu_datatable.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████| 64.0M/64.0M [00:06<00:00, 10.6MB/s]\n",
      "Successfully submitted to Predicting Molecular Properties"
     ]
    }
   ],
   "source": [
    "!kaggle competitions submit -f champs_scalar_counpling_h2o4gpu_datatable.csv -m \"predictions with h2o4gpu and datatable\" champs-scalar-coupling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}