PreProcessingMerge.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finalize dataset for tutorial"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eT7-MMpfrlHR"
   },
   "source": [
    "## Standard Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MDGbfWaTrlHS"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "#from IPython import display\n",
    "from IPython.display import display, HTML\n",
    "%matplotlib inline\n",
    "import time\n",
    "pd.set_option('display.max_columns', None) # to see all columns of df.head()\n",
    "np.random.seed(31415) # set the random seed\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 91424\n",
      "-rw-r--r--+ 1 rousseau  staff    498849 13 mai 17:01 dataWW_d1_3K.csv\n",
      "-rw-r--r--+ 1 rousseau  staff   9059494 13 mai 20:33 dataWW_d1_150k.csv.gz\n",
      "-rw-r--r--+ 1 rousseau  staff  35812815 13 mai 22:24 dataWW_d1_600k.csv.gz\n",
      "Will write data file to :  /Users/rousseau/Google Drive/GD_openData/dataWW_ATLAS_openData13TeV_filtered\n"
     ]
    }
   ],
   "source": [
    "path=\"/Users/rousseau/Downloads\"\n",
    "\n",
    "outputdatapath=\"/Users/rousseau/Google\\ Drive/GD_openData/dataWW_ATLAS_openData13TeV_filtered/\"\n",
    "\n",
    "!ls -lrt {outputdatapath}\n",
    "outputdatapath=os.path.abspath(outputdatapath).replace(\"\\ \", \" \")  # try to normalise the path (annoyance with the space)\n",
    "print (\"Will write data file to : \",outputdatapath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 323
    },
    "colab_type": "code",
    "id": "nBqFUzeXrlHX",
    "outputId": "74b1cc66-c765-4adf-8087-b274d2d9373a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eventNumber</th>\n",
       "      <th>channelNumber</th>\n",
       "      <th>met_et</th>\n",
       "      <th>met_phi</th>\n",
       "      <th>lep_n</th>\n",
       "      <th>lep_pt_0</th>\n",
       "      <th>lep_pt_1</th>\n",
       "      <th>lep_eta_0</th>\n",
       "      <th>lep_eta_1</th>\n",
       "      <th>lep_phi_0</th>\n",
       "      <th>lep_phi_1</th>\n",
       "      <th>lep_E_0</th>\n",
       "      <th>lep_E_1</th>\n",
       "      <th>lep_charge_0</th>\n",
       "      <th>lep_charge_1</th>\n",
       "      <th>lep_type_0</th>\n",
       "      <th>lep_type_1</th>\n",
       "      <th>jet_n</th>\n",
       "      <th>jet_pt_0</th>\n",
       "      <th>jet_pt_1</th>\n",
       "      <th>jet_eta_0</th>\n",
       "      <th>jet_eta_1</th>\n",
       "      <th>jet_phi_0</th>\n",
       "      <th>jet_phi_1</th>\n",
       "      <th>jet_E_0</th>\n",
       "      <th>jet_E_1</th>\n",
       "      <th>mcWeight</th>\n",
       "      <th>runNumber</th>\n",
       "      <th>XSection</th>\n",
       "      <th>SumWeights</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>384</td>\n",
       "      <td>345323</td>\n",
       "      <td>116.8600</td>\n",
       "      <td>-1.26540</td>\n",
       "      <td>2</td>\n",
       "      <td>57.987</td>\n",
       "      <td>33.692</td>\n",
       "      <td>-0.17389</td>\n",
       "      <td>-0.875490</td>\n",
       "      <td>-1.489800</td>\n",
       "      <td>-1.62820</td>\n",
       "      <td>58866.0</td>\n",
       "      <td>47451.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>29.895</td>\n",
       "      <td>29.083</td>\n",
       "      <td>-2.4547</td>\n",
       "      <td>1.21450</td>\n",
       "      <td>-2.4571</td>\n",
       "      <td>0.57501</td>\n",
       "      <td>175420.0</td>\n",
       "      <td>53507.0</td>\n",
       "      <td>3.7499</td>\n",
       "      <td>284500</td>\n",
       "      <td>1.9059</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>841</td>\n",
       "      <td>345323</td>\n",
       "      <td>112.0200</td>\n",
       "      <td>-0.86899</td>\n",
       "      <td>2</td>\n",
       "      <td>29.739</td>\n",
       "      <td>12.656</td>\n",
       "      <td>-1.02240</td>\n",
       "      <td>0.053344</td>\n",
       "      <td>0.935910</td>\n",
       "      <td>0.62834</td>\n",
       "      <td>46684.0</td>\n",
       "      <td>12675.0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>3</td>\n",
       "      <td>99.283</td>\n",
       "      <td>32.058</td>\n",
       "      <td>-2.3502</td>\n",
       "      <td>0.76057</td>\n",
       "      <td>3.1212</td>\n",
       "      <td>-1.51410</td>\n",
       "      <td>525480.0</td>\n",
       "      <td>42347.0</td>\n",
       "      <td>3.7499</td>\n",
       "      <td>284500</td>\n",
       "      <td>1.9059</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>184</td>\n",
       "      <td>345323</td>\n",
       "      <td>314.2400</td>\n",
       "      <td>-0.37074</td>\n",
       "      <td>2</td>\n",
       "      <td>42.921</td>\n",
       "      <td>18.685</td>\n",
       "      <td>-0.89607</td>\n",
       "      <td>-1.140800</td>\n",
       "      <td>-0.552370</td>\n",
       "      <td>-1.64300</td>\n",
       "      <td>61337.0</td>\n",
       "      <td>32220.0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>243.040</td>\n",
       "      <td>73.488</td>\n",
       "      <td>1.2938</td>\n",
       "      <td>-2.06750</td>\n",
       "      <td>2.7464</td>\n",
       "      <td>-2.92700</td>\n",
       "      <td>476640.0</td>\n",
       "      <td>295290.0</td>\n",
       "      <td>3.7499</td>\n",
       "      <td>284500</td>\n",
       "      <td>1.9059</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>817</td>\n",
       "      <td>345323</td>\n",
       "      <td>72.0230</td>\n",
       "      <td>-0.99177</td>\n",
       "      <td>2</td>\n",
       "      <td>67.623</td>\n",
       "      <td>65.901</td>\n",
       "      <td>0.90236</td>\n",
       "      <td>0.625700</td>\n",
       "      <td>-0.045364</td>\n",
       "      <td>-0.46053</td>\n",
       "      <td>97073.0</td>\n",
       "      <td>79228.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>213.950</td>\n",
       "      <td>23.831</td>\n",
       "      <td>2.4991</td>\n",
       "      <td>2.47340</td>\n",
       "      <td>2.9268</td>\n",
       "      <td>0.57894</td>\n",
       "      <td>1311000.0</td>\n",
       "      <td>142500.0</td>\n",
       "      <td>3.7499</td>\n",
       "      <td>284500</td>\n",
       "      <td>1.9059</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1588</td>\n",
       "      <td>345323</td>\n",
       "      <td>4.4252</td>\n",
       "      <td>-2.43290</td>\n",
       "      <td>2</td>\n",
       "      <td>40.543</td>\n",
       "      <td>21.005</td>\n",
       "      <td>0.44649</td>\n",
       "      <td>0.291370</td>\n",
       "      <td>1.649700</td>\n",
       "      <td>-0.47812</td>\n",
       "      <td>44652.0</td>\n",
       "      <td>21903.0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>-7.000</td>\n",
       "      <td>-7.000</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>3.7499</td>\n",
       "      <td>284500</td>\n",
       "      <td>1.9059</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   eventNumber  channelNumber    met_et  met_phi  lep_n  lep_pt_0  lep_pt_1  \\\n",
       "0          384         345323  116.8600 -1.26540      2    57.987    33.692   \n",
       "1          841         345323  112.0200 -0.86899      2    29.739    12.656   \n",
       "2          184         345323  314.2400 -0.37074      2    42.921    18.685   \n",
       "3          817         345323   72.0230 -0.99177      2    67.623    65.901   \n",
       "4         1588         345323    4.4252 -2.43290      2    40.543    21.005   \n",
       "\n",
       "   lep_eta_0  lep_eta_1  lep_phi_0  lep_phi_1  lep_E_0  lep_E_1  lep_charge_0  \\\n",
       "0   -0.17389  -0.875490  -1.489800   -1.62820  58866.0  47451.0            -1   \n",
       "1   -1.02240   0.053344   0.935910    0.62834  46684.0  12675.0             1   \n",
       "2   -0.89607  -1.140800  -0.552370   -1.64300  61337.0  32220.0             1   \n",
       "3    0.90236   0.625700  -0.045364   -0.46053  97073.0  79228.0            -1   \n",
       "4    0.44649   0.291370   1.649700   -0.47812  44652.0  21903.0             1   \n",
       "\n",
       "   lep_charge_1  lep_type_0  lep_type_1  jet_n  jet_pt_0  jet_pt_1  jet_eta_0  \\\n",
       "0             1          13          11      2    29.895    29.083    -2.4547   \n",
       "1            -1          11          13      3    99.283    32.058    -2.3502   \n",
       "2            -1          13          11      3   243.040    73.488     1.2938   \n",
       "3             1          13          13      2   213.950    23.831     2.4991   \n",
       "4            -1          11          13      0    -7.000    -7.000    -7.0000   \n",
       "\n",
       "   jet_eta_1  jet_phi_0  jet_phi_1    jet_E_0   jet_E_1  mcWeight  runNumber  \\\n",
       "0    1.21450    -2.4571    0.57501   175420.0   53507.0    3.7499     284500   \n",
       "1    0.76057     3.1212   -1.51410   525480.0   42347.0    3.7499     284500   \n",
       "2   -2.06750     2.7464   -2.92700   476640.0  295290.0    3.7499     284500   \n",
       "3    2.47340     2.9268    0.57894  1311000.0  142500.0    3.7499     284500   \n",
       "4   -7.00000    -7.0000   -7.00000       -7.0      -7.0    3.7499     284500   \n",
       "\n",
       "   XSection  SumWeights  \n",
       "0    1.9059   4390000.0  \n",
       "1    1.9059   4390000.0  \n",
       "2    1.9059   4390000.0  \n",
       "3    1.9059   4390000.0  \n",
       "4    1.9059   4390000.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eventNumber</th>\n",
       "      <th>channelNumber</th>\n",
       "      <th>met_et</th>\n",
       "      <th>met_phi</th>\n",
       "      <th>lep_n</th>\n",
       "      <th>lep_pt_0</th>\n",
       "      <th>lep_pt_1</th>\n",
       "      <th>lep_eta_0</th>\n",
       "      <th>lep_eta_1</th>\n",
       "      <th>lep_phi_0</th>\n",
       "      <th>lep_phi_1</th>\n",
       "      <th>lep_E_0</th>\n",
       "      <th>lep_E_1</th>\n",
       "      <th>lep_charge_0</th>\n",
       "      <th>lep_charge_1</th>\n",
       "      <th>lep_type_0</th>\n",
       "      <th>lep_type_1</th>\n",
       "      <th>jet_n</th>\n",
       "      <th>jet_pt_0</th>\n",
       "      <th>jet_pt_1</th>\n",
       "      <th>jet_eta_0</th>\n",
       "      <th>jet_eta_1</th>\n",
       "      <th>jet_phi_0</th>\n",
       "      <th>jet_phi_1</th>\n",
       "      <th>jet_E_0</th>\n",
       "      <th>jet_E_1</th>\n",
       "      <th>mcWeight</th>\n",
       "      <th>runNumber</th>\n",
       "      <th>XSection</th>\n",
       "      <th>SumWeights</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>2.000000e+05</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>2.000000e+05</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>2.000000e+05</td>\n",
       "      <td>2.000000e+05</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>2.000000e+05</td>\n",
       "      <td>200000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>5.618368e+05</td>\n",
       "      <td>345323.0</td>\n",
       "      <td>61.254961</td>\n",
       "      <td>-0.006217</td>\n",
       "      <td>2.006050</td>\n",
       "      <td>57.741068</td>\n",
       "      <td>25.521284</td>\n",
       "      <td>0.002152</td>\n",
       "      <td>0.002656</td>\n",
       "      <td>0.002530</td>\n",
       "      <td>0.009040</td>\n",
       "      <td>1.056596e+05</td>\n",
       "      <td>48120.910842</td>\n",
       "      <td>-0.002700</td>\n",
       "      <td>0.003400</td>\n",
       "      <td>11.908710</td>\n",
       "      <td>11.961920</td>\n",
       "      <td>1.326380</td>\n",
       "      <td>70.880788</td>\n",
       "      <td>12.022779</td>\n",
       "      <td>-1.434421</td>\n",
       "      <td>-4.479583</td>\n",
       "      <td>-1.445405</td>\n",
       "      <td>-4.479064</td>\n",
       "      <td>1.875346e+05</td>\n",
       "      <td>4.543356e+04</td>\n",
       "      <td>3.737188</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>1.905900e+00</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>3.208474e+05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>45.104270</td>\n",
       "      <td>1.812738</td>\n",
       "      <td>0.077996</td>\n",
       "      <td>33.593566</td>\n",
       "      <td>15.738560</td>\n",
       "      <td>1.158605</td>\n",
       "      <td>1.187890</td>\n",
       "      <td>1.810775</td>\n",
       "      <td>1.814271</td>\n",
       "      <td>8.409278e+04</td>\n",
       "      <td>42408.649736</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.995827</td>\n",
       "      <td>0.999277</td>\n",
       "      <td>1.058495</td>\n",
       "      <td>72.620442</td>\n",
       "      <td>31.605595</td>\n",
       "      <td>3.145023</td>\n",
       "      <td>3.491334</td>\n",
       "      <td>3.256724</td>\n",
       "      <td>3.533596</td>\n",
       "      <td>2.277730e+05</td>\n",
       "      <td>9.405599e+04</td>\n",
       "      <td>0.308509</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.714071e-12</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>345323.0</td>\n",
       "      <td>0.095622</td>\n",
       "      <td>-3.141600</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>25.001000</td>\n",
       "      <td>7.000300</td>\n",
       "      <td>-2.696400</td>\n",
       "      <td>-2.699900</td>\n",
       "      <td>-3.141500</td>\n",
       "      <td>-3.141500</td>\n",
       "      <td>2.502300e+04</td>\n",
       "      <td>7003.800000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>-3.749900</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>1.905900e+00</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>3.277008e+05</td>\n",
       "      <td>345323.0</td>\n",
       "      <td>33.251750</td>\n",
       "      <td>-1.576500</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>37.055000</td>\n",
       "      <td>14.428000</td>\n",
       "      <td>-0.848492</td>\n",
       "      <td>-0.892413</td>\n",
       "      <td>-1.567525</td>\n",
       "      <td>-1.563100</td>\n",
       "      <td>5.235075e+04</td>\n",
       "      <td>21771.750000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>24.793750</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-2.275500</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-2.790425</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>4.011950e+04</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>3.749900</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>1.905900e+00</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>5.014455e+05</td>\n",
       "      <td>345323.0</td>\n",
       "      <td>51.911500</td>\n",
       "      <td>-0.018328</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>49.256000</td>\n",
       "      <td>22.260000</td>\n",
       "      <td>0.004078</td>\n",
       "      <td>0.006082</td>\n",
       "      <td>0.014238</td>\n",
       "      <td>0.017809</td>\n",
       "      <td>7.927600e+04</td>\n",
       "      <td>35129.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>57.474000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-0.771710</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-0.828880</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>1.193700e+05</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>3.749900</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>1.905900e+00</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>7.697402e+05</td>\n",
       "      <td>345323.0</td>\n",
       "      <td>76.906000</td>\n",
       "      <td>1.560400</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>68.387000</td>\n",
       "      <td>31.862000</td>\n",
       "      <td>0.855852</td>\n",
       "      <td>0.897752</td>\n",
       "      <td>1.570525</td>\n",
       "      <td>1.577100</td>\n",
       "      <td>1.295025e+05</td>\n",
       "      <td>58667.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>101.700000</td>\n",
       "      <td>26.742000</td>\n",
       "      <td>1.089900</td>\n",
       "      <td>-1.168700</td>\n",
       "      <td>1.157900</td>\n",
       "      <td>-1.226900</td>\n",
       "      <td>2.514300e+05</td>\n",
       "      <td>5.723350e+04</td>\n",
       "      <td>3.749900</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>1.905900e+00</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>1.199997e+06</td>\n",
       "      <td>345323.0</td>\n",
       "      <td>3634.600000</td>\n",
       "      <td>3.141600</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3624.600000</td>\n",
       "      <td>382.170000</td>\n",
       "      <td>2.699800</td>\n",
       "      <td>2.699200</td>\n",
       "      <td>3.141500</td>\n",
       "      <td>3.141500</td>\n",
       "      <td>3.624700e+06</td>\n",
       "      <td>870750.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1658.200000</td>\n",
       "      <td>586.590000</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>2.499900</td>\n",
       "      <td>3.141600</td>\n",
       "      <td>3.141600</td>\n",
       "      <td>4.006900e+06</td>\n",
       "      <td>2.388900e+06</td>\n",
       "      <td>3.749900</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>1.905900e+00</td>\n",
       "      <td>4390000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        eventNumber  channelNumber         met_et        met_phi  \\\n",
       "count  2.000000e+05       200000.0  200000.000000  200000.000000   \n",
       "mean   5.618368e+05       345323.0      61.254961      -0.006217   \n",
       "std    3.208474e+05            0.0      45.104270       1.812738   \n",
       "min    2.000000e+00       345323.0       0.095622      -3.141600   \n",
       "25%    3.277008e+05       345323.0      33.251750      -1.576500   \n",
       "50%    5.014455e+05       345323.0      51.911500      -0.018328   \n",
       "75%    7.697402e+05       345323.0      76.906000       1.560400   \n",
       "max    1.199997e+06       345323.0    3634.600000       3.141600   \n",
       "\n",
       "               lep_n       lep_pt_0       lep_pt_1      lep_eta_0  \\\n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000   \n",
       "mean        2.006050      57.741068      25.521284       0.002152   \n",
       "std         0.077996      33.593566      15.738560       1.158605   \n",
       "min         2.000000      25.001000       7.000300      -2.696400   \n",
       "25%         2.000000      37.055000      14.428000      -0.848492   \n",
       "50%         2.000000      49.256000      22.260000       0.004078   \n",
       "75%         2.000000      68.387000      31.862000       0.855852   \n",
       "max         4.000000    3624.600000     382.170000       2.699800   \n",
       "\n",
       "           lep_eta_1      lep_phi_0      lep_phi_1       lep_E_0  \\\n",
       "count  200000.000000  200000.000000  200000.000000  2.000000e+05   \n",
       "mean        0.002656       0.002530       0.009040  1.056596e+05   \n",
       "std         1.187890       1.810775       1.814271  8.409278e+04   \n",
       "min        -2.699900      -3.141500      -3.141500  2.502300e+04   \n",
       "25%        -0.892413      -1.567525      -1.563100  5.235075e+04   \n",
       "50%         0.006082       0.014238       0.017809  7.927600e+04   \n",
       "75%         0.897752       1.570525       1.577100  1.295025e+05   \n",
       "max         2.699200       3.141500       3.141500  3.624700e+06   \n",
       "\n",
       "             lep_E_1   lep_charge_0   lep_charge_1     lep_type_0  \\\n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000   \n",
       "mean    48120.910842      -0.002700       0.003400      11.908710   \n",
       "std     42408.649736       0.999999       0.999997       0.995827   \n",
       "min      7003.800000      -1.000000      -1.000000      11.000000   \n",
       "25%     21771.750000      -1.000000      -1.000000      11.000000   \n",
       "50%     35129.000000      -1.000000       1.000000      11.000000   \n",
       "75%     58667.000000       1.000000       1.000000      13.000000   \n",
       "max    870750.000000       1.000000       1.000000      13.000000   \n",
       "\n",
       "          lep_type_1          jet_n       jet_pt_0       jet_pt_1  \\\n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000   \n",
       "mean       11.961920       1.326380      70.880788      12.022779   \n",
       "std         0.999277       1.058495      72.620442      31.605595   \n",
       "min        11.000000       0.000000      -7.000000      -7.000000   \n",
       "25%        11.000000       1.000000      24.793750      -7.000000   \n",
       "50%        11.000000       1.000000      57.474000      -7.000000   \n",
       "75%        13.000000       2.000000     101.700000      26.742000   \n",
       "max        13.000000      10.000000    1658.200000     586.590000   \n",
       "\n",
       "           jet_eta_0      jet_eta_1      jet_phi_0      jet_phi_1  \\\n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000   \n",
       "mean       -1.434421      -4.479583      -1.445405      -4.479064   \n",
       "std         3.145023       3.491334       3.256724       3.533596   \n",
       "min        -7.000000      -7.000000      -7.000000      -7.000000   \n",
       "25%        -2.275500      -7.000000      -2.790425      -7.000000   \n",
       "50%        -0.771710      -7.000000      -0.828880      -7.000000   \n",
       "75%         1.089900      -1.168700       1.157900      -1.226900   \n",
       "max         2.500000       2.499900       3.141600       3.141600   \n",
       "\n",
       "            jet_E_0       jet_E_1       mcWeight  runNumber      XSection  \\\n",
       "count  2.000000e+05  2.000000e+05  200000.000000   200000.0  2.000000e+05   \n",
       "mean   1.875346e+05  4.543356e+04       3.737188   284500.0  1.905900e+00   \n",
       "std    2.277730e+05  9.405599e+04       0.308509        0.0  7.714071e-12   \n",
       "min   -7.000000e+00 -7.000000e+00      -3.749900   284500.0  1.905900e+00   \n",
       "25%    4.011950e+04 -7.000000e+00       3.749900   284500.0  1.905900e+00   \n",
       "50%    1.193700e+05 -7.000000e+00       3.749900   284500.0  1.905900e+00   \n",
       "75%    2.514300e+05  5.723350e+04       3.749900   284500.0  1.905900e+00   \n",
       "max    4.006900e+06  2.388900e+06       3.749900   284500.0  1.905900e+00   \n",
       "\n",
       "       SumWeights  \n",
       "count    200000.0  \n",
       "mean    4390000.0  \n",
       "std           0.0  \n",
       "min     4390000.0  \n",
       "25%     4390000.0  \n",
       "50%     4390000.0  \n",
       "75%     4390000.0  \n",
       "max     4390000.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "# cross section to come from https://github.com/atlas-outreach-data-tools/atlas-outreach-cpp-framework-13tev/blob/master/Plotting/inputfiles/Files_HWW.txt\n",
    "# the one in the input file can be wrong\n",
    "labels=[\"qq\",\"ggH\",\"VBFH\"]\n",
    "detaillevel=\"d1\"\n",
    "cross_sections={\"qq\":12.466,\"ggH\":0.58,\"VBFH\":0.04} # in fb\n",
    "\n",
    "outputfilename=os.path.join(outputdatapath,\"dataWW_\"+detaillevel+\".csv.gz\")\n",
    "\n",
    "# total integrated luminosity ATLAS 2012 : 139fb^-1\n",
    "luminosity=10 #fb^-1 about 2015 data \n",
    "\n",
    "\n",
    "\n",
    "import os\n",
    "\n",
    "all_events={}\n",
    "for label in labels:        \n",
    "    filepath=os.path.join(path,\"dataWW_\"+label+\"_\"+detaillevel+\".csv.gz\")\n",
    "    all_events[label] = pd.read_csv(filepath) \n",
    "\n",
    "\n",
    "#qq_events = pd.read_csv(\"/Users/rousseau/Google Drive/GD_opendata/dataWW_ATLAS_openData13TeV_filtered/dataWW_qq_d0_1000k.csv.gz\",sep=\",\" ) \n",
    "#ggh_events = pd.read_csv(datapath+\"dataWW_ggH_d0_100k.csv.gz\",sep=\",\") \n",
    "#vbfh_events = pd.read_hdf(datapath+\"dataWW_VBFH.hdf\",\"VBFH\",WRITEABLE=False ) #Vector Boson Fusion http://opendata.cern.ch/record/3826\n",
    "all_events[\"VBFH\"].columns\n",
    "display(all_events[\"VBFH\"].head())\n",
    "display(all_events[\"VBFH\"].describe())\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For file  qq  n events  200000  sum of weights: 63409.528374299996  expected: 5039300.0  weight correction 1.577050051684822e-05\n",
      " now multiply all weights by  0.001965950594430299\n",
      "For file  ggH  n events  200000  sum of weights: 5673385.703999999  expected: 55833000.0  weight correction 1.762615926667834e-07\n",
      " now multiply all weights by  1.0223172374673438e-06\n",
      "For file  VBFH  n events  200000  sum of weights: 747437.5677999997  expected: 4390000.0  weight correction 1.3379043857046015e-06\n",
      " now multiply all weights by  5.351617542818406e-07\n"
     ]
    }
   ],
   "source": [
    "# compute normalisation factors\n",
    "for label in labels:\n",
    "    sumw=all_events[label][\"mcWeight\"].sum()\n",
    "    totalw=all_events[label][\"SumWeights\"][0]\n",
    "    weight_correction=1/sumw\n",
    "    print(\"For file \",label,\" n events \", all_events[label].shape[0],\" sum of weights:\",sumw, \" expected:\",totalw,\" weight correction\",weight_correction)\n",
    "    scaling=luminosity*cross_sections[label]*weight_correction\n",
    "    print (\" now multiply all weights by \",scaling)\n",
    "    all_events[label][\"mcWeight\"]*=scaling\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "OzfF0gNlrlHb",
    "outputId": "e2ea2bf3-5fa7-47b8-a85a-f3fb5d59cd71"
   },
   "outputs": [],
   "source": [
    "# add new label for each dataset\n",
    "all_events[\"qq\"][\"label\"] = 0 # qq background\n",
    "all_events[\"ggH\"][\"label\"] = 1 # ggH signal\n",
    "all_events[\"VBFH\"][\"label\"] = 1 # VBFH signal\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "OzfF0gNlrlHb",
    "outputId": "e2ea2bf3-5fa7-47b8-a85a-f3fb5d59cd71"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eventNumber</th>\n",
       "      <th>channelNumber</th>\n",
       "      <th>met_et</th>\n",
       "      <th>met_phi</th>\n",
       "      <th>lep_n</th>\n",
       "      <th>lep_pt_0</th>\n",
       "      <th>lep_pt_1</th>\n",
       "      <th>lep_eta_0</th>\n",
       "      <th>lep_eta_1</th>\n",
       "      <th>lep_phi_0</th>\n",
       "      <th>lep_phi_1</th>\n",
       "      <th>lep_E_0</th>\n",
       "      <th>lep_E_1</th>\n",
       "      <th>lep_charge_0</th>\n",
       "      <th>lep_charge_1</th>\n",
       "      <th>lep_type_0</th>\n",
       "      <th>lep_type_1</th>\n",
       "      <th>jet_n</th>\n",
       "      <th>jet_pt_0</th>\n",
       "      <th>jet_pt_1</th>\n",
       "      <th>jet_eta_0</th>\n",
       "      <th>jet_eta_1</th>\n",
       "      <th>jet_phi_0</th>\n",
       "      <th>jet_phi_1</th>\n",
       "      <th>jet_E_0</th>\n",
       "      <th>jet_E_1</th>\n",
       "      <th>mcWeight</th>\n",
       "      <th>runNumber</th>\n",
       "      <th>XSection</th>\n",
       "      <th>SumWeights</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>6.000000e+05</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>6.000000e+05</td>\n",
       "      <td>6.000000e+05</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>6.000000e+05</td>\n",
       "      <td>6.000000e+05</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>600000.0</td>\n",
       "      <td>600000.000000</td>\n",
       "      <td>6.000000e+05</td>\n",
       "      <td>600000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>7.029775e+05</td>\n",
       "      <td>351379.666667</td>\n",
       "      <td>60.365356</td>\n",
       "      <td>-0.008556</td>\n",
       "      <td>2.005748</td>\n",
       "      <td>60.270328</td>\n",
       "      <td>28.133970</td>\n",
       "      <td>0.000269</td>\n",
       "      <td>-0.000664</td>\n",
       "      <td>0.005219</td>\n",
       "      <td>0.008524</td>\n",
       "      <td>1.164008e+05</td>\n",
       "      <td>5.630948e+04</td>\n",
       "      <td>-0.027027</td>\n",
       "      <td>0.027037</td>\n",
       "      <td>11.907273</td>\n",
       "      <td>11.958010</td>\n",
       "      <td>1.334185</td>\n",
       "      <td>59.606114</td>\n",
       "      <td>13.521164</td>\n",
       "      <td>-1.934521</td>\n",
       "      <td>-4.402337</td>\n",
       "      <td>-1.939645</td>\n",
       "      <td>-4.401941</td>\n",
       "      <td>1.375410e+05</td>\n",
       "      <td>4.125468e+04</td>\n",
       "      <td>0.000218</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>9.448300</td>\n",
       "      <td>2.175410e+07</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>4.568961e+05</td>\n",
       "      <td>8564.720183</td>\n",
       "      <td>48.153210</td>\n",
       "      <td>1.812778</td>\n",
       "      <td>0.076061</td>\n",
       "      <td>42.676858</td>\n",
       "      <td>20.352272</td>\n",
       "      <td>1.210614</td>\n",
       "      <td>1.235890</td>\n",
       "      <td>1.811177</td>\n",
       "      <td>1.814205</td>\n",
       "      <td>1.083341e+05</td>\n",
       "      <td>5.748834e+04</td>\n",
       "      <td>0.999636</td>\n",
       "      <td>0.999635</td>\n",
       "      <td>0.995692</td>\n",
       "      <td>0.999119</td>\n",
       "      <td>1.232632</td>\n",
       "      <td>79.586143</td>\n",
       "      <td>38.074439</td>\n",
       "      <td>3.349404</td>\n",
       "      <td>3.491304</td>\n",
       "      <td>3.489197</td>\n",
       "      <td>3.559421</td>\n",
       "      <td>1.980717e+05</td>\n",
       "      <td>8.990068e+04</td>\n",
       "      <td>0.000736</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.368655</td>\n",
       "      <td>2.409890e+07</td>\n",
       "      <td>0.471405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>345323.000000</td>\n",
       "      <td>0.044211</td>\n",
       "      <td>-3.141600</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>7.000300</td>\n",
       "      <td>-2.700000</td>\n",
       "      <td>-2.699900</td>\n",
       "      <td>-3.141600</td>\n",
       "      <td>-3.141600</td>\n",
       "      <td>2.501300e+04</td>\n",
       "      <td>7.003800e+03</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>-0.045557</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>1.905900</td>\n",
       "      <td>4.390000e+06</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>3.498568e+05</td>\n",
       "      <td>345323.000000</td>\n",
       "      <td>33.196000</td>\n",
       "      <td>-1.577600</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>36.624750</td>\n",
       "      <td>15.316000</td>\n",
       "      <td>-0.899650</td>\n",
       "      <td>-0.946043</td>\n",
       "      <td>-1.560200</td>\n",
       "      <td>-1.562900</td>\n",
       "      <td>5.221675e+04</td>\n",
       "      <td>2.329100e+04</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>1.905900</td>\n",
       "      <td>4.390000e+06</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>6.678815e+05</td>\n",
       "      <td>345324.000000</td>\n",
       "      <td>50.082000</td>\n",
       "      <td>-0.018694</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>48.229500</td>\n",
       "      <td>23.644000</td>\n",
       "      <td>0.003413</td>\n",
       "      <td>0.002974</td>\n",
       "      <td>0.014583</td>\n",
       "      <td>0.018823</td>\n",
       "      <td>8.184450e+04</td>\n",
       "      <td>3.824900e+04</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>38.744000</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-0.891010</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>-1.210600</td>\n",
       "      <td>-7.000000</td>\n",
       "      <td>7.501700e+04</td>\n",
       "      <td>-7.000000e+00</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>12.466000</td>\n",
       "      <td>5.039300e+06</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>1.001273e+06</td>\n",
       "      <td>363492.000000</td>\n",
       "      <td>73.032250</td>\n",
       "      <td>1.559300</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>68.979000</td>\n",
       "      <td>34.149000</td>\n",
       "      <td>0.901950</td>\n",
       "      <td>0.942210</td>\n",
       "      <td>1.573000</td>\n",
       "      <td>1.577300</td>\n",
       "      <td>1.412400e+05</td>\n",
       "      <td>6.730825e+04</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>84.291250</td>\n",
       "      <td>26.881000</td>\n",
       "      <td>0.724705</td>\n",
       "      <td>-0.821005</td>\n",
       "      <td>0.967753</td>\n",
       "      <td>-1.088575</td>\n",
       "      <td>1.766200e+05</td>\n",
       "      <td>5.060500e+04</td>\n",
       "      <td>0.000197</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>13.973000</td>\n",
       "      <td>5.583300e+07</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>1.997995e+06</td>\n",
       "      <td>363492.000000</td>\n",
       "      <td>7062.200000</td>\n",
       "      <td>3.141600</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7041.400000</td>\n",
       "      <td>744.900000</td>\n",
       "      <td>2.699800</td>\n",
       "      <td>2.700000</td>\n",
       "      <td>3.141500</td>\n",
       "      <td>3.141600</td>\n",
       "      <td>7.041600e+06</td>\n",
       "      <td>2.107900e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>1658.200000</td>\n",
       "      <td>1429.800000</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>3.141600</td>\n",
       "      <td>3.141600</td>\n",
       "      <td>5.638800e+06</td>\n",
       "      <td>2.784300e+06</td>\n",
       "      <td>0.052420</td>\n",
       "      <td>284500.0</td>\n",
       "      <td>13.973000</td>\n",
       "      <td>5.583300e+07</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        eventNumber  channelNumber         met_et        met_phi  \\\n",
       "count  6.000000e+05  600000.000000  600000.000000  600000.000000   \n",
       "mean   7.029775e+05  351379.666667      60.365356      -0.008556   \n",
       "std    4.568961e+05    8564.720183      48.153210       1.812778   \n",
       "min    1.000000e+00  345323.000000       0.044211      -3.141600   \n",
       "25%    3.498568e+05  345323.000000      33.196000      -1.577600   \n",
       "50%    6.678815e+05  345324.000000      50.082000      -0.018694   \n",
       "75%    1.001273e+06  363492.000000      73.032250       1.559300   \n",
       "max    1.997995e+06  363492.000000    7062.200000       3.141600   \n",
       "\n",
       "               lep_n       lep_pt_0       lep_pt_1      lep_eta_0  \\\n",
       "count  600000.000000  600000.000000  600000.000000  600000.000000   \n",
       "mean        2.005748      60.270328      28.133970       0.000269   \n",
       "std         0.076061      42.676858      20.352272       1.210614   \n",
       "min         2.000000      25.000000       7.000300      -2.700000   \n",
       "25%         2.000000      36.624750      15.316000      -0.899650   \n",
       "50%         2.000000      48.229500      23.644000       0.003413   \n",
       "75%         2.000000      68.979000      34.149000       0.901950   \n",
       "max         5.000000    7041.400000     744.900000       2.699800   \n",
       "\n",
       "           lep_eta_1      lep_phi_0      lep_phi_1       lep_E_0  \\\n",
       "count  600000.000000  600000.000000  600000.000000  6.000000e+05   \n",
       "mean       -0.000664       0.005219       0.008524  1.164008e+05   \n",
       "std         1.235890       1.811177       1.814205  1.083341e+05   \n",
       "min        -2.699900      -3.141600      -3.141600  2.501300e+04   \n",
       "25%        -0.946043      -1.560200      -1.562900  5.221675e+04   \n",
       "50%         0.002974       0.014583       0.018823  8.184450e+04   \n",
       "75%         0.942210       1.573000       1.577300  1.412400e+05   \n",
       "max         2.700000       3.141500       3.141600  7.041600e+06   \n",
       "\n",
       "            lep_E_1   lep_charge_0   lep_charge_1     lep_type_0  \\\n",
       "count  6.000000e+05  600000.000000  600000.000000  600000.000000   \n",
       "mean   5.630948e+04      -0.027027       0.027037      11.907273   \n",
       "std    5.748834e+04       0.999636       0.999635       0.995692   \n",
       "min    7.003800e+03      -1.000000      -1.000000      11.000000   \n",
       "25%    2.329100e+04      -1.000000      -1.000000      11.000000   \n",
       "50%    3.824900e+04      -1.000000       1.000000      11.000000   \n",
       "75%    6.730825e+04       1.000000       1.000000      13.000000   \n",
       "max    2.107900e+06       1.000000       1.000000      13.000000   \n",
       "\n",
       "          lep_type_1          jet_n       jet_pt_0       jet_pt_1  \\\n",
       "count  600000.000000  600000.000000  600000.000000  600000.000000   \n",
       "mean       11.958010       1.334185      59.606114      13.521164   \n",
       "std         0.999119       1.232632      79.586143      38.074439   \n",
       "min        11.000000       0.000000      -7.000000      -7.000000   \n",
       "25%        11.000000       0.000000      -7.000000      -7.000000   \n",
       "50%        11.000000       1.000000      38.744000      -7.000000   \n",
       "75%        13.000000       2.000000      84.291250      26.881000   \n",
       "max        13.000000      13.000000    1658.200000    1429.800000   \n",
       "\n",
       "           jet_eta_0      jet_eta_1      jet_phi_0      jet_phi_1  \\\n",
       "count  600000.000000  600000.000000  600000.000000  600000.000000   \n",
       "mean       -1.934521      -4.402337      -1.939645      -4.401941   \n",
       "std         3.349404       3.491304       3.489197       3.559421   \n",
       "min        -7.000000      -7.000000      -7.000000      -7.000000   \n",
       "25%        -7.000000      -7.000000      -7.000000      -7.000000   \n",
       "50%        -0.891010      -7.000000      -1.210600      -7.000000   \n",
       "75%         0.724705      -0.821005       0.967753      -1.088575   \n",
       "max         2.500000       2.500000       3.141600       3.141600   \n",
       "\n",
       "            jet_E_0       jet_E_1       mcWeight  runNumber       XSection  \\\n",
       "count  6.000000e+05  6.000000e+05  600000.000000   600000.0  600000.000000   \n",
       "mean   1.375410e+05  4.125468e+04       0.000218   284500.0       9.448300   \n",
       "std    1.980717e+05  8.990068e+04       0.000736        0.0       5.368655   \n",
       "min   -7.000000e+00 -7.000000e+00      -0.045557   284500.0       1.905900   \n",
       "25%   -7.000000e+00 -7.000000e+00       0.000002   284500.0       1.905900   \n",
       "50%    7.501700e+04 -7.000000e+00       0.000029   284500.0      12.466000   \n",
       "75%    1.766200e+05  5.060500e+04       0.000197   284500.0      13.973000   \n",
       "max    5.638800e+06  2.784300e+06       0.052420   284500.0      13.973000   \n",
       "\n",
       "         SumWeights          label  \n",
       "count  6.000000e+05  600000.000000  \n",
       "mean   2.175410e+07       0.666667  \n",
       "std    2.409890e+07       0.471405  \n",
       "min    4.390000e+06       0.000000  \n",
       "25%    4.390000e+06       0.000000  \n",
       "50%    5.039300e+06       1.000000  \n",
       "75%    5.583300e+07       1.000000  \n",
       "max    5.583300e+07       1.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Index(['eventNumber', 'channelNumber', 'met_et', 'met_phi', 'lep_n',\n",
       "       'lep_pt_0', 'lep_pt_1', 'lep_eta_0', 'lep_eta_1', 'lep_phi_0',\n",
       "       'lep_phi_1', 'lep_E_0', 'lep_E_1', 'lep_charge_0', 'lep_charge_1',\n",
       "       'lep_type_0', 'lep_type_1', 'jet_n', 'jet_pt_0', 'jet_pt_1',\n",
       "       'jet_eta_0', 'jet_eta_1', 'jet_phi_0', 'jet_phi_1', 'jet_E_0',\n",
       "       'jet_E_1', 'mcWeight', 'runNumber', 'XSection', 'SumWeights', 'label'],\n",
       "      dtype='object')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# merge the dataset\n",
    "fulldata = pd.concat([all_events[\"qq\"],all_events[\"ggH\"],all_events[\"VBFH\"]]) # merge all dataset\n",
    "# randomise\n",
    "fulldata=fulldata.sample(frac=1).reset_index(drop=True) \n",
    "display(fulldata.describe())\n",
    "display(fulldata.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eventNumber</th>\n",
       "      <th>label</th>\n",
       "      <th>met_et</th>\n",
       "      <th>met_phi</th>\n",
       "      <th>lep_n</th>\n",
       "      <th>lep_pt_0</th>\n",
       "      <th>lep_pt_1</th>\n",
       "      <th>lep_eta_0</th>\n",
       "      <th>lep_eta_1</th>\n",
       "      <th>lep_phi_0</th>\n",
       "      <th>lep_phi_1</th>\n",
       "      <th>lep_E_0</th>\n",
       "      <th>lep_E_1</th>\n",
       "      <th>lep_charge_0</th>\n",
       "      <th>lep_charge_1</th>\n",
       "      <th>lep_type_0</th>\n",
       "      <th>lep_type_1</th>\n",
       "      <th>jet_n</th>\n",
       "      <th>jet_pt_0</th>\n",
       "      <th>jet_pt_1</th>\n",
       "      <th>jet_eta_0</th>\n",
       "      <th>jet_eta_1</th>\n",
       "      <th>jet_phi_0</th>\n",
       "      <th>jet_phi_1</th>\n",
       "      <th>jet_E_0</th>\n",
       "      <th>jet_E_1</th>\n",
       "      <th>mcWeight</th>\n",
       "      <th>runNumber</th>\n",
       "      <th>channelNumber</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>249632</td>\n",
       "      <td>1</td>\n",
       "      <td>25.118</td>\n",
       "      <td>1.14010</td>\n",
       "      <td>2</td>\n",
       "      <td>46.439</td>\n",
       "      <td>44.589</td>\n",
       "      <td>0.131030</td>\n",
       "      <td>0.40654</td>\n",
       "      <td>2.34760</td>\n",
       "      <td>-2.41510</td>\n",
       "      <td>46838.0</td>\n",
       "      <td>48325.0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>53.680</td>\n",
       "      <td>-7.000</td>\n",
       "      <td>2.4346</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>-0.76667</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>308720.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>284500</td>\n",
       "      <td>345323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>892975</td>\n",
       "      <td>1</td>\n",
       "      <td>27.974</td>\n",
       "      <td>0.84442</td>\n",
       "      <td>2</td>\n",
       "      <td>86.819</td>\n",
       "      <td>55.438</td>\n",
       "      <td>-0.017265</td>\n",
       "      <td>-0.16338</td>\n",
       "      <td>-1.44520</td>\n",
       "      <td>-1.72950</td>\n",
       "      <td>86832.0</td>\n",
       "      <td>56179.0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>126.700</td>\n",
       "      <td>-7.000</td>\n",
       "      <td>-1.2789</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>1.66290</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>245670.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>284500</td>\n",
       "      <td>345323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>730573</td>\n",
       "      <td>0</td>\n",
       "      <td>123.290</td>\n",
       "      <td>-2.94810</td>\n",
       "      <td>2</td>\n",
       "      <td>105.810</td>\n",
       "      <td>40.506</td>\n",
       "      <td>-1.338900</td>\n",
       "      <td>-1.03500</td>\n",
       "      <td>0.68539</td>\n",
       "      <td>-0.90527</td>\n",
       "      <td>215700.0</td>\n",
       "      <td>64211.0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>-7.000</td>\n",
       "      <td>-7.000</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>0.002468</td>\n",
       "      <td>284500</td>\n",
       "      <td>363492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>717584</td>\n",
       "      <td>0</td>\n",
       "      <td>78.558</td>\n",
       "      <td>1.04450</td>\n",
       "      <td>2</td>\n",
       "      <td>56.666</td>\n",
       "      <td>35.124</td>\n",
       "      <td>2.294600</td>\n",
       "      <td>1.88700</td>\n",
       "      <td>2.13110</td>\n",
       "      <td>-2.83820</td>\n",
       "      <td>283930.0</td>\n",
       "      <td>118570.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>79.799</td>\n",
       "      <td>53.942</td>\n",
       "      <td>1.6270</td>\n",
       "      <td>0.55314</td>\n",
       "      <td>-1.84880</td>\n",
       "      <td>-0.47055</td>\n",
       "      <td>211610.0</td>\n",
       "      <td>63629.0</td>\n",
       "      <td>0.000197</td>\n",
       "      <td>284500</td>\n",
       "      <td>363492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>319020</td>\n",
       "      <td>0</td>\n",
       "      <td>15.863</td>\n",
       "      <td>0.74840</td>\n",
       "      <td>2</td>\n",
       "      <td>94.708</td>\n",
       "      <td>72.811</td>\n",
       "      <td>0.812740</td>\n",
       "      <td>0.77954</td>\n",
       "      <td>1.84940</td>\n",
       "      <td>0.45967</td>\n",
       "      <td>127750.0</td>\n",
       "      <td>96077.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>143.180</td>\n",
       "      <td>-7.000</td>\n",
       "      <td>-0.6358</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>-1.91230</td>\n",
       "      <td>-7.00000</td>\n",
       "      <td>173810.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>0.000197</td>\n",
       "      <td>284500</td>\n",
       "      <td>363492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   eventNumber  label   met_et  met_phi  lep_n  lep_pt_0  lep_pt_1  lep_eta_0  \\\n",
       "0       249632      1   25.118  1.14010      2    46.439    44.589   0.131030   \n",
       "1       892975      1   27.974  0.84442      2    86.819    55.438  -0.017265   \n",
       "2       730573      0  123.290 -2.94810      2   105.810    40.506  -1.338900   \n",
       "3       717584      0   78.558  1.04450      2    56.666    35.124   2.294600   \n",
       "4       319020      0   15.863  0.74840      2    94.708    72.811   0.812740   \n",
       "\n",
       "   lep_eta_1  lep_phi_0  lep_phi_1   lep_E_0   lep_E_1  lep_charge_0  \\\n",
       "0    0.40654    2.34760   -2.41510   46838.0   48325.0             1   \n",
       "1   -0.16338   -1.44520   -1.72950   86832.0   56179.0             1   \n",
       "2   -1.03500    0.68539   -0.90527  215700.0   64211.0             1   \n",
       "3    1.88700    2.13110   -2.83820  283930.0  118570.0            -1   \n",
       "4    0.77954    1.84940    0.45967  127750.0   96077.0            -1   \n",
       "\n",
       "   lep_charge_1  lep_type_0  lep_type_1  jet_n  jet_pt_0  jet_pt_1  jet_eta_0  \\\n",
       "0            -1          13          11      1    53.680    -7.000     2.4346   \n",
       "1            -1          11          13      1   126.700    -7.000    -1.2789   \n",
       "2            -1          13          13      0    -7.000    -7.000    -7.0000   \n",
       "3             1          11          11      2    79.799    53.942     1.6270   \n",
       "4             1          11          11      1   143.180    -7.000    -0.6358   \n",
       "\n",
       "   jet_eta_1  jet_phi_0  jet_phi_1   jet_E_0  jet_E_1  mcWeight  runNumber  \\\n",
       "0   -7.00000   -0.76667   -7.00000  308720.0     -7.0  0.000002     284500   \n",
       "1   -7.00000    1.66290   -7.00000  245670.0     -7.0  0.000002     284500   \n",
       "2   -7.00000   -7.00000   -7.00000      -7.0     -7.0  0.002468     284500   \n",
       "3    0.55314   -1.84880   -0.47055  211610.0  63629.0  0.000197     284500   \n",
       "4   -7.00000   -1.91230   -7.00000  173810.0     -7.0  0.000197     284500   \n",
       "\n",
       "   channelNumber  \n",
       "0         345323  \n",
       "1         345323  \n",
       "2         363492  \n",
       "3         363492  \n",
       "4         363492  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#dop some columns\n",
    "fulldata.drop(columns=[\"XSection\",\"SumWeights\"],inplace=True)\n",
    "\n",
    "#reorder some columns\n",
    "varlist=fulldata.columns.tolist()\n",
    "oldvarlist=varlist.copy()\n",
    "\n",
    "#some stuff to the end\n",
    "for aname in [\"channelNumber\"]:\n",
    "    if not aname in varlist:\n",
    "        continue\n",
    "    varlist.remove(aname)\n",
    "    varlist.append(aname)\n",
    "    \n",
    "#some stuff at the beginning\n",
    "varlist.remove(\"label\")\n",
    "varlist.insert(1,\"label\")    \n",
    "\n",
    "# check we have only reordered\n",
    "if sorted(oldvarlist)!=sorted(varlist):\n",
    "    print (\"before:\",sorted(oldvarlist) )\n",
    "    print (\"after :\",sorted(varlist) )\n",
    "    raise Exception(\"something wrong when reordering the columns!\")\n",
    "\n",
    "# do the reordering\n",
    "fulldata = fulldata.reindex(columns=varlist)\n",
    "display(fulldata.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#write final dataset \n",
    "fulldata.to_csv(outputfilename,float_format='%.5g',index = False)# do csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "ST4PNT_ML_BDT_v2.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}