M1_Final_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "import re\n",
    "import imblearn\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "   MONTH  DAY_OF_MONTH  DAY_OF_WEEK OP_UNIQUE_CARRIER TAIL_NUM DEST  \\\n0     11             1            5                B6   N828JB  CHS   \n1     11             1            5                B6   N992JB  LAX   \n2     11             1            5                B6   N959JB  FLL   \n3     11             1            5                B6   N999JQ  MCO   \n4     11             1            5                DL   N880DN  ATL   \n\n   DEP_DELAY  CRS_ELAPSED_TIME  DISTANCE  CRS_DEP_M  ...  Dew Point  Humidity  \\\n0         -1               124       636        324  ...         34        58   \n1         -7               371      2475        340  ...         34        58   \n2         40               181      1069        301  ...         34        58   \n3         -2               168       944        345  ...         34        58   \n4         -4               139       760        360  ...         32        58   \n\n   Wind Wind Speed  Wind Gust Pressure     Condition  sch_dep  sch_arr  \\\n0     W         25         38    29.86  Fair / Windy        9       17   \n1     W         25         38    29.86  Fair / Windy        9       17   \n2     W         25         38    29.86  Fair / Windy        9       17   \n3     W         25         38    29.86  Fair / Windy        9       17   \n4     W         24         35    29.91  Fair / Windy        9       17   \n\n  TAXI_OUT  \n0       14  \n1       15  \n2       22  \n3       12  \n4       13  \n\n[5 rows x 23 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>MONTH</th>\n      <th>DAY_OF_MONTH</th>\n      <th>DAY_OF_WEEK</th>\n      <th>OP_UNIQUE_CARRIER</th>\n      <th>TAIL_NUM</th>\n      <th>DEST</th>\n      <th>DEP_DELAY</th>\n      <th>CRS_ELAPSED_TIME</th>\n      <th>DISTANCE</th>\n      <th>CRS_DEP_M</th>\n      <th>...</th>\n      <th>Dew Point</th>\n      <th>Humidity</th>\n      <th>Wind</th>\n      <th>Wind Speed</th>\n      <th>Wind Gust</th>\n      <th>Pressure</th>\n      <th>Condition</th>\n      <th>sch_dep</th>\n      <th>sch_arr</th>\n      <th>TAXI_OUT</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>11</td>\n      <td>1</td>\n      <td>5</td>\n      <td>B6</td>\n      <td>N828JB</td>\n      <td>CHS</td>\n      <td>-1</td>\n      <td>124</td>\n      <td>636</td>\n      <td>324</td>\n      <td>...</td>\n      <td>34</td>\n      <td>58</td>\n      <td>W</td>\n      <td>25</td>\n      <td>38</td>\n      <td>29.86</td>\n      <td>Fair / Windy</td>\n      <td>9</td>\n      <td>17</td>\n      <td>14</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>11</td>\n      <td>1</td>\n      <td>5</td>\n      <td>B6</td>\n      <td>N992JB</td>\n      <td>LAX</td>\n      <td>-7</td>\n      <td>371</td>\n      <td>2475</td>\n      <td>340</td>\n      <td>...</td>\n      <td>34</td>\n      <td>58</td>\n      <td>W</td>\n      <td>25</td>\n      <td>38</td>\n      <td>29.86</td>\n      <td>Fair / Windy</td>\n      <td>9</td>\n      <td>17</td>\n      <td>15</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>11</td>\n      <td>1</td>\n      <td>5</td>\n      <td>B6</td>\n      <td>N959JB</td>\n      <td>FLL</td>\n      <td>40</td>\n      <td>181</td>\n      <td>1069</td>\n      <td>301</td>\n      <td>...</td>\n      <td>34</td>\n      <td>58</td>\n      <td>W</td>\n      <td>25</td>\n      <td>38</td>\n      <td>29.86</td>\n      <td>Fair / Windy</td>\n      <td>9</td>\n      <td>17</td>\n      <td>22</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>11</td>\n      <td>1</td>\n      <td>5</td>\n      <td>B6</td>\n      <td>N999JQ</td>\n      <td>MCO</td>\n      <td>-2</td>\n      <td>168</td>\n      <td>944</td>\n      <td>345</td>\n      <td>...</td>\n      <td>34</td>\n      <td>58</td>\n      <td>W</td>\n      <td>25</td>\n      <td>38</td>\n      <td>29.86</td>\n      <td>Fair / Windy</td>\n      <td>9</td>\n      <td>17</td>\n      <td>12</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>11</td>\n      <td>1</td>\n      <td>5</td>\n      <td>DL</td>\n      <td>N880DN</td>\n      <td>ATL</td>\n      <td>-4</td>\n      <td>139</td>\n      <td>760</td>\n      <td>360</td>\n      <td>...</td>\n      <td>32</td>\n      <td>58</td>\n      <td>W</td>\n      <td>24</td>\n      <td>35</td>\n      <td>29.91</td>\n      <td>Fair / Windy</td>\n      <td>9</td>\n      <td>17</td>\n      <td>13</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 23 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('M1_final.csv')\n",
    "df_pre = df.copy()\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(28820, 23)"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "              MONTH  DAY_OF_MONTH   DAY_OF_WEEK     DEP_DELAY  \\\ncount  28820.000000  28820.000000  28820.000000  28820.000000   \nmean       7.894240     16.021096      4.008952      6.374983   \nstd        4.991723      8.750179      1.985230     38.735144   \nmin        1.000000      1.000000      1.000000    -22.000000   \n25%        1.000000      8.000000      2.000000     -6.000000   \n50%       11.000000     16.000000      4.000000     -3.000000   \n75%       12.000000     24.000000      6.000000      2.000000   \nmax       12.000000     31.000000      7.000000   1276.000000   \n\n       CRS_ELAPSED_TIME      DISTANCE     CRS_DEP_M    DEP_TIME_M  \\\ncount      28820.000000  28820.000000  28820.000000  28820.000000   \nmean         225.288203   1267.746079    831.003851    828.934698   \nstd          119.482417    889.343246    299.398525    305.864103   \nmin           57.000000     94.000000    301.000000      1.000000   \n25%          124.000000    483.000000    545.000000    542.000000   \n50%          188.000000   1029.000000    856.000000    854.000000   \n75%          365.000000   2248.000000   1095.000000   1097.000000   \nmax          697.000000   4983.000000   1439.000000   1440.000000   \n\n          CRS_ARR_M   Temperature      Humidity    Wind Speed     Wind Gust  \\\ncount  28820.000000  28820.000000  28820.000000  28820.000000  28820.000000   \nmean     910.874289     41.489833     57.732616     12.367627      5.535323   \nstd      345.411743      8.043533     23.468676      6.259298     11.886457   \nmin        1.000000     17.000000      0.000000      0.000000      0.000000   \n25%      667.000000     36.000000     46.000000      8.000000      0.000000   \n50%      918.000000     42.000000     59.000000     12.000000      0.000000   \n75%     1193.000000     47.000000     74.000000     16.000000      0.000000   \nmax     1439.000000     68.000000     97.000000     36.000000     49.000000   \n\n           Pressure       sch_dep       sch_arr      TAXI_OUT  \ncount  28820.000000  28820.000000  28820.000000  28820.000000  \nmean      30.092433     31.091256     28.432130     20.858570  \nstd        0.296160      9.510359      8.263043      6.851915  \nmin       29.200000      0.000000      0.000000      5.000000  \n25%       29.880000     26.000000     21.000000     16.000000  \n50%       30.110000     30.000000     30.000000     19.000000  \n75%       30.320000     37.000000     35.000000     25.000000  \nmax       30.750000     55.000000     46.000000     41.000000  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>MONTH</th>\n      <th>DAY_OF_MONTH</th>\n      <th>DAY_OF_WEEK</th>\n      <th>DEP_DELAY</th>\n      <th>CRS_ELAPSED_TIME</th>\n      <th>DISTANCE</th>\n      <th>CRS_DEP_M</th>\n      <th>DEP_TIME_M</th>\n      <th>CRS_ARR_M</th>\n      <th>Temperature</th>\n      <th>Humidity</th>\n      <th>Wind Speed</th>\n      <th>Wind Gust</th>\n      <th>Pressure</th>\n      <th>sch_dep</th>\n      <th>sch_arr</th>\n      <th>TAXI_OUT</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n      <td>28820.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>7.894240</td>\n      <td>16.021096</td>\n      <td>4.008952</td>\n      <td>6.374983</td>\n      <td>225.288203</td>\n      <td>1267.746079</td>\n      <td>831.003851</td>\n      <td>828.934698</td>\n      <td>910.874289</td>\n      <td>41.489833</td>\n      <td>57.732616</td>\n      <td>12.367627</td>\n      <td>5.535323</td>\n      <td>30.092433</td>\n      <td>31.091256</td>\n      <td>28.432130</td>\n      <td>20.858570</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>4.991723</td>\n      <td>8.750179</td>\n      <td>1.985230</td>\n      <td>38.735144</td>\n      <td>119.482417</td>\n      <td>889.343246</td>\n      <td>299.398525</td>\n      <td>305.864103</td>\n      <td>345.411743</td>\n      <td>8.043533</td>\n      <td>23.468676</td>\n      <td>6.259298</td>\n      <td>11.886457</td>\n      <td>0.296160</td>\n      <td>9.510359</td>\n      <td>8.263043</td>\n      <td>6.851915</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>-22.000000</td>\n      <td>57.000000</td>\n      <td>94.000000</td>\n      <td>301.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>17.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>29.200000</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>5.000000</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>1.000000</td>\n      <td>8.000000</td>\n      <td>2.000000</td>\n      <td>-6.000000</td>\n      <td>124.000000</td>\n      <td>483.000000</td>\n      <td>545.000000</td>\n      <td>542.000000</td>\n      <td>667.000000</td>\n      <td>36.000000</td>\n      <td>46.000000</td>\n      <td>8.000000</td>\n      <td>0.000000</td>\n      <td>29.880000</td>\n      <td>26.000000</td>\n      <td>21.000000</td>\n      <td>16.000000</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>11.000000</td>\n      <td>16.000000</td>\n      <td>4.000000</td>\n      <td>-3.000000</td>\n      <td>188.000000</td>\n      <td>1029.000000</td>\n      <td>856.000000</td>\n      <td>854.000000</td>\n      <td>918.000000</td>\n      <td>42.000000</td>\n      <td>59.000000</td>\n      <td>12.000000</td>\n      <td>0.000000</td>\n      <td>30.110000</td>\n      <td>30.000000</td>\n      <td>30.000000</td>\n      <td>19.000000</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>12.000000</td>\n      <td>24.000000</td>\n      <td>6.000000</td>\n      <td>2.000000</td>\n      <td>365.000000</td>\n      <td>2248.000000</td>\n      <td>1095.000000</td>\n      <td>1097.000000</td>\n      <td>1193.000000</td>\n      <td>47.000000</td>\n      <td>74.000000</td>\n      <td>16.000000</td>\n      <td>0.000000</td>\n      <td>30.320000</td>\n      <td>37.000000</td>\n      <td>35.000000</td>\n      <td>25.000000</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>12.000000</td>\n      <td>31.000000</td>\n      <td>7.000000</td>\n      <td>1276.000000</td>\n      <td>697.000000</td>\n      <td>4983.000000</td>\n      <td>1439.000000</td>\n      <td>1440.000000</td>\n      <td>1439.000000</td>\n      <td>68.000000</td>\n      <td>97.000000</td>\n      <td>36.000000</td>\n      <td>49.000000</td>\n      <td>30.750000</td>\n      <td>55.000000</td>\n      <td>46.000000</td>\n      <td>41.000000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "Index(['MONTH', 'DAY_OF_MONTH', 'DAY_OF_WEEK', 'OP_UNIQUE_CARRIER', 'TAIL_NUM',\n       'DEST', 'DEP_DELAY', 'CRS_ELAPSED_TIME', 'DISTANCE', 'CRS_DEP_M',\n       'DEP_TIME_M', 'CRS_ARR_M', 'Temperature', 'Dew Point', 'Humidity',\n       'Wind', 'Wind Speed', 'Wind Gust', 'Pressure', 'Condition', 'sch_dep',\n       'sch_arr', 'TAXI_OUT'],\n      dtype='object')"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# all columns of df\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'9E': 0, 'AA': 1, 'AS': 2, 'B6': 3, 'DL': 4, 'HA': 5, 'MQ': 6, 'OO': 7, 'YX': 8}\n"
     ]
    }
   ],
   "source": [
    "# Label encoding for OP_UNIQUE_CARRIER\\n\",\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "df['OP_UNIQUE_CARRIER'] = le.fit_transform(df['OP_UNIQUE_CARRIER'])\n",
    "le_name_mapping = dict(zip(le.classes_, le.transform(le.classes_)))\n",
    "print(le_name_mapping)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mostly,heavy - 3\n",
    "Partly, Light - 2\n",
    "columns - cloudy, windy, fair, rain,fog, drizzle, snow, wintry mix , freezing rain,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#add columns of cloudy, windy, fair, rain,fog, drizzle, snow, wintry mix , freezing rain,\n",
    "df['Cloudy'] = df['Condition'].apply(lambda x: 3 if re.search(r'Mostly Cloudy', x) else 1 if re.search(r'Partly Cloudy', x) else 2 if re.search(r'Cloudy', x) else 0)\n",
    "df['Windy'] = df['Condition'].apply(lambda x: 1 if re.search(r'Windy', x) else 0)\n",
    "df['Fair'] = df['Condition'].apply(lambda x: 1 if re.search(r'Fair', x) else 0)\n",
    "df['Rain'] = df['Condition'].apply(lambda x: 3 if re.search(r'Heavy Rain', x) else 2 if re.search(r'Rain', x) else 1 if re.search(r'Light Rain', x) else 0)\n",
    "df['Fog'] = df['Condition'].apply(lambda x: 1 if re.search(r'Fog', x) else 0)\n",
    "df['Drizzle'] = df['Condition'].apply(lambda x: 2 if re.search(r'Drizzle', x) else 1 if re.search(r'Light Drizzle', x) else 0)\n",
    "df['Snow'] = df['Condition'].apply(lambda x: 3 if re.search(r'Heavy Snow', x) else 2 if re.search(r'Snow', x) else 1 if re.search(r'Light Snow', x) else 0)\n",
    "df['Wintry Mix'] = df['Condition'].apply(lambda x: 1 if re.search(r'Wintry Mix', x) else 0)\n",
    "df['Freezing Rain'] = df['Condition'].apply(lambda x: 2 if re.search(r'Freezing Rain', x) else 1 if re.search(r'Light Freezing Rain', x) else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ABQ': 0, 'ATL': 1, 'AUS': 2, 'BNA': 3, 'BOS': 4, 'BQN': 5, 'BTV': 6, 'BUF': 7, 'BUR': 8, 'BWI': 9, 'CHS': 10, 'CLE': 11, 'CLT': 12, 'CMH': 13, 'CVG': 14, 'DCA': 15, 'DEN': 16, 'DFW': 17, 'DTW': 18, 'EGE': 19, 'FLL': 20, 'HNL': 21, 'IAD': 22, 'IAH': 23, 'IND': 24, 'JAC': 25, 'JAX': 26, 'LAS': 27, 'LAX': 28, 'LGB': 29, 'MCO': 30, 'MIA': 31, 'MSP': 32, 'MSY': 33, 'OAK': 34, 'ONT': 35, 'ORD': 36, 'ORF': 37, 'ORH': 38, 'PBI': 39, 'PDX': 40, 'PHL': 41, 'PHX': 42, 'PIT': 43, 'PSE': 44, 'PSP': 45, 'PWM': 46, 'RDU': 47, 'RIC': 48, 'RNO': 49, 'ROC': 50, 'RSW': 51, 'SAN': 52, 'SAT': 53, 'SAV': 54, 'SEA': 55, 'SFO': 56, 'SJC': 57, 'SJU': 58, 'SLC': 59, 'SMF': 60, 'SRQ': 61, 'STT': 62, 'SYR': 63, 'TPA': 64}\n",
      "{'-1': 0, '-2': 1, '-3': 2, '0\\xa0': 3, '10': 4, '11': 5, '12': 6, '13': 7, '14': 8, '15': 9, '16': 10, '17': 11, '18': 12, '19': 13, '1\\xa0': 14, '20': 15, '21': 16, '22': 17, '23': 18, '24': 19, '25': 20, '26': 21, '27': 22, '28': 23, '29': 24, '2\\xa0': 25, '30': 26, '31': 27, '32': 28, '33': 29, '34': 30, '35': 31, '36': 32, '37': 33, '38': 34, '39': 35, '3\\xa0': 36, '40': 37, '41': 38, '42': 39, '43': 40, '44': 41, '45': 42, '46': 43, '47': 44, '48': 45, '49': 46, '4\\xa0': 47, '50': 48, '51': 49, '52': 50, '53': 51, '54': 52, '55': 53, '56': 54, '57': 55, '58': 56, '59': 57, '5\\xa0': 58, '67': 59, '6\\xa0': 60, '7\\xa0': 61, '8\\xa0': 62, '9\\xa0': 63}\n",
      "{'CALM': 0, 'E': 1, 'ENE': 2, 'ESE': 3, 'N': 4, 'NE': 5, 'NNE': 6, 'NNW': 7, 'NW': 8, 'S': 9, 'SE': 10, 'SSE': 11, 'SSW': 12, 'SW': 13, 'VAR': 14, 'W': 15, 'WNW': 16, 'WSW': 17, nan: 18}\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 28820 entries, 0 to 28819\n",
      "Data columns (total 37 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   MONTH              28820 non-null  int64  \n",
      " 1   DAY_OF_MONTH       28820 non-null  int64  \n",
      " 2   DAY_OF_WEEK        28820 non-null  int64  \n",
      " 3   OP_UNIQUE_CARRIER  28820 non-null  int32  \n",
      " 4   DEST               28820 non-null  int32  \n",
      " 5   DEP_DELAY          28820 non-null  int64  \n",
      " 6   CRS_ELAPSED_TIME   28820 non-null  int64  \n",
      " 7   DISTANCE           28820 non-null  int64  \n",
      " 8   CRS_DEP_M          28820 non-null  int64  \n",
      " 9   DEP_TIME_M         28820 non-null  int64  \n",
      " 10  CRS_ARR_M          28820 non-null  int64  \n",
      " 11  Temperature        28820 non-null  int64  \n",
      " 12  Dew Point          28820 non-null  int32  \n",
      " 13  Humidity           28820 non-null  int64  \n",
      " 14  Wind               28820 non-null  int32  \n",
      " 15  Wind Speed         28820 non-null  int64  \n",
      " 16  Wind Gust          28820 non-null  int64  \n",
      " 17  Pressure           28820 non-null  float64\n",
      " 18  Condition          28820 non-null  object \n",
      " 19  sch_dep            28820 non-null  int64  \n",
      " 20  sch_arr            28820 non-null  int64  \n",
      " 21  TAXI_OUT           28820 non-null  int64  \n",
      " 22  Cloudy             28820 non-null  int64  \n",
      " 23  Windy              28820 non-null  int64  \n",
      " 24  Fair               28820 non-null  int64  \n",
      " 25  Rain               28820 non-null  int64  \n",
      " 26  Fog                28820 non-null  int64  \n",
      " 27  Drizzle            28820 non-null  int64  \n",
      " 28  Snow               28820 non-null  int64  \n",
      " 29  Wintry Mix         28820 non-null  int64  \n",
      " 30  Freezing Rain      28820 non-null  int64  \n",
      " 31  MONTH_sin          28820 non-null  float64\n",
      " 32  MONTH_cos          28820 non-null  float64\n",
      " 33  DAY_OF_MONTH_sin   28820 non-null  float64\n",
      " 34  DAY_OF_MONTH_cos   28820 non-null  float64\n",
      " 35  DAY_OF_WEEK_sin    28820 non-null  float64\n",
      " 36  DAY_OF_WEEK_cos    28820 non-null  float64\n",
      "dtypes: float64(7), int32(4), int64(25), object(1)\n",
      "memory usage: 7.7+ MB\n",
      "None\n",
      "(28820, 37)\n"
     ]
    }
   ],
   "source": [
    "#Cyclic encoding for Day of week\n",
    "def encode(data, col, max_val):\n",
    "    data[col + '_sin'] = np.sin(2 * np.pi * data[col] / max_val)\n",
    "    data[col + '_cos'] = np.cos(2 * np.pi * data[col] / max_val)\n",
    "    return data\n",
    "\n",
    "\n",
    "df = encode(df, 'MONTH', 12)\n",
    "df = encode(df, 'DAY_OF_MONTH', 31)\n",
    "df = encode(df, 'DAY_OF_WEEK', 7)\n",
    "\n",
    "df_pre = df.copy()\n",
    "# make dep delay as binary\n",
    "df_pre['DEP_DELAY'] = df_pre['DEP_DELAY'].apply(lambda x: 1 if x > 15 else 0)\n",
    "\n",
    "# df_pre  = df.copy()\n",
    "lb = LabelEncoder()\n",
    "lb_dest = LabelEncoder()\n",
    "df_pre['DEST'] = lb_dest.fit_transform(df_pre['DEST'])\n",
    "lb_name_mapping_dest = dict(zip(lb_dest.classes_, lb_dest.transform(lb_dest.classes_)))\n",
    "print(lb_name_mapping_dest)\n",
    "lb_dew = LabelEncoder()\n",
    "df_pre['Dew Point'] = lb_dew.fit_transform(df_pre['Dew Point'])\n",
    "lb_name_mapping_dew = dict(zip(lb_dew.classes_, lb_dew.transform(lb_dew.classes_)))\n",
    "print(lb_name_mapping_dew)\n",
    "lb_wind = LabelEncoder()\n",
    "df_pre['Wind'] = lb_wind.fit_transform(df_pre['Wind'])\n",
    "lb_name_mapping_wind = dict(zip(lb_wind.classes_, lb_wind.transform(lb_wind.classes_)))\n",
    "print(lb_name_mapping_wind)\n",
    "\n",
    "df_pre.drop(['TAIL_NUM'], axis=1, inplace=True)\n",
    "x = df_pre.drop(['DEP_DELAY'], axis=1)\n",
    "y = df_pre['DEP_DELAY']\n",
    "\n",
    "print(df_pre.info())\n",
    "print(df_pre.shape)\n",
    "\n",
    "df_pre.drop(['Condition'], axis=1, inplace=True)\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "standardScaler = StandardScaler()\n",
    "standardScaler.fit(df_pre)\n",
    "\n",
    "df_pre.head()\n",
    "# save df_pre as a new csv file\n",
    "df_pre.to_csv('df_preprocessed.csv', index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}