time_of_highest_activity_for_each_city.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import load_stop_times_df\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "from joypy import joyplot\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 9/400 [00:01<00:38, 10.14it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (3) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 40%|████      | 161/400 [00:58<06:14,  1.57s/it]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (5) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 42%|████▏     | 166/400 [01:00<02:36,  1.49it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (3) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 54%|█████▍    | 216/400 [01:30<01:01,  2.97it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (5) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 56%|█████▌    | 222/400 [01:31<00:45,  3.94it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (0,3) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 65%|██████▍   | 259/400 [01:47<01:08,  2.05it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (5) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 68%|██████▊   | 272/400 [01:49<00:14,  8.54it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (3) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 73%|███████▎  | 293/400 [02:00<01:00,  1.76it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (0) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 83%|████████▎ | 331/400 [02:05<00:12,  5.46it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (3) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      " 96%|█████████▋| 385/400 [02:46<00:04,  3.27it/s]e:\\TSA\\Datascience2\\utils.py:32: DtypeWarning: Columns (10) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(path, sep = \",\", header = 0)\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "C:\\Users\\Lauris\\AppData\\Local\\Temp\\ipykernel_3312\\616493329.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
      "100%|██████████| 400/400 [03:09<00:00,  2.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                         name  time\n",
      "0  data\\US_Alabama_Birmingham     9\n",
      "1  data\\US_Alabama_Huntsville    17\n",
      "2  data\\US_Alabama_Montgomery     8\n",
      "3    data\\US_Alaska_Anchorage    17\n",
      "4       data\\US_Alaska_Juneau    16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "result_list = []\n",
    "all_data = []\n",
    "\n",
    "def get_data(city):\n",
    "\n",
    "    df = load_stop_times_df(os.path.basename(city))\n",
    "\n",
    "    df[\"arrival_time\"] = pd.to_datetime(df.arrival_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
    "    df[\"departure_time\"] = pd.to_datetime(df.departure_time, errors = \"coerce\", format = \"%H:%M:%S\")\n",
    "\n",
    "    resdict = {}\n",
    "\n",
    "    for i in range(0, 24):\n",
    "\n",
    "        # year, month, day, uhrzeit\n",
    "        time_format = '%m/%d/%Y %H:%M:%S'\n",
    "\n",
    "        # startuhrzeit\n",
    "        hour = datetime(1900, 1, 1, i,0,0 )\n",
    "\n",
    "        # enduhrzeit, da 24 Uhr gleich 0 sein muss, \n",
    "        # Datensatz ist komisch, deswegen zwischen 23 und 0 Uhr keine Fahrten\n",
    "        if i != 24:\n",
    "            hour_later = hour + pd.Timedelta(hours = 1)\n",
    "            #print(hour_later)\n",
    "        else:\n",
    "            hour_later = datetime.strptime(\"01/01/1900 \" + \"00:00:00\", time_format)\n",
    "\n",
    "        number = len(df.query(\"@hour <= arrival_time <= @hour_later\"))\n",
    "        \n",
    "        # print(\"von\", hour, \"bis\", hour_later, number)\n",
    "\n",
    "        resdict[i] = number\n",
    "\n",
    "    # we dont have values between 23 and 24, datetimes are hard, this aproximates it and makes the graph prettier :)\n",
    "    resdict[24] = resdict[0]\n",
    "\n",
    "    reslist = []\n",
    "    for i in resdict.values():\n",
    "        reslist.append(i)\n",
    "\n",
    "    x = list(resdict.keys())\n",
    "    y = list(resdict.values())\n",
    "\n",
    "    # peak time \n",
    "    peakx = np.argmax(y)\n",
    "\n",
    "    result_list.append([city, peakx])\n",
    "\n",
    "    # append all data to a dict\n",
    "    all_data.append([city, x, y])\n",
    "    return\n",
    "\n",
    "all_city_paths = glob(\"data/*\")\n",
    "all_city_paths.pop(0)\n",
    "\n",
    "for city_name in tqdm(all_city_paths):\n",
    "    try:\n",
    "        get_data(city_name)\n",
    "    except Exception as e:\n",
    "        pass\n",
    "\n",
    "fulldf = pd.DataFrame(all_data, columns = [\"name\", \"x\",\"y\"])\n",
    "\n",
    "res_df = pd.DataFrame(result_list, columns = [\"name\",\"time\"])\n",
    "print(res_df.head())\n",
    "\n",
    "res_df.to_csv(\"peakusage.csv\")\n",
    "fulldf.to_csv(\"fullusagedata.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}