data exploration and munging.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import subprocess\n",
    "import re\n",
    "import json\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "events = [{'hc': 'Tu BiShvat', 'short': 'tb'},\n",
    "          {'hc': 'Erev Purim', 'short': 'pu'},\n",
    "          {'hc': 'Erev Pesach', 'short': 'pe'},\n",
    "          {'hc': 'Erev Shavuot', 'short': 'sh'},\n",
    "          {'hc': 'Erev Tish\\'a B\\'Av', 'short': 'ta'},\n",
    "          {'hc': 'Erev Rosh Hashana', 'short': 'r'},\n",
    "          {'hc': 'Erev Yom Kippur', 'short': 'y'},\n",
    "          {'hc': 'Erev Sukkot', 'short': 'su'},\n",
    "          {'hc': 'Chanukah: 1 Candle', 'short': 'c'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [],
   "source": [
    "one_day = datetime.timedelta(days=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dates(year):\n",
    "    hebcal_str = subprocess.run('../hebcal-master/./hebcal ' + str(year), shell=True, stdout=subprocess.PIPE, encoding='utf8').stdout\n",
    "    results = {}\n",
    "    heb_year = int(re.search(r'Rosh Hashana ([0-9]{4})', hebcal_str).group(1))\n",
    "    leaps = [3, 6, 8, 11, 14, 17, 0]\n",
    "    rh_date = None\n",
    "\n",
    "    for i, event in enumerate(events):\n",
    "        date_search = re.search(r'(([0-9]{1,2})/([0-9]{1,2})/[0-9]{4}) ' + event['hc'], hebcal_str)\n",
    "        month = int(date_search.group(2))\n",
    "        day = int(date_search.group(3))\n",
    "        if event['hc'] == 'Tu BiShvat':\n",
    "            tb_date = datetime.date(year, month, day) - one_day\n",
    "            month = tb_date.month\n",
    "            day = tb_date.day\n",
    "        results[event['short']] = {'e': event['short'], 'y': year, 'm': month, 'd': day}\n",
    "        if event['hc'] == 'Erev Rosh Hashana': rh_date = date(year, month, day)\n",
    "    \n",
    "    for event_name in results:\n",
    "        this_event = results[event_name]\n",
    "        is_prev = date(this_event['y'], this_event['m'], this_event['d']) < rh_date\n",
    "        this_heb_year = heb_year - (1 if is_prev else 0)\n",
    "        this_event['hy'] = this_heb_year\n",
    "        this_event['l'] = (this_heb_year % 19) in leaps\n",
    "    return results.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates = []\n",
    "for y in range(1583, 2400):\n",
    "    dates.extend(get_dates(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day</th>\n",
       "      <th>event</th>\n",
       "      <th>heb_year</th>\n",
       "      <th>leap</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [day, event, heb_year, leap, month, year]\n",
       "Index: []"
      ]
     },
     "execution_count": 347,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(dates).query('leap == False & event == \"Erev Pesach\" & month == 4 & day > 19')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export raw data\n",
    "with open('data.json', 'w') as file:\n",
    "    json.dump(dates, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "event    Tu BiShvat\n",
       "month            12\n",
       "day              31\n",
       "count            17\n",
       "dtype: object"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# export aggregated data\n",
    "pd.DataFrame(pd.DataFrame(dates).groupby(['event', 'month', 'day']).size(), columns=['count']).reset_index().to_json('agg_data.json', orient='records')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}