listings-combiner-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/ual/rental-listings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd, os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rents = pd.DataFrame()\n",
    "data_dir = 'raw-data/'\n",
    "filenames = os.listdir(data_dir)\n",
    "len(filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "raw-data/craigslist-2014-05-12-02-13-44.csv\n",
      "raw-data/craigslist-2014-05-13-02-38-06.csv\n",
      "raw-data/craigslist-2014-05-14-02-31-08.csv\n",
      "raw-data/craigslist-2014-05-15-02-29-54.csv\n",
      "raw-data/craigslist-2014-05-16-02-34-04.csv\n",
      "raw-data/craigslist-2014-05-17-02-32-27.csv\n",
      "raw-data/craigslist-2014-05-18-02-21-14.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Paul/anaconda/envs/py3/lib/python3.5/site-packages/IPython/core/interactiveshell.py:2705: DtypeWarning: Columns (4,7) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "raw-data/craigslist-2014-05-19-02-18-45.csv\n",
      "raw-data/craigslist-2014-05-20-02-41-19.csv\n",
      "raw-data/craigslist-2014-05-21-02-32-37.csv\n",
      "raw-data/craigslist-2014-05-22-02-33-41.csv\n",
      "raw-data/craigslist-2014-05-23-02-34-37.csv\n",
      "raw-data/craigslist-2014-05-24-02-27-51.csv\n",
      "raw-data/craigslist-2014-05-25-02-20-32.csv\n",
      "raw-data/craigslist-2014-05-26-02-16-15.csv\n",
      "raw-data/craigslist-2014-05-27-02-18-31.csv\n",
      "raw-data/craigslist-2014-05-28-02-41-59.csv\n",
      "raw-data/craigslist-2014-05-29-02-28-02.csv\n",
      "raw-data/craigslist-2014-05-30-02-34-32.csv\n",
      "raw-data/craigslist-2014-05-31-02-24-10.csv\n",
      "raw-data/craigslist-2014-06-01-02-21-23.csv\n",
      "raw-data/craigslist-2014-06-02-02-19-26.csv\n",
      "raw-data/craigslist-2014-06-03-02-34-37.csv\n",
      "raw-data/craigslist-2014-06-04-02-28-32.csv\n",
      "raw-data/craigslist-2014-06-05-02-18-02.csv\n",
      "raw-data/craigslist-2014-06-06-02-10-19.csv\n",
      "raw-data/craigslist-2014-06-07-02-26-29.csv\n",
      "raw-data/craigslist-2014-06-08-02-20-38.csv\n",
      "raw-data/craigslist-2014-06-09-02-17-01.csv\n",
      "raw-data/craigslist-2014-06-10-02-40-34.csv\n",
      "raw-data/craigslist-2014-06-11-02-30-45.csv\n",
      "raw-data/craigslist-2014-06-12-02-36-22.csv\n",
      "raw-data/craigslist-2014-06-13-02-30-38.csv\n",
      "raw-data/craigslist-2014-06-14-02-27-08.csv\n",
      "raw-data/craigslist-2014-06-15-02-20-08.csv\n",
      "raw-data/craigslist-2014-06-16-02-14-00.csv\n",
      "raw-data/craigslist-2014-06-17-02-39-25.csv\n",
      "raw-data/craigslist-2014-06-18-02-30-51.csv\n",
      "raw-data/craigslist-2014-06-19-02-32-47.csv\n",
      "raw-data/craigslist-2014-06-20-02-30-17.csv\n",
      "raw-data/craigslist-2014-06-21-02-48-30.csv\n",
      "raw-data/craigslist-2014-06-22-02-24-10.csv\n",
      "raw-data/craigslist-2014-06-23-02-20-48.csv\n",
      "raw-data/craigslist-2014-06-24-02-46-12.csv\n",
      "raw-data/craigslist-2014-06-25-02-36-51.csv\n",
      "raw-data/craigslist-2014-06-26-02-32-06.csv\n",
      "raw-data/craigslist-2014-06-27-02-27-32.csv\n",
      "raw-data/craigslist-2014-06-28-02-29-11.csv\n",
      "raw-data/craigslist-2014-06-29-02-19-23.csv\n",
      "raw-data/craigslist-2014-06-30-02-16-57.csv\n",
      "raw-data/craigslist-2014-07-01-02-36-07.csv\n",
      "raw-data/craigslist-2014-07-02-02-27-36.csv\n",
      "raw-data/craigslist-2014-07-03-02-31-12.csv\n",
      "raw-data/craigslist-2014-07-04-02-24-21.csv\n",
      "raw-data/craigslist-2014-07-05-02-11-07.csv\n",
      "raw-data/craigslist-2014-07-06-02-18-34.csv\n",
      "raw-data/craigslist-2014-07-07-02-16-34.csv\n",
      "raw-data/craigslist-2014-07-08-02-31-57.csv\n",
      "raw-data/craigslist-2014-07-09-02-25-55.csv\n",
      "raw-data/craigslist-2014-07-10-02-28-42.csv\n",
      "raw-data/craigslist-2014-07-11-02-24-53.csv\n",
      "raw-data/craigslist-2014-07-12-02-27-25.csv\n",
      "raw-data/craigslist-2014-07-13-02-19-27.csv\n",
      "raw-data/craigslist-2014-07-14-02-14-03.csv\n",
      "raw-data/craigslist-2014-07-15-02-11-01.csv\n",
      "raw-data/craigslist-2014-07-16-02-11-39.csv\n",
      "raw-data/craigslist-2014-07-17-02-17-35.csv\n",
      "raw-data/craigslist-2014-07-18-02-29-10.csv\n"
     ]
    }
   ],
   "source": [
    "# for every csv file in the data directory, load it and add it to the dataframe\n",
    "for filename in filenames:\n",
    "    if filename.startswith('craigs'):\n",
    "        f = data_dir+filename\n",
    "        print(f)\n",
    "        rents = pd.concat([rents, pd.read_csv(f)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10958376, 11)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rents.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# drop any rows that lack a 'link' value\n",
    "rents = rents[pd.notnull(rents['link'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10958373"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save the final combined dataframe as a csv file\n",
    "rents.to_csv('usa.csv', index=False, encoding='utf-8')\n",
    "len(rents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}