jupyter/TwitterTrends-1-TrendsToFile.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TwitterTrends-1-TrendsToFile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[32mimport \u001b[39m\u001b[36m$ivy.$                                   \r\n",
       "\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36m$ivy.$                                   \r\n",
       "\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36m$ivy.$                                    \u001b[39m"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import $ivy.`org.twitter4j:twitter4j-core:4.0.6`\n",
    "\n",
    "import $ivy.`com.typesafe.play::play-json:2.6.2`\n",
    "\n",
    "import $ivy.`org.apache.commons:commons-io:1.3.2`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[32mimport \u001b[39m\u001b[36mtwitter4j.conf.ConfigurationBuilder\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mtwitter4j.Twitter\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mtwitter4j.TwitterFactory\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mtwitter4j.Location\r\n",
       "\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mscala.collection.JavaConverters._\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mscala.util.control.Breaks._\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mscala.collection.mutable.ListBuffer\r\n",
       "\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mjava.io._\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mjava.util.Calendar\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mjava.net.URLEncoder\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mjava.text.SimpleDateFormat\r\n",
       "\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36mplay.api.libs.json._\r\n",
       "\r\n",
       "\u001b[39m\r\n",
       "\u001b[32mimport \u001b[39m\u001b[36morg.apache.commons.io.FileUtils\u001b[39m"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import twitter4j.conf.ConfigurationBuilder\n",
    "import twitter4j.Twitter\n",
    "import twitter4j.TwitterFactory\n",
    "import twitter4j.Location\n",
    "\n",
    "import scala.collection.JavaConverters._\n",
    "import scala.util.control.Breaks._\n",
    "import scala.collection.mutable.ListBuffer\n",
    "\n",
    "import java.io._\n",
    "import java.util.Calendar\n",
    "import java.net.URLEncoder\n",
    "import java.text.SimpleDateFormat\n",
    "\n",
    "import play.api.libs.json._\n",
    "\n",
    "import org.apache.commons.io.FileUtils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting Twitter API Keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val consumerKey = \"YOU_CONSUMER_KEY\"\n",
    "val consumerSecret = \"YOU_CONSUMER_SECRET\"\n",
    "val accessToken = \"YOU_ACCESS_TOKEN\"\n",
    "val accessTokenSecret = \"YOU_TOKEN_SECRET\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Note: If you are not using Jupyter better load this tokens in a [more proper way](https://danielasfregola.com/2015/06/01/loading-configurations-in-scala/) and use a code like this in order to load the properties:*\n",
    "\n",
    "```Scala\n",
    "def loadProperties() {\n",
    "  val consumerKey = ConfigFactory.load().getString(\"twitter.consumer.key\")\n",
    "  val consumerSecret = ConfigFactory.load().getString(\"twitter.consumer.secret\")\n",
    "  val accessToken = ConfigFactory.load().getString(\"twitter.access.key\")\n",
    "  val accessTokenSecret = ConfigFactory.load().getString(\"twitter.access.secret\")\n",
    "}\n",
    "\n",
    "loadProperties()\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instantiate objects and variables\n",
    "\n",
    "*getTwitterInstance* will load the Twitter API Keys we just have set and will create the [TwitterFactory](http://twitter4j.org/javadoc/twitter4j/TwitterFactory.html), which will allow us to comunicate with the [Twitter API](https://dev.twitter.com/overview/api)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTwitterInstance: Twitter = {\n",
    "  val cb = new ConfigurationBuilder()\n",
    "  cb.setDebugEnabled(true)\n",
    "    .setOAuthConsumerKey(consumerKey)\n",
    "    .setOAuthConsumerSecret(consumerSecret)\n",
    "    .setOAuthAccessToken(accessToken)\n",
    "    .setOAuthAccessTokenSecret(accessTokenSecret)\n",
    "  return new TwitterFactory(cb.build()).getInstance\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set all variables needed in the process:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[36mtwitter\u001b[39m: \u001b[32mTwitter\u001b[39m = TwitterImpl{INCLUDE_MY_RETWEET=PostParameter{name='include_my_retweet', value='true', file=null, fileBody=null}}\r\n",
       "\u001b[36mavailableTrends\u001b[39m: \u001b[32mtwitter4j\u001b[39m.\u001b[32mResponseList\u001b[39m[\u001b[32mLocation\u001b[39m] = null\r\n",
       "\u001b[36mplaceTrends\u001b[39m: \u001b[32mtwitter4j\u001b[39m.\u001b[32mTrends\u001b[39m = null\r\n",
       "\u001b[36mSEP\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"|\"\u001b[39m\r\n",
       "\u001b[36mkey\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mnumTowns\u001b[39m: \u001b[32mInt\u001b[39m = \u001b[32m0\u001b[39m\r\n",
       "\u001b[36mMAX_NUM_TOWNS\u001b[39m: \u001b[32mInt\u001b[39m = \u001b[32m5\u001b[39m\r\n",
       "\u001b[36mplaceName\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mwoeid\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mcountryName\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mcountryNameURL\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mcityName\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mcityNameURL\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mresponse\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mresponseJson\u001b[39m: \u001b[32mJsValue\u001b[39m = null\r\n",
       "\u001b[36mlon\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mlat\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mtrendsDate\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mtrendsDateFormat\u001b[39m: \u001b[32mSimpleDateFormat\u001b[39m = java.text.SimpleDateFormat@72dd7cbc\r\n",
       "\u001b[36mfilenameDateFormat\u001b[39m: \u001b[32mSimpleDateFormat\u001b[39m = java.text.SimpleDateFormat@6da3d6ee\r\n",
       "\u001b[36mkeyDateFormat\u001b[39m: \u001b[32mSimpleDateFormat\u001b[39m = java.text.SimpleDateFormat@b184d160\r\n",
       "\u001b[36mtrendsList\u001b[39m: \u001b[32mListBuffer\u001b[39m[\u001b[32mString\u001b[39m] = \u001b[33mListBuffer\u001b[39m()\r\n",
       "\u001b[36mtrends\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mtrendLine\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\u001b[39m\r\n",
       "\u001b[36mfDateString\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"_2017-08-10-234037\"\u001b[39m\r\n",
       "\u001b[36mkeyDateString\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"20170810234037\"\u001b[39m\r\n",
       "\u001b[36mnL\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"\"\"\n",
       "\r\n",
       "\n",
       "\"\"\"\u001b[39m"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val twitter = getTwitterInstance\n",
    "var availableTrends : twitter4j.ResponseList[twitter4j.Location] = null\n",
    "var placeTrends : twitter4j.Trends = null\n",
    "val SEP = \"|\"\n",
    "var key = \"\"\n",
    "var numTowns = 0\n",
    "val MAX_NUM_TOWNS = 5\n",
    "var placeName = \"\"\n",
    "var woeid = \"\"\n",
    "var countryName = \"\"\n",
    "var countryNameURL = \"\"\n",
    "var cityName = \"\"\n",
    "var cityNameURL = \"\"\n",
    "var response = \"\"\n",
    "var responseJson : JsValue = null\n",
    "var lon = \"\"\n",
    "var lat = \"\"\n",
    "var trendsDate = \"\"\n",
    "val trendsDateFormat = new SimpleDateFormat(\"HH:mm:ss|dd/MM/yyyy\") // date format for the 'date' field included in textfile\n",
    "val filenameDateFormat = new SimpleDateFormat(\"_YYYY-MM-dd-HHmmss\") // date format for the filename\n",
    "val keyDateFormat = new SimpleDateFormat(\"YYYYMMddHHmmss\") // date format for the 'key' field included in textfile\n",
    "var trendsList = new ListBuffer[String]()\n",
    "var trends = \"\"\n",
    "var trendLine = \"\"\n",
    "var fDateString = filenameDateFormat.format(Calendar.getInstance().getTime())\n",
    "var keyDateString = keyDateFormat.format(Calendar.getInstance().getTime())\n",
    "val nL = sys.props(\"line.separator\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defined \u001b[32mfunction\u001b[39m \u001b[36mmyGetPlaceTrends\u001b[39m\r\n",
       "defined \u001b[32mfunction\u001b[39m \u001b[36mget\u001b[39m\r\n",
       "defined \u001b[32mfunction\u001b[39m \u001b[36mgetLocationData\u001b[39m\r\n",
       "defined \u001b[32mfunction\u001b[39m \u001b[36misValidTown\u001b[39m"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// Returns the top 10 trending topics for a specific WOEID\n",
    "def myGetPlaceTrends(woeid: String) : String = {\n",
    "    placeTrends = twitter.trends.getPlaceTrends(woeid.toInt)\n",
    "    trendsDate = trendsDateFormat.format(placeTrends.getTrendAt())\n",
    "    trendsList = new ListBuffer[String]()\n",
    "\n",
    "    for (trend <- placeTrends.getTrends()) {\n",
    "        trendsList += trend.getName()\n",
    "    }\n",
    "\n",
    "    trends = trendsDate + SEP + (trendsList.mkString(\",\"))\n",
    "    return trends\n",
    "}\n",
    "\n",
    "// Returns the response from the called webservice\n",
    "@throws(classOf[java.io.IOException])\n",
    "def get(url: String) : String = scala.io.Source.fromURL(url, \"UTF-8\").mkString\n",
    "\n",
    "\n",
    "// Returns specific location's trendic topic info: country, city, woeid, longitude and latitude \n",
    "def getLocationData(location: Location) : String = {\n",
    "    woeid = location.getWoeid.toString\n",
    "    countryName = location.getCountryName\n",
    "    countryNameURL = URLEncoder.encode(countryName, \"UTF-8\")\n",
    "    cityName = location.getName\n",
    "    cityNameURL = URLEncoder.encode(cityName, \"UTF-8\")\n",
    "    lon = \"\"\n",
    "    lat = \"\"\n",
    "\n",
    "    try {\n",
    "        response = get(\"http://nominatim.openstreetmap.org/search?country=\" + countryNameURL + \"&city=\" + cityNameURL + \"&limit=1&format=json\")\n",
    "        responseJson = Json.parse(response.drop(1).dropRight(1)) // delete the first ([) and last char (] returned by Twitter)\n",
    "        lon = responseJson(\"lon\").toString.replace(\"\\\"\", \"\") // delete \" chars returned by Twitter\n",
    "        lat = responseJson(\"lat\").toString.replace(\"\\\"\", \"\") // delete \" chars returned by Twitter\n",
    "    } catch {\n",
    "        case ioe: java.io.IOException => return(\"ERROR: Got java.io.IOException: \" + ioe) // TODO: Exception handling\n",
    "        case ste: java.net.SocketTimeoutException => return(\"ERROR: Got java.net.SocketTimeoutException: \" + ste) // TODO: Exception handling\n",
    "    }\n",
    "\n",
    "    return countryName + SEP + cityName + SEP + woeid + SEP + lon + SEP + lat\n",
    "}\n",
    "\n",
    "\n",
    "// Twitter's response has \n",
    "def isValidTown(location: Location) : Boolean = {\n",
    "    placeName = location.getPlaceName\n",
    "    if (placeName.equals(\"Town\")) return true\n",
    "    else return false\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main program"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will do a loop in which, every 10 minutes, we obtain the top trending topic from 5 random cities. \n",
    "\n",
    "The 2 main methods are:\n",
    "- [getAvailableTrends()](http://twitter4j.org/javadoc/twitter4j/api/TrendsResources.html#getAvailableTrends--), [available](https://dev.twitter.com/rest/reference/get/trends/available) in Twitter API: Returns the locations that Twitter has trending topic information for.\n",
    "- [getPlaceTrends(int woeid)](http://twitter4j.org/javadoc/twitter4j/api/TrendsResources.html#getPlaceTrends-int-), [place](https://dev.twitter.com/rest/reference/get/trends/place) in Twitter API: Returns the top 10 trending topics for a specific WOEID, if trending information is available for it.\n",
    "\n",
    "The rest of the code is basically some data quality and formatting strings.\n",
    "\n",
    "*Note: You cat set the loop time as you want, but take care about limit usage. Both methods are **rate limited** as stated in \"Resource Information\" section from Twitter documentation*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start batch\n",
      "Finish batch, waiting 10 min till next one...\n",
      "Start batch\n",
      "Finish batch, waiting 10 min till next one...\n",
      "Start batch\n",
      "Finish batch, waiting 10 min till next one...\n"
     ]
    }
   ],
   "source": [
    "while (true){\n",
    "    println(\"Start batch\")\n",
    "    \n",
    "    fDateString = filenameDateFormat.format(Calendar.getInstance().getTime()) // get current date with desired format\n",
    "    var writer = new PrintWriter(new File(\"../datasets/trendFiles/tmp/trends\" + fDateString + \".csv\")) // open new file for writing\n",
    "\n",
    "    numTowns = 0\n",
    "\n",
    "    // Returns the locations that Twitter has trending topic information for\n",
    "    availableTrends = twitter.trends.getAvailableTrends()\n",
    "\n",
    "    // The next location loop must be breakable (we must break once we find 5 cities)\n",
    "    breakable {\n",
    "        for (location <- scala.util.Random.shuffle(availableTrends.asScala)) {\n",
    "            trendLine = \"\"\n",
    "\n",
    "            // We will only get 5 towns\n",
    "            if (numTowns < MAX_NUM_TOWNS) {\n",
    "               if (isValidTown(location) == true) {\n",
    "                    keyDateString = keyDateFormat.format(Calendar.getInstance().getTime())\n",
    "                   // create a unique key based on date and some other info\n",
    "                    key = (numTowns + 1).toString + \"-\" + keyDateString + \"-\" + java.util.UUID.randomUUID.toString\n",
    "                    trendLine += key + \";\" + getLocationData(location)\n",
    "                    // Write only valid info with no error\n",
    "                    if (!trendLine.contains(\"ERROR:\")){\n",
    "                        trendLine += SEP + myGetPlaceTrends(woeid)\n",
    "                        writer.write(trendLine + nL)\n",
    "                        numTowns += 1\n",
    "                    }\n",
    "                }\n",
    "            } else {\n",
    "                break\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "    \n",
    "    writer.flush\n",
    "    writer.close\n",
    "\n",
    "    FileUtils.copyFileToDirectory(new File(\"../datasets/trendFiles/tmp/trends\" + fDateString + \".csv\"), new File(\"../datasets/trendFiles/\"))\n",
    "    FileUtils.forceDelete(new File(\"../datasets/trendFiles/tmp/trends\" + fDateString + \".csv\"))\n",
    "\n",
    "    println(\"Finish batch, waiting 10 min till next one...\")\n",
    "    Thread.sleep(600000) //wait 10 minutes\n",
    "}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Scala 2.11.11",
   "language": "scala",
   "name": "scala"
  },
  "language_info": {
   "codemirror_mode": "text/x-scala",
   "file_extension": ".scala",
   "mimetype": "text/x-scala",
   "name": "scala211",
   "nbconvert_exporter": "script",
   "pygments_lexer": "scala",
   "version": "2.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}