WebScraping RT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "── \u001b[1mAttaching packages\u001b[22m ─────────────────────────────────────── tidyverse 1.3.0 ──\n",
      "\n",
      "\u001b[32m✔\u001b[39m \u001b[34mggplot2\u001b[39m 3.3.2     \u001b[32m✔\u001b[39m \u001b[34mpurrr  \u001b[39m 0.3.4\n",
      "\u001b[32m✔\u001b[39m \u001b[34mtibble \u001b[39m 3.0.3     \u001b[32m✔\u001b[39m \u001b[34mdplyr  \u001b[39m 1.0.1\n",
      "\u001b[32m✔\u001b[39m \u001b[34mtidyr  \u001b[39m 1.1.1     \u001b[32m✔\u001b[39m \u001b[34mstringr\u001b[39m 1.4.0\n",
      "\u001b[32m✔\u001b[39m \u001b[34mreadr  \u001b[39m 1.3.1     \u001b[32m✔\u001b[39m \u001b[34mforcats\u001b[39m 0.5.0\n",
      "\n",
      "── \u001b[1mConflicts\u001b[22m ────────────────────────────────────────── tidyverse_conflicts() ──\n",
      "\u001b[31m✖\u001b[39m \u001b[34mdplyr\u001b[39m::\u001b[32mfilter()\u001b[39m masks \u001b[34mstats\u001b[39m::filter()\n",
      "\u001b[31m✖\u001b[39m \u001b[34mdplyr\u001b[39m::\u001b[32mlag()\u001b[39m    masks \u001b[34mstats\u001b[39m::lag()\n",
      "\n",
      "Loading required package: xml2\n",
      "\n",
      "\n",
      "Attaching package: ‘rvest’\n",
      "\n",
      "\n",
      "The following object is masked from ‘package:purrr’:\n",
      "\n",
      "    pluck\n",
      "\n",
      "\n",
      "The following object is masked from ‘package:readr’:\n",
      "\n",
      "    guess_encoding\n",
      "\n",
      "\n",
      "\n",
      "Attaching package: ‘magrittr’\n",
      "\n",
      "\n",
      "The following object is masked from ‘package:purrr’:\n",
      "\n",
      "    set_names\n",
      "\n",
      "\n",
      "The following object is masked from ‘package:tidyr’:\n",
      "\n",
      "    extract\n",
      "\n",
      "\n",
      "\n",
      "Attaching package: ‘glue’\n",
      "\n",
      "\n",
      "The following object is masked from ‘package:dplyr’:\n",
      "\n",
      "    collapse\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Importing the required libraries\n",
    "library(tidyverse)\n",
    "library(tidyverse)\n",
    "library(rvest)\n",
    "library(tidytext)\n",
    "library(dplyr)\n",
    "library(polite) \n",
    "library(magrittr) # better handling of pipes\n",
    "library(purrr) # to work with lists and map functionsmusic_text\n",
    "library(glue) # to paste strings\n",
    "library(stringr) # to hand strings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Url to get the top 100 movies of all time\n",
    "#url_base <-\"https://www.rottentomatoes.com/top/bestofrt/\"\n",
    "#page_html <- read_html(url_base)\n",
    "\n",
    "# URL to get the stats for the year 2019\n",
    "url_string = \"https://www.rottentomatoes.com/top/bestofrt/\"\n",
    "page_html <- read_html(url_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracting rating - .tMeterScore\n",
    "movie_rating <- page_html %>%\n",
    "   html_nodes(\".tMeterScore\")  %>% html_text()\n",
    "#head(movie_rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<caption>A tibble: 5 × 1</caption>\n",
       "<thead>\n",
       "\t<tr><th scope=col>movie_rating</th></tr>\n",
       "\t<tr><th scope=col>&lt;dbl&gt;</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "\t<tr><td> 9.8</td></tr>\n",
       "\t<tr><td> 9.9</td></tr>\n",
       "\t<tr><td> 9.7</td></tr>\n",
       "\t<tr><td>10.0</td></tr>\n",
       "\t<tr><td> 9.6</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "A tibble: 5 × 1\n",
       "\\begin{tabular}{l}\n",
       " movie\\_rating\\\\\n",
       " <dbl>\\\\\n",
       "\\hline\n",
       "\t  9.8\\\\\n",
       "\t  9.9\\\\\n",
       "\t  9.7\\\\\n",
       "\t 10.0\\\\\n",
       "\t  9.6\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "A tibble: 5 × 1\n",
       "\n",
       "| movie_rating &lt;dbl&gt; |\n",
       "|---|\n",
       "|  9.8 |\n",
       "|  9.9 |\n",
       "|  9.7 |\n",
       "| 10.0 |\n",
       "|  9.6 |\n",
       "\n"
      ],
      "text/plain": [
       "  movie_rating\n",
       "1  9.8        \n",
       "2  9.9        \n",
       "3  9.7        \n",
       "4 10.0        \n",
       "5  9.6        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "movie_rating<- movie_rating[27:126]\n",
    "movie_rating <- str_trim(gsub(\"%\",0,movie_rating))\n",
    "movie_rating <- tibble(movie_rating = str_trim(gsub(\"[\\r\\n\\t]\",\"\",movie_rating)))\n",
    "movie_rating$movie_rating <- as.numeric(as.character(movie_rating$movie_rating))\n",
    "movie_rating$movie_rating <- round(movie_rating$movie_rating/100,2)\n",
    "head(movie_rating,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "100"
      ],
      "text/latex": [
       "100"
      ],
      "text/markdown": [
       "100"
      ],
      "text/plain": [
       "[1] 100"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nrow(movie_rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracting movie title - .table a\n",
    "movie_title <- page_html %>%\n",
    "   html_nodes(\".table a\")  %>% html_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<caption>A tibble: 6 × 1</caption>\n",
       "<thead>\n",
       "\t<tr><th scope=col>Movie_names</th></tr>\n",
       "\t<tr><th scope=col>&lt;chr&gt;</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "\t<tr><td>Black Panther     </td></tr>\n",
       "\t<tr><td>Avengers: Endgame </td></tr>\n",
       "\t<tr><td>Us                </td></tr>\n",
       "\t<tr><td>Toy Story 4       </td></tr>\n",
       "\t<tr><td>The Wizard of Oz  </td></tr>\n",
       "\t<tr><td>Lady Bird         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "A tibble: 6 × 1\n",
       "\\begin{tabular}{l}\n",
       " Movie\\_names\\\\\n",
       " <chr>\\\\\n",
       "\\hline\n",
       "\t Black Panther     \\\\\n",
       "\t Avengers: Endgame \\\\\n",
       "\t Us                \\\\\n",
       "\t Toy Story 4       \\\\\n",
       "\t The Wizard of Oz  \\\\\n",
       "\t Lady Bird         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "A tibble: 6 × 1\n",
       "\n",
       "| Movie_names &lt;chr&gt; |\n",
       "|---|\n",
       "| Black Panther      |\n",
       "| Avengers: Endgame  |\n",
       "| Us                 |\n",
       "| Toy Story 4        |\n",
       "| The Wizard of Oz   |\n",
       "| Lady Bird          |\n",
       "\n"
      ],
      "text/plain": [
       "  Movie_names       \n",
       "1 Black Panther     \n",
       "2 Avengers: Endgame \n",
       "3 Us                \n",
       "4 Toy Story 4       \n",
       "5 The Wizard of Oz  \n",
       "6 Lady Bird         "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# extraction of Movie Name\n",
    "mov_name <- str_trim(gsub(\"[\\r\\n\\t]\",\"\",movie_title))\n",
    "mov_name <- gsub('.{6}$', '', mov_name)\n",
    "movie_name <- tibble(Movie_names = mov_name)\n",
    "head(movie_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<caption>A tibble: 6 × 1</caption>\n",
       "<thead>\n",
       "\t<tr><th scope=col>Year</th></tr>\n",
       "\t<tr><th scope=col>&lt;chr&gt;</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "\t<tr><td>2018</td></tr>\n",
       "\t<tr><td>2019</td></tr>\n",
       "\t<tr><td>2019</td></tr>\n",
       "\t<tr><td>2019</td></tr>\n",
       "\t<tr><td>1939</td></tr>\n",
       "\t<tr><td>2017</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "A tibble: 6 × 1\n",
       "\\begin{tabular}{l}\n",
       " Year\\\\\n",
       " <chr>\\\\\n",
       "\\hline\n",
       "\t 2018\\\\\n",
       "\t 2019\\\\\n",
       "\t 2019\\\\\n",
       "\t 2019\\\\\n",
       "\t 1939\\\\\n",
       "\t 2017\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "A tibble: 6 × 1\n",
       "\n",
       "| Year &lt;chr&gt; |\n",
       "|---|\n",
       "| 2018 |\n",
       "| 2019 |\n",
       "| 2019 |\n",
       "| 2019 |\n",
       "| 1939 |\n",
       "| 2017 |\n",
       "\n"
      ],
      "text/plain": [
       "  Year\n",
       "1 2018\n",
       "2 2019\n",
       "3 2019\n",
       "4 2019\n",
       "5 1939\n",
       "6 2017"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mov_year  <- str_sub(movie_title_year,-5,-2)\n",
    "mov_year <- tibble(Year = mov_year)\n",
    "head(mov_year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracting the number of reviews\n",
    "number_of_reviews <- page_html %>%\n",
    "   html_nodes(\".table td:nth-child(4)\") %>% html_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "100"
      ],
      "text/latex": [
       "100"
      ],
      "text/markdown": [
       "100"
      ],
      "text/plain": [
       "[1] 100"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "100"
      ],
      "text/latex": [
       "100"
      ],
      "text/markdown": [
       "100"
      ],
      "text/plain": [
       "[1] 100"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "NULL"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "100"
      ],
      "text/latex": [
       "100"
      ],
      "text/markdown": [
       "100"
      ],
      "text/plain": [
       "[1] 100"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nrow(movie_name)\n",
    "nrow(mov_year)\n",
    "nrow(number_of_revie\n",
    "     ws)\n",
    "nrow(movie_rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "#number_of_reviews <- tibble(number_of_reviews = number_of_reviews)\n",
    "#head(number_of_reviews,5)\n",
    "rt_data <- NULL\n",
    "rt_data <- movie_name %>% add_column(Year =mov_year$Year,rt_rating=movie_rating$movie_rating)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<caption>A tibble: 6 × 3</caption>\n",
       "<thead>\n",
       "\t<tr><th scope=col>Movie_names</th><th scope=col>Year</th><th scope=col>rt_rating</th></tr>\n",
       "\t<tr><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "\t<tr><td>Black Panther     </td><td>2018</td><td> 9.8</td></tr>\n",
       "\t<tr><td>Avengers: Endgame </td><td>2019</td><td> 9.9</td></tr>\n",
       "\t<tr><td>Us                </td><td>2019</td><td> 9.7</td></tr>\n",
       "\t<tr><td>Toy Story 4       </td><td>2019</td><td>10.0</td></tr>\n",
       "\t<tr><td>The Wizard of Oz  </td><td>1939</td><td> 9.6</td></tr>\n",
       "\t<tr><td>Lady Bird         </td><td>2017</td><td> 9.6</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "A tibble: 6 × 3\n",
       "\\begin{tabular}{lll}\n",
       " Movie\\_names & Year & rt\\_rating\\\\\n",
       " <chr> & <chr> & <dbl>\\\\\n",
       "\\hline\n",
       "\t Black Panther      & 2018 &  9.8\\\\\n",
       "\t Avengers: Endgame  & 2019 &  9.9\\\\\n",
       "\t Us                 & 2019 &  9.7\\\\\n",
       "\t Toy Story 4        & 2019 & 10.0\\\\\n",
       "\t The Wizard of Oz   & 1939 &  9.6\\\\\n",
       "\t Lady Bird          & 2017 &  9.6\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "A tibble: 6 × 3\n",
       "\n",
       "| Movie_names &lt;chr&gt; | Year &lt;chr&gt; | rt_rating &lt;dbl&gt; |\n",
       "|---|---|---|\n",
       "| Black Panther      | 2018 |  9.8 |\n",
       "| Avengers: Endgame  | 2019 |  9.9 |\n",
       "| Us                 | 2019 |  9.7 |\n",
       "| Toy Story 4        | 2019 | 10.0 |\n",
       "| The Wizard of Oz   | 1939 |  9.6 |\n",
       "| Lady Bird          | 2017 |  9.6 |\n",
       "\n"
      ],
      "text/plain": [
       "  Movie_names        Year rt_rating\n",
       "1 Black Panther      2018  9.8     \n",
       "2 Avengers: Endgame  2019  9.9     \n",
       "3 Us                 2019  9.7     \n",
       "4 Toy Story 4        2019 10.0     \n",
       "5 The Wizard of Oz   1939  9.6     \n",
       "6 Lady Bird          2017  9.6     "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "head(rt_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "'/home/prasoons/project'"
      ],
      "text/latex": [
       "'/home/prasoons/project'"
      ],
      "text/markdown": [
       "'/home/prasoons/project'"
      ],
      "text/plain": [
       "[1] \"/home/prasoons/project\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Stroing data into CSV file\n",
    "getwd()  # getting current working directory\n",
    "filepath_rotten_tomatoes <- paste(getwd(),'//softscraped/rotten_tomatoes_data.csv',sep='')  # creating file path using 'paste' function with seprator a whitespace\n",
    "write.csv(rt_data,filepath_rotten_tomatoes, row.names = TRUE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "4.0.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}