Steam Game Web Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "sapphire-jonathan",
   "metadata": {},
   "source": [
    "# Steam Game Web Scraping\n",
    "\n",
    "Web Scraper for Steam review data. \n",
    "We will obtain review data, include: \n",
    "* User Steam ID\n",
    "* Profile URL\n",
    "* Review Content\n",
    "* Review Length (Approximately)\n",
    "* Recommend or Not Recommend\n",
    "* Play Hours\n",
    "* Date Posted\n",
    "* Number of Award of a Review\n",
    "* Number of Helpful and Funny of a Review\n",
    "\n",
    "Inside the `while` loop to modify the data you want to obtain."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "comprehensive-bacteria",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import selenium\n",
    "from msedge.selenium_tools import Edge, EdgeOptions\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from time import sleep\n",
    "from datetime import datetime\n",
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "existing-absorption",
   "metadata": {},
   "source": [
    "# Before Web Scraping\n",
    "\n",
    "You will need to install the Selenium package and corresponding Web Driver for your browser.\n",
    "\n",
    "To install the corresponding driver, check the browser version that you are planning to use and go to the [Selenium](https://www.selenium.dev/downloads/) page under **Browsers**, and click into the documentation of the browser you plan to use and download the corresponding version.\n",
    "\n",
    "After your download, simply move the Web Driver to your jupyter notebook directory. No INSTALLATION is required!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "furnished-injection",
   "metadata": {},
   "source": [
    "# Game ID\n",
    "\n",
    "Each game in Steam will have a unique id and that will help us to do the web scraping.\n",
    "\n",
    "For the review URL link, you can filter the language and then replace it with the game id to do the web scraping.\n",
    "\n",
    "https://steamcommunity.com/app/289070/reviews/?browsefilter=toprated&snr=1_5_100010_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "loved-bidding",
   "metadata": {},
   "outputs": [],
   "source": [
    "game_id = 289070"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "executed-contemporary",
   "metadata": {},
   "outputs": [],
   "source": [
    "# default URL Review Link\n",
    "template = 'https://steamcommunity.com/app/{}/reviews/?browsefilter=toprated&snr=1_5_100010_'\n",
    "\n",
    "# URL after filter with only Enlgish\n",
    "template_with_language = 'https://steamcommunity.com/app/{}/reviews/?browsefilter=toprated&snr=1_5_100010&filterLanguage=english'\n",
    "\n",
    "# insert the ID into the URL\n",
    "url = template_with_language.format(game_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "controversial-immunology",
   "metadata": {},
   "outputs": [],
   "source": [
    "# edge browser settings\n",
    "options = EdgeOptions()\n",
    "options.use_chromium = True\n",
    "options.add_experimental_option('useAutomationExtension', False)\n",
    "options.add_experimental_option('excludeSwitches', ['enable-automation'])\n",
    "options.add_experimental_option(\"detach\", True)\n",
    "driver = Edge(options=options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "attempted-marijuana",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.maximize_window()\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "written-warrant",
   "metadata": {},
   "outputs": [],
   "source": [
    "last_position = driver.execute_script(\"return window.pageYOffset;\")\n",
    "\n",
    "reviews = []\n",
    "review_ids = set()\n",
    "running = True\n",
    "\n",
    "while running:\n",
    "    # get data on the page\n",
    "    cards = driver.find_elements_by_class_name('apphub_Card')\n",
    "\n",
    "    for card in cards[-20:]:  # only the tail end are new data\n",
    "\n",
    "        # profile url\n",
    "        profile_url = card.find_element_by_xpath('.//div[@class=\"apphub_friend_block\"]/div/a[2]').get_attribute('href')\n",
    "\n",
    "        # steam user id\n",
    "        steam_id = profile_url.split('/')[-2]\n",
    "        \n",
    "        # check to see if collected for this review\n",
    "        if steam_id in review_ids:\n",
    "            continue\n",
    "        else:\n",
    "            review_ids.add(steam_id)\n",
    "\n",
    "        # username\n",
    "        user_name = card.find_element_by_xpath('.//div[@class=\"apphub_friend_block\"]/div/a[2]').text\n",
    "\n",
    "        # language of the review\n",
    "        date_posted = card.find_element_by_xpath('.//div[@class=\"apphub_CardTextContent\"]/div').text\n",
    "        review_content = card.find_element_by_xpath('.//div[@class=\"apphub_CardTextContent\"]').text.replace(date_posted,'').strip()    \n",
    "\n",
    "        # helpful & funny\n",
    "        helpful = card.find_element_by_xpath('.//div[@class=\"apphub_UserReviewCardContent\"]/div').text\n",
    "        \n",
    "        # award\n",
    "        award = card.find_element_by_xpath('.//div[@class=\"found_helpful\"]/div').text\n",
    "        \n",
    "        # review length\n",
    "        review_length = len(review_content.replace(' ', ''))    \n",
    "\n",
    "        # recommendation\n",
    "        thumb_text = card.find_element_by_xpath('.//div[@class=\"reviewInfo\"]/div[2]').text\n",
    "        thumb_text    \n",
    "\n",
    "        # amount of play hours\n",
    "        play_hours = card.find_element_by_xpath('.//div[@class=\"reviewInfo\"]/div[3]').text\n",
    "        play_hours    \n",
    "\n",
    "        # save review\n",
    "        review = (steam_id, profile_url, review_content, thumb_text, review_length, play_hours, date_posted, award, helpful)\n",
    "        reviews.append(review)    \n",
    "        \n",
    "    # attempt to scroll down by position 3 times, then break\n",
    "    scroll_attempt = 0\n",
    "    while True:\n",
    "        \n",
    "        try:\n",
    "            driver.find_element_by_xpath('.//div[@class=\"apphub_GetMoreContent\"]/div/a[1]').click();\n",
    "        except NoSuchElementException:\n",
    "            pass\n",
    "            \n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")    \n",
    "        sleep(1)\n",
    "        curr_position = driver.execute_script(\"return window.pageYOffset;\")\n",
    "        \n",
    "        \n",
    "        if curr_position == last_position:\n",
    "            scroll_attempt += 1\n",
    "            sleep(1)\n",
    "            \n",
    "            if curr_position >= 3:\n",
    "                running = False\n",
    "                break\n",
    "        else:\n",
    "            last_position = curr_position\n",
    "            break  # continue scraping the results\n",
    "\n",
    "# shutdown the web driver (close the browser)\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "satellite-capitol",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the file to a CSV file\n",
    "today = datetime.today().strftime('%Y%m%d')   \n",
    "with open(f'Steam_Reviews_{game_id}_{today}.csv', 'w', newline='', encoding='utf-8') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow(['SteamId', 'ProfileURL', 'ReviewText', 'Review', 'ReviewLength(Chars)', 'PlayHours', 'DatePosted', 'Award','HelpFunny'])\n",
    "    writer.writerows(reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "generic-advancement",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.8 64-bit ('base': conda)",
   "language": "python",
   "name": "python36864bitbaseconda8143bf76a2004fda854d7e537f0cc215"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}