6th_sept_web_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To install requests-html and jupyter in virtualenv\n",
    "pipenv install requests-html jupyter\n",
    "### To start jupyter notebook from virtualenv\n",
    "pipenv run jupyter notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "from requests_html import HTMLSession\n",
    "\n",
    "session = HTMLSession()\n",
    "\n",
    "website_url = \"https://news.ycombinator.com/\"\n",
    "\n",
    "response = session.get(website_url)\n",
    "\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['How to do a code review (google.github.io)', 'TP-82 Cosmonaut survival pistol (wikipedia.org)', 'At 97, lithium-ion battery pioneer John Goodenough says his work is not done (acs.org)', 'Dead Malls (deadmalls.com)', 'Was Etsy too good to be true? (vox.com)', 'US city rejects $5.3M ransom demand and restores encrypted files from backup (secalerts.co)', 'Cobol Web Development (infogoal.com)', 'MLIR Primer: A Compiler Infrastructure for the End of Moore’s Law (ai.google)', 'Apple has copied some of the most popular apps in the App Store (washingtonpost.com)', 'Sites with dumb password rules (github.com)', 'EASA Insists on Testing Boeing 737 Max Itself Before Lifting Ban (simpleflying.com)', 'Apple Music Web Client (music.apple.com)', 'PowerToys: Windows system utilities to maximize productivity (github.com)', 'Richard Stallman at Microsoft (no, we do not dream) (zdnet.fr)', 'Why doesn’t anyone weep for Docker? (techrepublic.com)', 'I Got More Data, My Model Is More Refined, but My Estimator Is Getting Worse [pdf] (columbia.edu)', 'Show HN: Taken Photos – Free Stock Photos Annotated by AI (taken.photos)', 'Memory Layout of a Program in C (utk.edu)', 'Are We Wrong About Black Holes? (quantamagazine.org)', 'A meteorite older than Earth shows evidence of ancient volcanism (syfy.com)', 'Stripe Capital (stripe.com)', 'Differentiable Programming Mega-Proposal (swift.org)', 'Psychological Characteristics of Romance Scam Victims (liebertpub.com)', 'How to learn D3.js (wattenberger.com)', 'Shift to electric vehicles will radically change auto factories (detroitnews.com)', 'Don’t Trust Facebook with Your Love Life (nytimes.com)', 'An innovative approach to making electricity from the wind (economist.com)', 'Nordstrom Will Let You Return Your Macy’s Online Order in Store (bloomberg.com)', 'Stegasuras: Neural Linguistic Steganography (steganography.live)', 'Samsung Announces Key-Value SSD Prototype (anandtech.com)']\n"
     ]
    }
   ],
   "source": [
    "links_table = response.html.find(\".itemlist\", first=True)\n",
    "\n",
    "article_name_and_link_list = links_table.find(\".athing\")\n",
    "\n",
    "article_headlines = []\n",
    "\n",
    "for article_html in article_name_and_link_list:\n",
    "    article_headlines.append(article_html.text.split(\"\\n\")[1])\n",
    "    \n",
    "print(article_headlines)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rank': 1, 'headline': 'Why Finnish babies sleep in cardboard boxes (2013) (bbc.com)', 'url': 'https://www.bbc.com/news/magazine-22751415'}\n",
      "{'rank': 2, 'headline': 'Mach kernel (wikipedia.org)', 'url': 'https://en.wikipedia.org/wiki/Mach_(kernel)'}\n",
      "{'rank': 3, 'headline': 'Google: Enabling developers and organizations to use differential privacy (googleblog.com)', 'url': 'https://developers.googleblog.com/2019/09/enabling-developers-and-organizations.html'}\n",
      "{'rank': 4, 'headline': 'Graphics Programming Black Book by Michael Abrash – Markdown source (2001) (github.com)', 'url': 'https://github.com/jagregory/abrash-black-book'}\n",
      "{'rank': 5, 'headline': 'Human speech may have a universal transmission rate: 39 bits per second (sciencemag.org)', 'url': 'https://www.sciencemag.org/news/2019/09/human-speech-may-have-universal-transmission-rate-39-bits-second'}\n",
      "{'rank': 6, 'headline': 'Secushare: A decentralized, secure social network built on GNUnet (secushare.org)', 'url': 'https://secushare.org/'}\n",
      "{'rank': 7, 'headline': 'A database of Facebook users’ phone numbers found online (techcrunch.com)', 'url': 'https://techcrunch.com/2019/09/04/facebook-phone-numbers-exposed/'}\n",
      "{'rank': 8, 'headline': 'The Next Hot Job: Pretending to Be a Robot (wsj.com)', 'url': 'https://www.wsj.com/articles/the-next-hot-job-pretending-to-be-a-robot-11567224001?mod=rsswn'}\n",
      "{'rank': 9, 'headline': 'Weighted blankets might ease insomnia and anxiety (washingtonpost.com)', 'url': 'https://www.washingtonpost.com/lifestyle/home/weighted-blankets-might-ease-insomnia-and-anxiety-heres-what-to-know-before-buying-one/2019/01/18/71b1aefa-1a6f-11e9-8813-cb9dec761e73_story.html'}\n",
      "{'rank': 10, 'headline': 'Creating a Magical Videobooth for a Wedding (jamesporter.me)', 'url': 'https://www.jamesporter.me/2019/08/30/wedding-videobooth.html'}\n",
      "{'rank': 11, 'headline': 'Johns Hopkins Launches Center for Psychedelic Research (jhu.edu)', 'url': 'https://hub.jhu.edu/2019/09/04/hopkins-launches-psychedelic-center/'}\n",
      "{'rank': 12, 'headline': 'Ask HN: Good Remote Work Literature?', 'url': 'item?id=20884974'}\n",
      "{'rank': 13, 'headline': 'Sally Floyd, who helped things run smoothly online, has died (nytimes.com)', 'url': 'https://www.nytimes.com/2019/09/04/science/sally-floyd-dead.html'}\n",
      "{'rank': 14, 'headline': 'Judge Rules Terrorism Watchlist Violates Constitutional Rights (nytimes.com)', 'url': 'https://www.nytimes.com/2019/09/04/us/politics/terrorism-watchlist-constitution.html'}\n",
      "{'rank': 15, 'headline': 'Drone Bubble Bursts, Wiping Out Startups and Hammering VC Firms (bloomberg.com)', 'url': 'https://www.bloomberg.com/news/articles/2019-08-30/drone-bubble-bursts-wiping-out-startups-and-hammering-vc-firms'}\n",
      "{'rank': 16, 'headline': 'Woman is first to receive cornea made from ‘reprogrammed’ stem cells (nature.com)', 'url': 'https://www.nature.com/articles/d41586-019-02597-2'}\n",
      "{'rank': 17, 'headline': 'Browser Fingerprinting: An Introduction and the Challenges Ahead (torproject.org)', 'url': 'https://blog.torproject.org/browser-fingerprinting-introduction-and-challenges-ahead'}\n",
      "{'rank': 18, 'headline': 'In an area without cellphone reception, I was unable to open the car (theatlantic.com)', 'url': 'https://www.theatlantic.com/ideas/archive/2019/09/zipcar-into-the-wilderness/597217/'}\n",
      "{'rank': 19, 'headline': 'Amazon’s Shipping Empire Is Challenging UPS and FedEx (wsj.com)', 'url': 'https://www.wsj.com/articles/how-amazons-shipping-empire-is-challenging-ups-and-fedex-11567071003?mod=rsswn'}\n",
      "{'rank': 20, 'headline': \"Holes in T. rex's skull now thought to have regulated body temperature (sciencedaily.com)\", 'url': 'https://www.sciencedaily.com/releases/2019/09/190904125337.htm'}\n",
      "{'rank': 21, 'headline': 'CircleCI Security Incident (circleci.com)', 'url': 'https://support.circleci.com/hc/en-us/articles/360034852194-Security-Incident-on-8-31-2019-Details-and-FAQs-'}\n",
      "{'rank': 22, 'headline': 'From bitter loss to sweet relief: baking as therapy (spectator.co.uk)', 'url': 'https://www.spectator.co.uk/2019/08/from-bitter-loss-to-sweet-relief-baking-as-therapy/'}\n",
      "{'rank': 23, 'headline': 'A mile or two off Yarmouth (2012) (bbc.co.uk)', 'url': 'https://www.bbc.co.uk/blogs/adamcurtis/entries/512cde83-3afb-3048-9ece-dba774b10f89'}\n",
      "{'rank': 24, 'headline': 'Lego to open 160 new stores as other toy retailers suffer (ft.com)', 'url': 'https://www.ft.com/content/d2b7554e-ce18-11e9-99a4-b5ded7a7fe3f'}\n",
      "{'rank': 25, 'headline': 'Google’s GDPR Workaround (brave.com)', 'url': 'https://brave.com/google-gdpr-workaround/'}\n",
      "{'rank': 26, 'headline': 'Tesla’s Autopilot found partly to blame for 2018 crash on the 405 (latimes.com)', 'url': 'https://www.latimes.com/business/story/2019-09-04/tesla-autopilot-is-found-partly-to-blame-for-2018-freeway-crash'}\n",
      "{'rank': 27, 'headline': 'Whatever happened to Six Sigma? (qz.com)', 'url': 'https://qz.com/work/1635960/whatever-happened-to-six-sigma/'}\n",
      "{'rank': 28, 'headline': 'Fuzzy and semantic search for captioned YouTube videos (github.com)', 'url': 'https://github.com/RameshAditya/scoper'}\n",
      "{'rank': 29, 'headline': 'Dr. Donald Lindberg, who opened medical research to the world, has died (nytimes.com)', 'url': 'https://www.nytimes.com/2019/09/02/obituaries/dr-donald-lindberg-dead.html'}\n",
      "{'rank': 30, 'headline': 'Show HN: fast visual-inertial odometry/SLAM for AR/VR/Robotics (github.com)', 'url': 'https://github.com/ucla-vision/xivo'}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def article_html_to_dict(single_article):\n",
    "    article_rank_raw = single_article.find(\".rank\", first=True).text\n",
    "\n",
    "    article_rank = int(article_rank_raw.replace(\".\",\"\"))\n",
    "    article_url = single_article.find(\".storylink\", first=True).attrs[\"href\"]\n",
    "    article_headline = single_article.text.split(\"\\n\")[1]\n",
    "\n",
    "    article_data = {\"rank\": article_rank, \n",
    "                    \"headline\": article_headline, \n",
    "                    \"url\": article_url}\n",
    "    \n",
    "    return article_data\n",
    "\n",
    "for single_article in article_name_and_link_list:\n",
    "    print(article_html_to_dict(single_article))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'username': 'iamaelephant', 'score': 1053, 'age': '14 hours ago', 'comments': 289}, {'username': 'BerislavLopac', 'score': 43, 'age': '3 hours ago', 'comments': 11}, {'username': 'sohkamyung', 'score': 28, 'age': '3 hours ago', 'comments': 7}, {'username': 'wallflower', 'score': 42, 'age': '4 hours ago', 'comments': 31}, {'username': 'detaro', 'score': 22, 'age': '2 hours ago', 'comments': 16}, {'username': 'GiulioS', 'score': 372, 'age': '15 hours ago', 'comments': 187}, {'username': 'gscott', 'score': 50, 'age': '5 hours ago', 'comments': 13}, {'username': 'sandGorgon', 'score': 69, 'age': '6 hours ago', 'comments': 12}, {'username': 'rgbrenner', 'score': 337, 'age': '14 hours ago', 'comments': 231}, {'username': 'enjoyyourlife', 'score': 386, 'age': '15 hours ago', 'comments': 244}, {'username': 'kunkurus', 'score': 122, 'age': '9 hours ago', 'comments': 44}, {'username': 'aplusplus', 'score': 270, 'age': '14 hours ago', 'comments': 161}, {'username': 'dgellow', 'score': 238, 'age': '14 hours ago', 'comments': 83}, {'username': 'Khelouiati', 'score': 31, 'age': '2 hours ago', 'comments': 22}, {'username': 'jumpingdeeps', 'score': 74, 'age': '7 hours ago', 'comments': 99}, {'username': 'charlysl', 'score': 57, 'age': '6 hours ago', 'comments': 13}, {'username': 'msamoylov', 'score': 17, 'age': '1 hour ago', 'comments': 10}, {'username': 'jbquark', 'score': 149, 'age': '14 hours ago', 'comments': 34}, {'username': 'theafh', 'score': 60, 'age': '10 hours ago', 'comments': 31}, {'username': 'sohkamyung', 'score': 43, 'age': '7 hours ago', 'comments': 1}, {'username': 'jonknee', 'score': 730, 'age': '18 hours ago', 'comments': 233}, {'username': 'xmmrm', 'score': 134, 'age': '14 hours ago', 'comments': 59}, {'username': 'laurex', 'score': 34, 'age': '9 hours ago', 'comments': 7}, {'username': 'slowhand09', 'score': 732, 'age': '20 hours ago', 'comments': 90}, {'username': 'rmason', 'score': 35, 'age': '7 hours ago', 'comments': 47}, {'username': 'whalabi', 'score': 90, 'age': '4 hours ago', 'comments': 54}, {'username': 'prostoalex', 'score': 41, 'age': '10 hours ago', 'comments': 29}, {'username': 'hhs', 'score': 25, 'age': '8 hours ago', 'comments': 4}, {'username': 'kcazyz', 'score': 89, 'age': '16 hours ago', 'comments': 23}, {'username': 'nikhizzle', 'score': 393, 'age': '18 hours ago', 'comments': 205}]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def get_article_subinfo(single_article_subtext):\n",
    "    score_raw = single_article_subtext.find(\".score\", first=True).text\n",
    "    score = int(score_raw.split(\" \")[0])\n",
    "\n",
    "    username = single_article_subtext.find(\".hnuser\", first=True).text\n",
    "    age = single_article_subtext.find(\".age\", first=True).text\n",
    "\n",
    "    comments_raw =  single_article_subtext.find(\"a\")[-1].text\n",
    "    no_of_comments = int(comments_raw.split()[0])\n",
    "\n",
    "    subinfo = {\"username\": username, \n",
    "               \"score\": score, \n",
    "               \"age\": age,\n",
    "               \"comments\": no_of_comments}\n",
    "    \n",
    "    return subinfo\n",
    "\n",
    "subtext_list = response.html.find(\".subtext\")\n",
    "\n",
    "subinfo_list = []\n",
    "for subtext in subtext_list:\n",
    "    subinfo = get_article_subinfo(subtext)\n",
    "    subinfo_list.append(subinfo)\n",
    "\n",
    "print(subinfo_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'username': 'iamaelephant', 'score': 1053, 'age': '14 hours ago', 'comments': 289}, {'username': 'BerislavLopac', 'score': 43, 'age': '3 hours ago', 'comments': 11}, {'username': 'sohkamyung', 'score': 28, 'age': '3 hours ago', 'comments': 7}, {'username': 'wallflower', 'score': 42, 'age': '4 hours ago', 'comments': 31}, {'username': 'detaro', 'score': 22, 'age': '2 hours ago', 'comments': 16}, {'username': 'GiulioS', 'score': 372, 'age': '15 hours ago', 'comments': 187}, {'username': 'gscott', 'score': 50, 'age': '5 hours ago', 'comments': 13}, {'username': 'sandGorgon', 'score': 69, 'age': '6 hours ago', 'comments': 12}, {'username': 'rgbrenner', 'score': 337, 'age': '14 hours ago', 'comments': 231}, {'username': 'enjoyyourlife', 'score': 386, 'age': '15 hours ago', 'comments': 244}, {'username': 'kunkurus', 'score': 122, 'age': '9 hours ago', 'comments': 44}, {'username': 'aplusplus', 'score': 270, 'age': '14 hours ago', 'comments': 161}, {'username': 'dgellow', 'score': 238, 'age': '14 hours ago', 'comments': 83}, {'username': 'Khelouiati', 'score': 31, 'age': '2 hours ago', 'comments': 22}, {'username': 'jumpingdeeps', 'score': 74, 'age': '7 hours ago', 'comments': 99}, {'username': 'charlysl', 'score': 57, 'age': '6 hours ago', 'comments': 13}, {'username': 'msamoylov', 'score': 17, 'age': '1 hour ago', 'comments': 10}, {'username': 'jbquark', 'score': 149, 'age': '14 hours ago', 'comments': 34}, {'username': 'theafh', 'score': 60, 'age': '10 hours ago', 'comments': 31}, {'username': 'sohkamyung', 'score': 43, 'age': '7 hours ago', 'comments': 1}, {'username': 'jonknee', 'score': 730, 'age': '18 hours ago', 'comments': 233}, {'username': 'xmmrm', 'score': 134, 'age': '14 hours ago', 'comments': 59}, {'username': 'laurex', 'score': 34, 'age': '9 hours ago', 'comments': 7}, {'username': 'slowhand09', 'score': 732, 'age': '20 hours ago', 'comments': 90}, {'username': 'rmason', 'score': 35, 'age': '7 hours ago', 'comments': 47}, {'username': 'whalabi', 'score': 90, 'age': '4 hours ago', 'comments': 54}, {'username': 'prostoalex', 'score': 41, 'age': '10 hours ago', 'comments': 29}, {'username': 'hhs', 'score': 25, 'age': '8 hours ago', 'comments': 4}, {'username': 'kcazyz', 'score': 89, 'age': '16 hours ago', 'comments': 23}, {'username': 'nikhizzle', 'score': 393, 'age': '18 hours ago', 'comments': 205}]\n"
     ]
    }
   ],
   "source": [
    "subtext_list = response.html.find(\".subtext\")\n",
    "\n",
    "subinfo_list = list(map(get_article_subinfo, subtext_list))\n",
    "\n",
    "print(subinfo_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['1', '1', '1', '1', '1']\n",
      "[1, 1, 1, 1, 1]\n",
      "[1, 1, 1, 1, 1]\n",
      "['1', '1', '1', '1', '1']\n"
     ]
    }
   ],
   "source": [
    "my_list = [\"1\"] * 5\n",
    "\n",
    "print(my_list)\n",
    "\n",
    "new_list = []\n",
    "for num in my_list:\n",
    "    int_num = int(num)\n",
    "    new_list.append(int_num)\n",
    "    \n",
    "print(new_list)\n",
    "\n",
    "\n",
    "new_list_with_map = list(map(int, my_list))\n",
    "print(new_list_with_map)\n",
    "\n",
    "str_list_with_map = list(map(str, new_list_with_map))\n",
    "print(str_list_with_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'username': 'nailer', 'score': 24, 'age': '11 minutes ago', 'comments': 2} {'rank': 1, 'headline': 'Fancy Zones, a tiling window manager from Microsoft (github.com)', 'url': 'https://github.com/microsoft/PowerToys/blob/master/src/modules/fancyzones/README.md'}\n"
     ]
    }
   ],
   "source": [
    "from requests_html import HTMLSession\n",
    "\n",
    "\n",
    "def get_article_maininfo(single_article_maintext):\n",
    "    article_rank_raw = single_article_maintext.find(\".rank\", first=True).text\n",
    "\n",
    "    article_rank = int(article_rank_raw.replace(\".\",\"\"))\n",
    "    article_url = single_article_maintext.find(\".storylink\", first=True).attrs[\"href\"]\n",
    "    article_headline = single_article_maintext.text.split(\"\\n\")[1]\n",
    "\n",
    "    article_data = {\"rank\": article_rank, \n",
    "                    \"headline\": article_headline, \n",
    "                    \"url\": article_url}\n",
    "    \n",
    "    return article_data\n",
    "\n",
    "\n",
    "def get_article_subinfo(single_article_subtext):\n",
    "    score_raw = single_article_subtext.find(\".score\", first=True).text\n",
    "    score = int(score_raw.split(\" \")[0])\n",
    "\n",
    "    username = single_article_subtext.find(\".hnuser\", first=True).text\n",
    "    age = single_article_subtext.find(\".age\", first=True).text\n",
    "\n",
    "    comments_raw =  single_article_subtext.find(\"a\")[-1].text\n",
    "    no_of_comments = int(comments_raw.split()[0])\n",
    "\n",
    "    subinfo = {\"username\": username, \n",
    "               \"score\": score, \n",
    "               \"age\": age,\n",
    "               \"comments\": no_of_comments}\n",
    "    \n",
    "    return subinfo\n",
    "\n",
    "\n",
    "session = HTMLSession()\n",
    "website_url = \"https://news.ycombinator.com/\"\n",
    "response = session.get(website_url)\n",
    "\n",
    "links_table = response.html.find(\".itemlist\", first=True)\n",
    "\n",
    "maintext_list = links_table.find(\".athing\")\n",
    "subtext_list = response.html.find(\".subtext\")\n",
    "\n",
    "maininfo_list = list(map(get_article_maininfo, maintext_list))\n",
    "subinfo_list = list(map(get_article_subinfo, subtext_list))\n",
    "print(subinfo_list[0], maininfo_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rank': 1, 'headline': 'Fancy Zones, a tiling window manager from Microsoft (github.com)', 'url': 'https://github.com/microsoft/PowerToys/blob/master/src/modules/fancyzones/README.md', 'username': 'nailer', 'score': 24, 'age': '11 minutes ago', 'comments': 2}\n"
     ]
    }
   ],
   "source": [
    "def add_two_dicts(first_dict, second_dict):\n",
    "    final_dict = {}\n",
    "\n",
    "    final_dict.update(first_dict)\n",
    "    final_dict.update(second_dict)\n",
    "\n",
    "    return final_dict\n",
    "\n",
    "\n",
    "article_info_list = []\n",
    "\n",
    "for index_num in range(len(maininfo_list)):\n",
    "    final_dict = add_two_dicts(maininfo_list[index_num], subinfo_list[index_num])\n",
    "    article_info_list.append(final_dict)\n",
    "    \n",
    "print(article_info_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://news.ycombinator.com/item?id=20895031\n",
      "https://news.ycombinator.com/item?id=20888156\n",
      "https://news.ycombinator.com/item?id=20894157\n",
      "https://news.ycombinator.com/item?id=20890059\n",
      "https://news.ycombinator.com/item?id=20894044\n",
      "https://news.ycombinator.com/item?id=20891075\n",
      "https://news.ycombinator.com/item?id=20860777\n",
      "https://news.ycombinator.com/item?id=20893407\n",
      "https://news.ycombinator.com/item?id=20890717\n",
      "https://news.ycombinator.com/item?id=20890682\n",
      "https://news.ycombinator.com/item?id=20889636\n",
      "https://news.ycombinator.com/item?id=20892421\n",
      "https://news.ycombinator.com/item?id=20890828\n",
      "https://news.ycombinator.com/item?id=20890876\n",
      "https://news.ycombinator.com/item?id=20893260\n",
      "https://news.ycombinator.com/item?id=20893087\n",
      "https://news.ycombinator.com/item?id=20890866\n",
      "https://news.ycombinator.com/item?id=20892129\n",
      "https://news.ycombinator.com/item?id=20888207\n",
      "https://news.ycombinator.com/item?id=20888817\n",
      "https://news.ycombinator.com/item?id=20894382\n",
      "https://news.ycombinator.com/item?id=20890292\n",
      "https://news.ycombinator.com/item?id=20889800\n",
      "https://news.ycombinator.com/item?id=20890149\n",
      "https://news.ycombinator.com/item?id=20887708\n",
      "https://news.ycombinator.com/item?id=20893737\n",
      "https://news.ycombinator.com/item?id=20893061\n",
      "https://news.ycombinator.com/item?id=20890381\n",
      "https://news.ycombinator.com/item?id=20892288\n",
      "https://news.ycombinator.com/item?id=20889881\n"
     ]
    }
   ],
   "source": [
    "subtext_list = response.html.find(\".subtext\")\n",
    "\n",
    "# single_subtext = subtext_list[9]\n",
    "comments_list = []\n",
    "for single_subtext in subtext_list:\n",
    "    comments = single_subtext.find(\"a\")[-1]\n",
    "    if \"comments\" in comments.text:\n",
    "        comments_list.append(comments)\n",
    "    else:\n",
    "        pass\n",
    "\n",
    "for comment in comments_list:\n",
    "    comments_url = comment.absolute_links.pop()\n",
    "    print(comments_url)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://news.ycombinator.com/item?id=20889881\n",
      "<Response [200]>\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1, 2, 3, 4, 5, 6, 7, 8}\n"
     ]
    }
   ],
   "source": [
    "# Set remove duplicates from list or any other iterable.\n",
    "my_numbers = [1,2,2,3,4,5,4,6,7,8]\n",
    "print(set(my_numbers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'username': 'AllegedAlec', 'text': \"Cool. It's fun to see how Turing patterns are everywhere in biology. However, as Turing himself said (allegedly): 'Well the stripes are easy, but what about the horse part?'\"}, {'username': 'ptah', 'text': 'can this be used in generating textures for 3d models'}]\n"
     ]
    }
   ],
   "source": [
    "session = HTMLSession()\n",
    "comments_url = \"https://news.ycombinator.com/item?id=20895212\"\n",
    "response = session.get(comments_url)\n",
    "\n",
    "\n",
    "comment_tree = response.html.find(\".comment-tree\", first=True)\n",
    "\n",
    "list_of_comments = comment_tree.find(\".comtr\")\n",
    "\n",
    "single_comment = list_of_comments[0]\n",
    "\n",
    "def get_username_and_comment(single_comment):\n",
    "    comments_filtered_1 = single_comment.find(\".default\", first=True)\n",
    "    comment_username = comments_filtered_1.find(\".hnuser\", first=True).text\n",
    "\n",
    "    comment_text = comments_filtered_1.find(\".commtext\", first=True).text\n",
    "\n",
    "    comment_dict = {\"username\":comment_username,\"text\": comment_text}\n",
    "    \n",
    "    return comment_dict\n",
    "\n",
    "comment_data_list = list(map(get_username_and_comment, list_of_comments))\n",
    "print(comment_data_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}