Web_scrapping.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Web_scrapping.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPC+y6bWyQs5PBi5lop+0WD",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ritesh-chafer/coronavirus-analysis/blob/master/Web_scrapping.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uS3HYbRxNxrM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "98df0d8d-649f-4d21-e619-eca22691baa5"
      },
      "source": [
        "from datetime import date\n",
        "today = date.today()\n",
        "d = today.strftime(\"%m-%d-%y\")\n",
        "print(\"date =\", d)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "date = 06-29-20\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MNaNjUwKseU2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cnn_url = \"https://edition.cnn.com/world/live-news/coronavirus-pandemic-{}-intl/index.html\".format(d)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DtamYVgNseR3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4dba3c84-1556-4403-829c-5fd98e28e230"
      },
      "source": [
        "print(cnn_url)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xBDVJovCsePe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from bs4 import BeautifulSoup\n",
        "import requests"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "itlKtXcfseM_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "html = requests.get(cnn_url).text"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r4Z_YyEmseKO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bd33942e-b98a-457f-90c1-565152f8e97d"
      },
      "source": [
        "soup = BeautifulSoup(html)\n",
        "print(soup.title)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<title data-rh=\"true\">Coronavirus pandemic: Updates from around the world</title>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZcpEwM-csd8B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "nlp = spacy.load('en_core_web_sm')"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NADRpM7ct05i",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "outputId": "a54ba3af-9a60-42b8-9413-0f59de59fcf3"
      },
      "source": [
        "for link in soup.find_all(\"h2\"):\n",
        "  print(\"headline : {}\".format(link.text))\n",
        "  for ent in nlp(link.text).ents:\n",
        "    print(\"\\text : {}, Entity :{}\".format(ent.text,ent.label_))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "headline : What you need to know\n",
            "headline : \"We did open too quickly,\" California county public health official says\n",
            "\text : California county, Entity :GPE\n",
            "headline : Why has the guidance on wearing face masks changed so much?\n",
            "headline : Amazon workers go on strike in Germany over coronavirus cases\n",
            "\text : Amazon, Entity :ORG\n",
            "\text : Germany, Entity :GPE\n",
            "headline : Florida reports more than 5,000 new coronavirus cases\n",
            "\text : Florida, Entity :GPE\n",
            "\text : more than 5,000, Entity :CARDINAL\n",
            "headline : College deans explain how their admissions priorities have changed in the Covid-19 pandemic\n",
            "headline : New York City hopes to avert labor layoffs in October as a result of budget cuts\n",
            "\text : New York City, Entity :GPE\n",
            "\text : October, Entity :DATE\n",
            "headline : New York City's daily Covid-19 indicators continue to be under maximum thresholds, mayor says\n",
            "\text : New York City's, Entity :GPE\n",
            "\text : daily, Entity :DATE\n",
            "headline : Indoor dining in NYC \"now in question\" for next phase of reopening, mayor says\n",
            "\text : NYC, Entity :ORG\n",
            "headline : Miami-Dade mayor on Covid-19 surge: Young people are disregarding the rules\n",
            "\text : Miami-Dade, Entity :GPE\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fh_F0v_lt02x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nbc_url = 'https://www.nbcnews.com/health/coronavirus'\n",
        "cnbc_rss_url = 'https://www.cnbc.com/id/10000108/device/rss/rss.html'"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HQTJ6CI6t0z1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "urls = [cnn_url, nbc_url, cnbc_rss_url]\n",
        "formats = ['html.parser','html.parser','xml']\n",
        "tags = ['h2', 'h2', 'description']\n",
        "website = ['CNN', 'NBC','CNBC']"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CCsFmgPHt0xB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "dbaf7ebf-2ad2-4b00-aa73-30a6b61850e3"
      },
      "source": [
        "crawl_len=0\n",
        "for url in urls:\n",
        "  print(\"crawling web page ..........{}\".format(url))\n",
        "  response = requests.get(url)\n",
        "  soup = BeautifulSoup(response.content, formats[crawl_len])\n",
        "  for link in soup.find_all(tags[crawl_len]):\n",
        "    if(len(link.text.split(\" \")) > 4):\n",
        "      print(\"Headline : {}\".format(link.text))\n",
        "      entities = []\n",
        "      for ent in nlp(link.text).ents:\n",
        "        print(\"\\tText: {}, Entity:{}\".format(ent.text, ent.label_))\n",
        "  crawl_len = crawl_len+1\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "crawling web page ..........https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html\n",
            "Headline : What you need to know\n",
            "Headline : 8 people died from coronavirus in New York yesterday \n",
            "\tText: 8, Entity:CARDINAL\n",
            "\tText: New York, Entity:GPE\n",
            "\tText: yesterday, Entity:DATE\n",
            "Headline : Broadway shows suspended for rest of year\n",
            "\tText: Broadway, Entity:FAC\n",
            "Headline : \"We did open too quickly,\" California county public health official says\n",
            "\tText: California county, Entity:GPE\n",
            "Headline : Why has the guidance on wearing face masks changed so much?\n",
            "Headline : Amazon workers go on strike in Germany over coronavirus cases\n",
            "\tText: Amazon, Entity:ORG\n",
            "\tText: Germany, Entity:GPE\n",
            "Headline : Florida reports more than 5,000 new coronavirus cases\n",
            "\tText: Florida, Entity:GPE\n",
            "\tText: more than 5,000, Entity:CARDINAL\n",
            "Headline : College deans explain how their admissions priorities have changed in the Covid-19 pandemic\n",
            "Headline : New York City hopes to avert labor layoffs in October as a result of budget cuts\n",
            "\tText: New York City, Entity:GPE\n",
            "\tText: October, Entity:DATE\n",
            "Headline : New York City's daily Covid-19 indicators continue to be under maximum thresholds, mayor says\n",
            "\tText: New York City's, Entity:GPE\n",
            "\tText: daily, Entity:DATE\n",
            "crawling web page ..........https://www.nbcnews.com/health/coronavirus\n",
            "Headline : In contrast to Trump, Pence encourages wearing masks to prevent coronavirus spread\n",
            "\tText: Trump, Entity:LOC\n",
            "Headline : Global COVID-19 death toll surpasses 500,000\n",
            "\tText: 500,000, Entity:CARDINAL\n",
            "Headline : Some COVID-19 patients aren't getting better. Major medical centers are seeking solutions.\n",
            "Headline : At least 85 people connected with visit to Michigan bar test positive for coronavirus\n",
            "\tText: At least 85, Entity:CARDINAL\n",
            "\tText: Michigan, Entity:GPE\n",
            "Headline : Drug company to charge thousands for coronavirus treatment\n",
            "\tText: thousands, Entity:CARDINAL\n",
            "Headline : Country stars Chase Rice, Chris Janson spark outrage with videos of packed concerts\n",
            "\tText: Chase Rice, Entity:PERSON\n",
            "\tText: Chris Janson, Entity:PERSON\n",
            "Headline : California orders bars closed in seven counties as coronavirus surges\n",
            "\tText: California, Entity:GPE\n",
            "\tText: seven, Entity:CARDINAL\n",
            "Headline : Police departments face one-two punch: Defund protests and coronavirus\n",
            "\tText: one-two, Entity:CARDINAL\n",
            "Headline : These states have the most coronavirus cases. See the day-by-day breakdown.\n",
            "Headline : Graphic: COVID-19 case counts in Florida and Texas are surging\n",
            "\tText: Florida, Entity:GPE\n",
            "\tText: Texas, Entity:GPE\n",
            "Headline : Germany cautions virus risk still high as economies restart\n",
            "\tText: Germany, Entity:GPE\n",
            "Headline : How coronavirus shut schools — and shut down mean girls\n",
            "Headline : Phoenix rally shows how gospel of Trump can be more attractive than the Bible\n",
            "\tText: Phoenix, Entity:GPE\n",
            "\tText: Trump, Entity:LOC\n",
            "\tText: Bible, Entity:WORK_OF_ART\n",
            "Headline : What I learned after 9/11 can help devastated restaurant workers now\n",
            "\tText: 9/11, Entity:DATE\n",
            "Headline : The secret war many doctors on the coronavirus front lines now face\n",
            "Headline : The complicated balancing act of church, state and the coronavirus \n",
            "Headline : Trump rants about fraud. But here's the secret to keeping voting by mail secure.\n",
            "Headline : Comedy is no coronavirus vaccine. But can it make living through a pandemic easier?\n",
            "Headline : Arguments over reclining are so yesterday. Here’s what the future of flying looks like.\n",
            "\tText: yesterday, Entity:DATE\n",
            "Headline : Does air conditioning spread the coronavirus?\n",
            "Headline : Trump blames testing for spike in COVID-19 cases. Experts disagree. \n",
            "Headline : Is this the second wave of COVID-19 in the U.S.? Or are we still in the first?\n",
            "\tText: second, Entity:ORDINAL\n",
            "\tText: U.S., Entity:GPE\n",
            "\tText: first, Entity:ORDINAL\n",
            "Headline : Trump falsely claims coronavirus numbers are 'going down almost everywhere'\n",
            "\tText: Trump, Entity:PRODUCT\n",
            "Headline : What you need to know about coronavirus home-testing kits\n",
            "Headline : Doctors, nurses and hospital staff\n",
            "Headline : Small businesses and their employees\n",
            "Headline : Senior citizens and the elderly\n",
            "Headline : What cleaning products kill the coronavirus?\n",
            "Headline : Are at-home testing kits available?\n",
            "Headline : Can you catch COVID-19 twice? \n",
            "Headline : Gyms are eager to reopen, but are they safe?\n",
            "Headline : How will coronavirus change the way we fly?\n",
            "Headline : Covid Chronicles, Vol. 3: Opera singer's voice carries during quarantine\n",
            "\tText: Covid Chronicles, Entity:PERSON\n",
            "\tText: Vol, Entity:GPE\n",
            "\tText: 3, Entity:CARDINAL\n",
            "\tText: Opera singer's, Entity:PERSON\n",
            "Headline : It's been months since the first U.S. coronavirus death. Here's a look at some of the lives lost.\n",
            "\tText: months, Entity:DATE\n",
            "\tText: first, Entity:ORDINAL\n",
            "\tText: U.S., Entity:GPE\n",
            "Headline : Affairs, hoarders and hope: Read anonymous confessions in the time of the coronavirus\n",
            "Headline : MSNBC Special Report: Testing and the Road to Reopening\n",
            "\tText: MSNBC, Entity:ORG\n",
            "Headline : 'Into The Red Zone': An NBC News NOW documentary \n",
            "\tText: The Red Zone', Entity:ORG\n",
            "\tText: NBC News, Entity:ORG\n",
            "Headline : Portrait of a neighborhood under lockdown\n",
            "Headline : Photos capture empty cities across the globe\n",
            "Headline : New Yorkers are giving nightly ovations to health workers. These are their portraits\n",
            "\tText: New Yorkers, Entity:NORP\n",
            "Headline : In case you missed it\n",
            "Headline : Mass graves and mercenaries: Libya plagued by war decade after U.S.-backed forces toppled Gadhafi\n",
            "\tText: Libya, Entity:GPE\n",
            "\tText: war decade, Entity:DATE\n",
            "\tText: Gadhafi, Entity:GPE\n",
            "Headline : Seeds of happiness: South Korea fights coronavirus depression with 'pet plants'\n",
            "\tText: South Korea, Entity:GPE\n",
            "Headline : Judge orders release of migrant children over coronavirus\n",
            "Headline : Florida counties fight coronavirus surge by closing beaches, mailing out masks\n",
            "\tText: Florida, Entity:GPE\n",
            "Headline : E.U. closer to barring travelers from U.S., where coronavirus is surging\n",
            "\tText: U.S., Entity:GPE\n",
            "Headline : Puerto Ricans in urban regions most exposed to COVID-19, study finds\n",
            "\tText: Puerto Ricans, Entity:NORP\n",
            "Headline : Supreme Court rejects mail-in voting for all in Texas during pandemic\n",
            "\tText: Supreme Court, Entity:ORG\n",
            "\tText: Texas, Entity:GPE\n",
            "Headline : For three weeks, a Texas county had zero coronavirus cases. Then the governor intervened.\n",
            "\tText: three weeks, Entity:DATE\n",
            "\tText: Texas, Entity:GPE\n",
            "\tText: zero, Entity:CARDINAL\n",
            "Headline : Texas and Florida close bars to combat COVID-19 spread\n",
            "\tText: Texas, Entity:GPE\n",
            "\tText: Florida, Entity:GPE\n",
            "Headline : Amid coronavirus surge, Texas has a contact tracing problem: reporting cases by fax\n",
            "\tText: Texas, Entity:GPE\n",
            "crawling web page ..........https://www.cnbc.com/id/10000108/device/rss/rss.html\n",
            "Headline : Latest news for health care. \n",
            "Headline : \"We all want this to be over. We all want to get on with our lives, but the hard reality is that this is not even close to being over,\" WHO chief Tedros Adhanom Ghebreyesus said.\n",
            "\tText: Tedros Adhanom Ghebreyesus, Entity:ORG\n",
            "Headline : The number of new Covid-19 cases per day in the U.S. jumped 42% over the past week to an average of about 38,200 on Sunday, based on a seven-day moving average.\n",
            "\tText: U.S., Entity:GPE\n",
            "\tText: 42%, Entity:PERCENT\n",
            "\tText: the past week, Entity:DATE\n",
            "\tText: about 38,200, Entity:CARDINAL\n",
            "\tText: Sunday, Entity:DATE\n",
            "\tText: seven-day, Entity:DATE\n",
            "Headline : The governor is expected to make an announcement on the current reopening guidelines in New York City and the state's response to the ongoing pandemic.\n",
            "\tText: New York City, Entity:GPE\n",
            "Headline : The coronavirus has now infected more than 10 million people across the globe, according to Johns Hopkins University data.\n",
            "\tText: more than 10 million, Entity:CARDINAL\n",
            "\tText: Johns Hopkins University, Entity:ORG\n",
            "Headline : Psychologist George James, associate program director for the Couple & Family Therapy Program at Thomas Jefferson University and a member of the CNBC Invest in You Financial Wellness Council, spoke with CNBC about the connection between finances and mental health, the Covid-19 crisis and the impact of recent events on the Black community.\n",
            "\tText: George James, Entity:PERSON\n",
            "\tText: the Couple & Family Therapy Program, Entity:ORG\n",
            "\tText: Thomas Jefferson University, Entity:ORG\n",
            "\tText: the CNBC Invest in You Financial Wellness Council, Entity:ORG\n",
            "\tText: CNBC, Entity:ORG\n",
            "Headline : House Minority Leader Kevin McCarthy on Monday partly blamed spikes in the number of coronavirus cases nationwide on demonstrations in support of Black Lives Matter.\n",
            "\tText: House, Entity:ORG\n",
            "\tText: Kevin McCarthy, Entity:PERSON\n",
            "\tText: Monday, Entity:DATE\n",
            "\tText: Black Lives Matter, Entity:PRODUCT\n",
            "Headline : \"We are definitely seeing this affect young people, and they're getting quite ill. So we really need everybody to do their part,\" Houston Methodist CEO Dr. Marc Boom told CNBC on Monday.\n",
            "\tText: Houston Methodist, Entity:WORK_OF_ART\n",
            "\tText: Marc Boom, Entity:PERSON\n",
            "\tText: CNBC, Entity:ORG\n",
            "\tText: Monday, Entity:DATE\n",
            "Headline : The coronavirus has infected more than 10.15 million people around the world as of Monday, killing at least 502,048 people.\n",
            "\tText: more than 10.15 million, Entity:CARDINAL\n",
            "\tText: Monday, Entity:DATE\n",
            "\tText: at least 502,048, Entity:CARDINAL\n",
            "Headline : \"At the level we have priced remdesivir and with government programs in place, along with additional Gilead assistance as needed, we believe all patients will have access,\" Gilead CEO Daniel O'Day said in an open letter.\n",
            "\tText: Gilead, Entity:PERSON\n",
            "\tText: Gilead, Entity:ORG\n",
            "\tText: Daniel O'Day, Entity:PERSON\n",
            "Headline : U.S. stock futures rose on Monday even as U.S. coronavirus cases continued to surge. Dow stock Boeing jumped more than 3% in the premarket.\n",
            "\tText: U.S., Entity:GPE\n",
            "\tText: Monday, Entity:DATE\n",
            "\tText: U.S., Entity:GPE\n",
            "\tText: Boeing, Entity:ORG\n",
            "\tText: more than 3%, Entity:PERCENT\n",
            "Headline : There are still no drugs approved by the U.S. Food and Drug Administration to treat Covid-19.\n",
            "\tText: the U.S. Food and Drug Administration, Entity:ORG\n",
            "Headline : \"COVID-19 is still circulating in California, and in some parts of the state, growing stronger,\" Newsom said in a written statement.\n",
            "\tText: California, Entity:GPE\n",
            "\tText: Newsom, Entity:PERSON\n",
            "Headline : Azar also denied that states reopening prematurely was tied to the spike in coronavirus cases and instead blamed \"inappropriate individual behavior\" that has spread the virus.\n",
            "\tText: Azar, Entity:PERSON\n",
            "Headline : Mask wearing has become a point of contention across the U.S., despite research showing that face coverings prevent coronavirus transmission.\n",
            "\tText: U.S., Entity:GPE\n",
            "Headline : As with many facets of life nowadays, checking into a hotel has changed in light of the coronavirus pandemic. Here's a look at what one stay at a Hudson Valley hotel in upstate New York was like in late June.\n",
            "\tText: one, Entity:CARDINAL\n",
            "\tText: Hudson Valley, Entity:LOC\n",
            "\tText: New York, Entity:GPE\n",
            "\tText: late June, Entity:DATE\n",
            "Headline : As the coronavirus continues to spread across the U.S., causing dramatic spikes in the number of new cases in states that have reopened, the older population may have to brace themselves for another round of health risks.\n",
            "\tText: U.S., Entity:GPE\n",
            "Headline : Roughly a quarter of confirmed coronavirus cases are in the U.S., followed by Brazil, Russia and India.\n",
            "\tText: Roughly a quarter, Entity:CARDINAL\n",
            "\tText: U.S., Entity:GPE\n",
            "\tText: Brazil, Entity:GPE\n",
            "\tText: Russia, Entity:GPE\n",
            "\tText: India, Entity:GPE\n",
            "Headline : Four people who were in contact with the student have since tested positive for the coronavirus since the drive-in ceremony was held on June 20.\n",
            "\tText: Four, Entity:CARDINAL\n",
            "\tText: June 20, Entity:DATE\n",
            "Headline : Coronavirus cases have been rising by the thousands in Florida in recent days, along with a surge in many other states, particularly in the South and West.\n",
            "\tText: Coronavirus, Entity:ORG\n",
            "\tText: thousands, Entity:CARDINAL\n",
            "\tText: Florida, Entity:GPE\n",
            "\tText: recent days, Entity:DATE\n",
            "\tText: South, Entity:LOC\n",
            "\tText: West, Entity:LOC\n",
            "Headline : There were 45,255 additional Covid-19 cases reported across the nation on Friday, bringing the total to more than 2.46 million cases, according to Johns Hopkins University data.\n",
            "\tText: 45,255, Entity:CARDINAL\n",
            "\tText: Friday, Entity:DATE\n",
            "\tText: more than 2.46 million, Entity:CARDINAL\n",
            "\tText: Johns Hopkins University, Entity:ORG\n",
            "Headline : Digital health has been thriving during the pandemic, as patients see alternatives to in-person care, but not every business will do well, according to experts.\n",
            "\tText: Digital, Entity:ORG\n",
            "Headline : Health care workers are fighting a new battle regarding their mental health and well-being due to Covid-19, as deaths have recently topped 120,000 people.\n",
            "\tText: 120,000, Entity:CARDINAL\n",
            "Headline : Older people with underlying health conditions are most vulnerable to coronavirus and heat-related illness as a particularly blistering summer approaches.\n",
            "\tText: summer, Entity:DATE\n",
            "Headline : Children held more than 20 days at certain facilities should be released by July 17, Judge Dolly Gee said.\n",
            "\tText: more than 20 days, Entity:DATE\n",
            "\tText: July 17, Entity:DATE\n",
            "\tText: Dolly Gee, Entity:PERSON\n",
            "Headline : Delta and the pilots' labor union reached an agreement for early retirements as the carrier scrambles to reduce headcount.\n",
            "\tText: Delta, Entity:ORG\n",
            "Headline : \"We're always busy in the summertime and what we're seeing now is a typical summer for us,\" Memorial Hermann Health System CEO Dr. David Callender told CNBC.\n",
            "\tText: Memorial Hermann Health System CEO, Entity:WORK_OF_ART\n",
            "\tText: David Callender, Entity:PERSON\n",
            "\tText: CNBC, Entity:ORG\n",
            "Headline : San Francisco Mayor London Breed said in a tweet Friday that the city will temporarily delay its planned reopening for Monday, which included nail salons, barbershops, outdoor bars.\n",
            "\tText: San Francisco, Entity:GPE\n",
            "\tText: London Breed, Entity:PERSON\n",
            "\tText: Friday, Entity:DATE\n",
            "\tText: Monday, Entity:DATE\n",
            "Headline : Ro, which previously specialized in men's health, is now generating $250 million in annual revenue after expanding its product line.\n",
            "\tText: $250 million, Entity:MONEY\n",
            "\tText: annual, Entity:DATE\n",
            "Headline : \"People are infecting other people and ultimately you will infect someone who's vulnerable,\" Dr. Anthony Fauci, director of the National Institute of Allergy and Infectious Diseases, said during a press briefing.\n",
            "\tText: Anthony Fauci, Entity:PERSON\n",
            "\tText: the National Institute of Allergy and Infectious Diseases, Entity:ORG\n",
            "Headline : Some states might have to consider shutting down businesses and implementing restrictions on movement again to keep the health-care system from getting overwhelmed, Fauci said.\n",
            "\tText: Fauci, Entity:PERSON\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R3o7YNL8t0t-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "crawl_len = 0\n",
        "news_dict = []\n",
        "for url in urls:\n",
        "  response = requests.get(url)\n",
        "  soup = BeautifulSoup(response.content, formats[crawl_len])\n",
        "  for link in soup.find_all(tags[crawl_len]):\n",
        "    if(len(link.text.split(\" \")) > 4):\n",
        "      entities = []\n",
        "      entities = [(ent.text, ent.label_) for ent in nlp(link.text).ents]\n",
        "      news_dict.append({'website':website[crawl_len],'url':url,'headline':link.text,'entities':entities})\n",
        "  crawl_len = crawl_len + 1    "
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bvx35ZAot0mG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "62185897-2360-4e56-879d-799339e185e1"
      },
      "source": [
        "news_dict"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'entities': [],\n",
              "  'headline': 'What you need to know',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('Coronavirus', 'ORG')],\n",
              "  'headline': 'Coronavirus pandemic \"not even close to being over,\" WHO says ',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('VMAs', 'ORG'),\n",
              "   ('Brooklyn', 'GPE'),\n",
              "   ('August', 'DATE'),\n",
              "   ('New York', 'GPE')],\n",
              "  'headline': 'VMAs will be held in Brooklyn in August, New York governor says',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('New York', 'GPE'),\n",
              "   ('Wednesday', 'DATE'),\n",
              "   ('NYC', 'ORG'),\n",
              "   ('3', 'CARDINAL'),\n",
              "   ('Cuomo', 'ORG')],\n",
              "  'headline': 'New York will decide by Wednesday whether NYC can have indoor dining in phase 3, Cuomo says',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('8', 'CARDINAL'), ('New York', 'GPE'), ('yesterday', 'DATE')],\n",
              "  'headline': '8 people died from coronavirus in New York yesterday ',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('Broadway', 'FAC')],\n",
              "  'headline': 'Broadway shows suspended for rest of year',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('California county', 'GPE')],\n",
              "  'headline': '\"We did open too quickly,\" California county public health official says',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [],\n",
              "  'headline': 'Why has the guidance on wearing face masks changed so much?',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('Amazon', 'ORG'), ('Germany', 'GPE')],\n",
              "  'headline': 'Amazon workers go on strike in Germany over coronavirus cases',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('Florida', 'GPE'), ('more than 5,000', 'CARDINAL')],\n",
              "  'headline': 'Florida reports more than 5,000 new coronavirus cases',\n",
              "  'url': 'https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html',\n",
              "  'website': 'CNN'},\n",
              " {'entities': [('Trump', 'LOC')],\n",
              "  'headline': 'In contrast to Trump, Pence encourages wearing masks to prevent coronavirus spread',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('500,000', 'CARDINAL')],\n",
              "  'headline': 'Global COVID-19 death toll surpasses 500,000',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': \"Some COVID-19 patients aren't getting better. Major medical centers are seeking solutions.\",\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('At least 85', 'CARDINAL'), ('Michigan', 'GPE')],\n",
              "  'headline': 'At least 85 people connected with visit to Michigan bar test positive for coronavirus',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('thousands', 'CARDINAL')],\n",
              "  'headline': 'Drug company to charge thousands for coronavirus treatment',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Chase Rice', 'PERSON'), ('Chris Janson', 'PERSON')],\n",
              "  'headline': 'Country stars Chase Rice, Chris Janson spark outrage with videos of packed concerts',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('California', 'GPE'), ('seven', 'CARDINAL')],\n",
              "  'headline': 'California orders bars closed in seven counties as coronavirus surges',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('one-two', 'CARDINAL')],\n",
              "  'headline': 'Police departments face one-two punch: Defund protests and coronavirus',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'These states have the most coronavirus cases. See the day-by-day breakdown.',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Florida', 'GPE'), ('Texas', 'GPE')],\n",
              "  'headline': 'Graphic: COVID-19 case counts in Florida and Texas are surging',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Germany', 'GPE')],\n",
              "  'headline': 'Germany cautions virus risk still high as economies restart',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'How coronavirus shut schools — and shut down mean girls',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Phoenix', 'GPE'), ('Trump', 'LOC'), ('Bible', 'WORK_OF_ART')],\n",
              "  'headline': 'Phoenix rally shows how gospel of Trump can be more attractive than the Bible',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('9/11', 'DATE')],\n",
              "  'headline': 'What I learned after 9/11 can help devastated restaurant workers now',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'The secret war many doctors on the coronavirus front lines now face',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'The complicated balancing act of church, state and the coronavirus ',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': \"Trump rants about fraud. But here's the secret to keeping voting by mail secure.\",\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Comedy is no coronavirus vaccine. But can it make living through a pandemic easier?',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('yesterday', 'DATE')],\n",
              "  'headline': 'Arguments over reclining are so yesterday. Here’s what the future of flying looks like.',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Does air conditioning spread the coronavirus?',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Trump blames testing for spike in COVID-19 cases. Experts disagree. ',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('second', 'ORDINAL'), ('U.S.', 'GPE'), ('first', 'ORDINAL')],\n",
              "  'headline': 'Is this the second wave of COVID-19 in the U.S.? Or are we still in the first?',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Trump', 'PRODUCT')],\n",
              "  'headline': \"Trump falsely claims coronavirus numbers are 'going down almost everywhere'\",\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'What you need to know about coronavirus home-testing kits',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Doctors, nurses and hospital staff',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Small businesses and their employees',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Senior citizens and the elderly',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'What cleaning products kill the coronavirus?',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Are at-home testing kits available?',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Can you catch COVID-19 twice? ',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Gyms are eager to reopen, but are they safe?',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'How will coronavirus change the way we fly?',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Covid Chronicles', 'PERSON'),\n",
              "   ('Vol', 'GPE'),\n",
              "   ('3', 'CARDINAL'),\n",
              "   (\"Opera singer's\", 'PERSON')],\n",
              "  'headline': \"Covid Chronicles, Vol. 3: Opera singer's voice carries during quarantine\",\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('months', 'DATE'), ('first', 'ORDINAL'), ('U.S.', 'GPE')],\n",
              "  'headline': \"It's been months since the first U.S. coronavirus death. Here's a look at some of the lives lost.\",\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Affairs, hoarders and hope: Read anonymous confessions in the time of the coronavirus',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('MSNBC', 'ORG')],\n",
              "  'headline': 'MSNBC Special Report: Testing and the Road to Reopening',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [(\"The Red Zone'\", 'ORG'), ('NBC News', 'ORG')],\n",
              "  'headline': \"'Into The Red Zone': An NBC News NOW documentary \",\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Portrait of a neighborhood under lockdown',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Photos capture empty cities across the globe',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('New Yorkers', 'NORP')],\n",
              "  'headline': 'New Yorkers are giving nightly ovations to health workers. These are their portraits',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'In case you missed it',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Libya', 'GPE'), ('war decade', 'DATE'), ('Gadhafi', 'GPE')],\n",
              "  'headline': 'Mass graves and mercenaries: Libya plagued by war decade after U.S.-backed forces toppled Gadhafi',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('South Korea', 'GPE')],\n",
              "  'headline': \"Seeds of happiness: South Korea fights coronavirus depression with 'pet plants'\",\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Judge orders release of migrant children over coronavirus',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Florida', 'GPE')],\n",
              "  'headline': 'Florida counties fight coronavirus surge by closing beaches, mailing out masks',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('U.S.', 'GPE')],\n",
              "  'headline': 'E.U. closer to barring travelers from U.S., where coronavirus is surging',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Puerto Ricans', 'NORP')],\n",
              "  'headline': 'Puerto Ricans in urban regions most exposed to COVID-19, study finds',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Supreme Court', 'ORG'), ('Texas', 'GPE')],\n",
              "  'headline': 'Supreme Court rejects mail-in voting for all in Texas during pandemic',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('three weeks', 'DATE'),\n",
              "   ('Texas', 'GPE'),\n",
              "   ('zero', 'CARDINAL')],\n",
              "  'headline': 'For three weeks, a Texas county had zero coronavirus cases. Then the governor intervened.',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Texas', 'GPE'), ('Florida', 'GPE')],\n",
              "  'headline': 'Texas and Florida close bars to combat COVID-19 spread',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [('Texas', 'GPE')],\n",
              "  'headline': 'Amid coronavirus surge, Texas has a contact tracing problem: reporting cases by fax',\n",
              "  'url': 'https://www.nbcnews.com/health/coronavirus',\n",
              "  'website': 'NBC'},\n",
              " {'entities': [],\n",
              "  'headline': 'Latest news for health care. ',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('second', 'ORDINAL'),\n",
              "   ('second', 'ORDINAL'),\n",
              "   ('second', 'ORDINAL'),\n",
              "   ('Instagram', 'NORP'),\n",
              "   ('Kevin Systrom', 'PERSON'),\n",
              "   ('CNBC', 'ORG'),\n",
              "   ('Monday', 'DATE')],\n",
              "  'headline': '\"People like to say we\\'re not in a second wave. I don\\'t know what a second wave [is] if that\\'s not a second wave,\" Instagram co-founder Kevin Systrom told CNBC on Monday.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Tedros Adhanom Ghebreyesus', 'ORG')],\n",
              "  'headline': '\"We all want this to be over. We all want to get on with our lives, but the hard reality is that this is not even close to being over,\" WHO chief Tedros Adhanom Ghebreyesus said.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('U.S.', 'GPE'),\n",
              "   ('42%', 'PERCENT'),\n",
              "   ('the past week', 'DATE'),\n",
              "   ('about 38,200', 'CARDINAL'),\n",
              "   ('Sunday', 'DATE'),\n",
              "   ('seven-day', 'DATE')],\n",
              "  'headline': 'The number of new Covid-19 cases per day in the U.S. jumped 42% over the past week to an average of about 38,200 on Sunday, based on a seven-day moving average.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('New York City', 'GPE')],\n",
              "  'headline': \"The governor is expected to make an announcement on the current reopening guidelines in New York City and the state's response to the ongoing pandemic.\",\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('more than 10 million', 'CARDINAL'),\n",
              "   ('Johns Hopkins University', 'ORG')],\n",
              "  'headline': 'The coronavirus has now infected more than 10 million people across the globe, according to Johns Hopkins University data.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('George James', 'PERSON'),\n",
              "   ('the Couple & Family Therapy Program', 'ORG'),\n",
              "   ('Thomas Jefferson University', 'ORG'),\n",
              "   ('the CNBC Invest in You Financial Wellness Council', 'ORG'),\n",
              "   ('CNBC', 'ORG')],\n",
              "  'headline': 'Psychologist George James, associate program director for the Couple & Family Therapy Program at Thomas Jefferson University and a member of the CNBC Invest in You Financial Wellness Council, spoke with CNBC about the connection between finances and mental health, the Covid-19 crisis and the impact of recent events on the Black community.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('House', 'ORG'),\n",
              "   ('Kevin McCarthy', 'PERSON'),\n",
              "   ('Monday', 'DATE'),\n",
              "   ('Black Lives Matter', 'PRODUCT')],\n",
              "  'headline': 'House Minority Leader Kevin McCarthy on Monday partly blamed spikes in the number of coronavirus cases nationwide on demonstrations in support of Black Lives Matter.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Houston Methodist', 'WORK_OF_ART'),\n",
              "   ('Marc Boom', 'PERSON'),\n",
              "   ('CNBC', 'ORG'),\n",
              "   ('Monday', 'DATE')],\n",
              "  'headline': '\"We are definitely seeing this affect young people, and they\\'re getting quite ill. So we really need everybody to do their part,\" Houston Methodist CEO Dr. Marc Boom told CNBC on Monday.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('more than 10.15 million', 'CARDINAL'),\n",
              "   ('Monday', 'DATE'),\n",
              "   ('at least 502,048', 'CARDINAL')],\n",
              "  'headline': 'The coronavirus has infected more than 10.15 million people around the world as of Monday, killing at least 502,048 people.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Gilead', 'PERSON'),\n",
              "   ('Gilead', 'ORG'),\n",
              "   (\"Daniel O'Day\", 'PERSON')],\n",
              "  'headline': '\"At the level we have priced remdesivir and with government programs in place, along with additional Gilead assistance as needed, we believe all patients will have access,\" Gilead CEO Daniel O\\'Day said in an open letter.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('U.S.', 'GPE'),\n",
              "   ('Monday', 'DATE'),\n",
              "   ('U.S.', 'GPE'),\n",
              "   ('Boeing', 'ORG'),\n",
              "   ('more than 3%', 'PERCENT')],\n",
              "  'headline': 'U.S. stock futures rose on Monday even as U.S. coronavirus cases continued to surge. Dow stock Boeing jumped more than 3% in the premarket.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('the U.S. Food and Drug Administration', 'ORG')],\n",
              "  'headline': 'There are still no drugs approved by the U.S. Food and Drug Administration to treat Covid-19.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('California', 'GPE'), ('Newsom', 'PERSON')],\n",
              "  'headline': '\"COVID-19 is still circulating in California, and in some parts of the state, growing stronger,\" Newsom said in a written statement.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Azar', 'PERSON')],\n",
              "  'headline': 'Azar also denied that states reopening prematurely was tied to the spike in coronavirus cases and instead blamed \"inappropriate individual behavior\" that has spread the virus.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('U.S.', 'GPE')],\n",
              "  'headline': 'Mask wearing has become a point of contention across the U.S., despite research showing that face coverings prevent coronavirus transmission.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('one', 'CARDINAL'),\n",
              "   ('Hudson Valley', 'LOC'),\n",
              "   ('New York', 'GPE'),\n",
              "   ('late June', 'DATE')],\n",
              "  'headline': \"As with many facets of life nowadays, checking into a hotel has changed in light of the coronavirus pandemic. Here's a look at what one stay at a Hudson Valley hotel in upstate New York was like in late June.\",\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('U.S.', 'GPE')],\n",
              "  'headline': 'As the coronavirus continues to spread across the U.S., causing dramatic spikes in the number of new cases in states that have reopened, the older population may have to brace themselves for another round of health risks.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Roughly a quarter', 'CARDINAL'),\n",
              "   ('U.S.', 'GPE'),\n",
              "   ('Brazil', 'GPE'),\n",
              "   ('Russia', 'GPE'),\n",
              "   ('India', 'GPE')],\n",
              "  'headline': 'Roughly a quarter of confirmed coronavirus cases are in the U.S., followed by Brazil, Russia and India.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Four', 'CARDINAL'), ('June 20', 'DATE')],\n",
              "  'headline': 'Four people who were in contact with the student have since tested positive for the coronavirus since the drive-in ceremony was held on June 20.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Coronavirus', 'ORG'),\n",
              "   ('thousands', 'CARDINAL'),\n",
              "   ('Florida', 'GPE'),\n",
              "   ('recent days', 'DATE'),\n",
              "   ('South', 'LOC'),\n",
              "   ('West', 'LOC')],\n",
              "  'headline': 'Coronavirus cases have been rising by the thousands in Florida in recent days, along with a surge in many other states, particularly in the South and West.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('45,255', 'CARDINAL'),\n",
              "   ('Friday', 'DATE'),\n",
              "   ('more than 2.46 million', 'CARDINAL'),\n",
              "   ('Johns Hopkins University', 'ORG')],\n",
              "  'headline': 'There were 45,255 additional Covid-19 cases reported across the nation on Friday, bringing the total to more than 2.46 million cases, according to Johns Hopkins University data.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Digital', 'ORG')],\n",
              "  'headline': 'Digital health has been thriving during the pandemic, as patients see alternatives to in-person care, but not every business will do well, according to experts.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('120,000', 'CARDINAL')],\n",
              "  'headline': 'Health care workers are fighting a new battle regarding their mental health and well-being due to Covid-19, as deaths have recently topped 120,000 people.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('summer', 'DATE')],\n",
              "  'headline': 'Older people with underlying health conditions are most vulnerable to coronavirus and heat-related illness as a particularly blistering summer approaches.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('more than 20 days', 'DATE'),\n",
              "   ('July 17', 'DATE'),\n",
              "   ('Dolly Gee', 'PERSON')],\n",
              "  'headline': 'Children held more than 20 days at certain facilities should be released by July 17, Judge Dolly Gee said.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Delta', 'ORG')],\n",
              "  'headline': \"Delta and the pilots' labor union reached an agreement for early retirements as the carrier scrambles to reduce headcount.\",\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Memorial Hermann Health System CEO', 'WORK_OF_ART'),\n",
              "   ('David Callender', 'PERSON'),\n",
              "   ('CNBC', 'ORG')],\n",
              "  'headline': '\"We\\'re always busy in the summertime and what we\\'re seeing now is a typical summer for us,\" Memorial Hermann Health System CEO Dr. David Callender told CNBC.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('San Francisco', 'GPE'),\n",
              "   ('London Breed', 'PERSON'),\n",
              "   ('Friday', 'DATE'),\n",
              "   ('Monday', 'DATE')],\n",
              "  'headline': 'San Francisco Mayor London Breed said in a tweet Friday that the city will temporarily delay its planned reopening for Monday, which included nail salons, barbershops, outdoor bars.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('$250 million', 'MONEY'), ('annual', 'DATE')],\n",
              "  'headline': \"Ro, which previously specialized in men's health, is now generating $250 million in annual revenue after expanding its product line.\",\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'},\n",
              " {'entities': [('Anthony Fauci', 'PERSON'),\n",
              "   ('the National Institute of Allergy and Infectious Diseases', 'ORG')],\n",
              "  'headline': '\"People are infecting other people and ultimately you will infect someone who\\'s vulnerable,\" Dr. Anthony Fauci, director of the National Institute of Allergy and Infectious Diseases, said during a press briefing.',\n",
              "  'url': 'https://www.cnbc.com/id/10000108/device/rss/rss.html',\n",
              "  'website': 'CNBC'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KHJCORSB0pDQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8peR6r1R02Em",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "news_df = pd.DataFrame(news_dict)"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-El74wVj0o9F",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "02446706-6ce8-4b19-b6aa-5e5810ec4e83"
      },
      "source": [
        "pd.set_option('max_colwidth', 800)\n",
        "news_df.head(30)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>website</th>\n",
              "      <th>url</th>\n",
              "      <th>headline</th>\n",
              "      <th>entities</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>What you need to know</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>Coronavirus pandemic \"not even close to being over,\" WHO says</td>\n",
              "      <td>[(Coronavirus, ORG)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>VMAs will be held in Brooklyn in August, New York governor says</td>\n",
              "      <td>[(VMAs, ORG), (Brooklyn, GPE), (August, DATE), (New York, GPE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>New York will decide by Wednesday whether NYC can have indoor dining in phase 3, Cuomo says</td>\n",
              "      <td>[(New York, GPE), (Wednesday, DATE), (NYC, ORG), (3, CARDINAL), (Cuomo, ORG)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>8 people died from coronavirus in New York yesterday</td>\n",
              "      <td>[(8, CARDINAL), (New York, GPE), (yesterday, DATE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>Broadway shows suspended for rest of year</td>\n",
              "      <td>[(Broadway, FAC)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>\"We did open too quickly,\" California county public health official says</td>\n",
              "      <td>[(California county, GPE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>Why has the guidance on wearing face masks changed so much?</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>Amazon workers go on strike in Germany over coronavirus cases</td>\n",
              "      <td>[(Amazon, ORG), (Germany, GPE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>Florida reports more than 5,000 new coronavirus cases</td>\n",
              "      <td>[(Florida, GPE), (more than 5,000, CARDINAL)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>In contrast to Trump, Pence encourages wearing masks to prevent coronavirus spread</td>\n",
              "      <td>[(Trump, LOC)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Global COVID-19 death toll surpasses 500,000</td>\n",
              "      <td>[(500,000, CARDINAL)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Some COVID-19 patients aren't getting better. Major medical centers are seeking solutions.</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>At least 85 people connected with visit to Michigan bar test positive for coronavirus</td>\n",
              "      <td>[(At least 85, CARDINAL), (Michigan, GPE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Drug company to charge thousands for coronavirus treatment</td>\n",
              "      <td>[(thousands, CARDINAL)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Country stars Chase Rice, Chris Janson spark outrage with videos of packed concerts</td>\n",
              "      <td>[(Chase Rice, PERSON), (Chris Janson, PERSON)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>California orders bars closed in seven counties as coronavirus surges</td>\n",
              "      <td>[(California, GPE), (seven, CARDINAL)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Police departments face one-two punch: Defund protests and coronavirus</td>\n",
              "      <td>[(one-two, CARDINAL)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>These states have the most coronavirus cases. See the day-by-day breakdown.</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Graphic: COVID-19 case counts in Florida and Texas are surging</td>\n",
              "      <td>[(Florida, GPE), (Texas, GPE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Germany cautions virus risk still high as economies restart</td>\n",
              "      <td>[(Germany, GPE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>How coronavirus shut schools — and shut down mean girls</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Phoenix rally shows how gospel of Trump can be more attractive than the Bible</td>\n",
              "      <td>[(Phoenix, GPE), (Trump, LOC), (Bible, WORK_OF_ART)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>What I learned after 9/11 can help devastated restaurant workers now</td>\n",
              "      <td>[(9/11, DATE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>The secret war many doctors on the coronavirus front lines now face</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>The complicated balancing act of church, state and the coronavirus</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Trump rants about fraud. But here's the secret to keeping voting by mail secure.</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Comedy is no coronavirus vaccine. But can it make living through a pandemic easier?</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Arguments over reclining are so yesterday. Here’s what the future of flying looks like.</td>\n",
              "      <td>[(yesterday, DATE)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>NBC</td>\n",
              "      <td>https://www.nbcnews.com/health/coronavirus</td>\n",
              "      <td>Does air conditioning spread the coronavirus?</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   website  ...                                                                       entities\n",
              "0      CNN  ...                                                                             []\n",
              "1      CNN  ...                                                           [(Coronavirus, ORG)]\n",
              "2      CNN  ...                [(VMAs, ORG), (Brooklyn, GPE), (August, DATE), (New York, GPE)]\n",
              "3      CNN  ...  [(New York, GPE), (Wednesday, DATE), (NYC, ORG), (3, CARDINAL), (Cuomo, ORG)]\n",
              "4      CNN  ...                            [(8, CARDINAL), (New York, GPE), (yesterday, DATE)]\n",
              "5      CNN  ...                                                              [(Broadway, FAC)]\n",
              "6      CNN  ...                                                     [(California county, GPE)]\n",
              "7      CNN  ...                                                                             []\n",
              "8      CNN  ...                                                [(Amazon, ORG), (Germany, GPE)]\n",
              "9      CNN  ...                                  [(Florida, GPE), (more than 5,000, CARDINAL)]\n",
              "10     NBC  ...                                                                 [(Trump, LOC)]\n",
              "11     NBC  ...                                                          [(500,000, CARDINAL)]\n",
              "12     NBC  ...                                                                             []\n",
              "13     NBC  ...                                     [(At least 85, CARDINAL), (Michigan, GPE)]\n",
              "14     NBC  ...                                                        [(thousands, CARDINAL)]\n",
              "15     NBC  ...                                 [(Chase Rice, PERSON), (Chris Janson, PERSON)]\n",
              "16     NBC  ...                                         [(California, GPE), (seven, CARDINAL)]\n",
              "17     NBC  ...                                                          [(one-two, CARDINAL)]\n",
              "18     NBC  ...                                                                             []\n",
              "19     NBC  ...                                                 [(Florida, GPE), (Texas, GPE)]\n",
              "20     NBC  ...                                                               [(Germany, GPE)]\n",
              "21     NBC  ...                                                                             []\n",
              "22     NBC  ...                           [(Phoenix, GPE), (Trump, LOC), (Bible, WORK_OF_ART)]\n",
              "23     NBC  ...                                                                 [(9/11, DATE)]\n",
              "24     NBC  ...                                                                             []\n",
              "25     NBC  ...                                                                             []\n",
              "26     NBC  ...                                                                             []\n",
              "27     NBC  ...                                                                             []\n",
              "28     NBC  ...                                                            [(yesterday, DATE)]\n",
              "29     NBC  ...                                                                             []\n",
              "\n",
              "[30 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVTODptX0ox8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 640
        },
        "outputId": "3c3db0a7-8ab4-4dad-8cd3-7a87a2995cd5"
      },
      "source": [
        "pd.concat([news_df[['website','url','headline']],news_df['entities'].apply(pd.Series)],axis=1)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>website</th>\n",
              "      <th>url</th>\n",
              "      <th>headline</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>What you need to know</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>Coronavirus pandemic \"not even close to being over,\" WHO says</td>\n",
              "      <td>(Coronavirus, ORG)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>VMAs will be held in Brooklyn in August, New York governor says</td>\n",
              "      <td>(VMAs, ORG)</td>\n",
              "      <td>(Brooklyn, GPE)</td>\n",
              "      <td>(August, DATE)</td>\n",
              "      <td>(New York, GPE)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>New York will decide by Wednesday whether NYC can have indoor dining in phase 3, Cuomo says</td>\n",
              "      <td>(New York, GPE)</td>\n",
              "      <td>(Wednesday, DATE)</td>\n",
              "      <td>(NYC, ORG)</td>\n",
              "      <td>(3, CARDINAL)</td>\n",
              "      <td>(Cuomo, ORG)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CNN</td>\n",
              "      <td>https://edition.cnn.com/world/live-news/coronavirus-pandemic-06-29-20-intl/index.html</td>\n",
              "      <td>8 people died from coronavirus in New York yesterday</td>\n",
              "      <td>(8, CARDINAL)</td>\n",
              "      <td>(New York, GPE)</td>\n",
              "      <td>(yesterday, DATE)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>CNBC</td>\n",
              "      <td>https://www.cnbc.com/id/10000108/device/rss/rss.html</td>\n",
              "      <td>Delta and the pilots' labor union reached an agreement for early retirements as the carrier scrambles to reduce headcount.</td>\n",
              "      <td>(Delta, ORG)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>CNBC</td>\n",
              "      <td>https://www.cnbc.com/id/10000108/device/rss/rss.html</td>\n",
              "      <td>\"We're always busy in the summertime and what we're seeing now is a typical summer for us,\" Memorial Hermann Health System CEO Dr. David Callender told CNBC.</td>\n",
              "      <td>(Memorial Hermann Health System CEO, WORK_OF_ART)</td>\n",
              "      <td>(David Callender, PERSON)</td>\n",
              "      <td>(CNBC, ORG)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>CNBC</td>\n",
              "      <td>https://www.cnbc.com/id/10000108/device/rss/rss.html</td>\n",
              "      <td>San Francisco Mayor London Breed said in a tweet Friday that the city will temporarily delay its planned reopening for Monday, which included nail salons, barbershops, outdoor bars.</td>\n",
              "      <td>(San Francisco, GPE)</td>\n",
              "      <td>(London Breed, PERSON)</td>\n",
              "      <td>(Friday, DATE)</td>\n",
              "      <td>(Monday, DATE)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>CNBC</td>\n",
              "      <td>https://www.cnbc.com/id/10000108/device/rss/rss.html</td>\n",
              "      <td>Ro, which previously specialized in men's health, is now generating $250 million in annual revenue after expanding its product line.</td>\n",
              "      <td>($250 million, MONEY)</td>\n",
              "      <td>(annual, DATE)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>CNBC</td>\n",
              "      <td>https://www.cnbc.com/id/10000108/device/rss/rss.html</td>\n",
              "      <td>\"People are infecting other people and ultimately you will infect someone who's vulnerable,\" Dr. Anthony Fauci, director of the National Institute of Allergy and Infectious Diseases, said during a press briefing.</td>\n",
              "      <td>(Anthony Fauci, PERSON)</td>\n",
              "      <td>(the National Institute of Allergy and Infectious Diseases, ORG)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>92 rows × 10 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   website  ...    6\n",
              "0      CNN  ...  NaN\n",
              "1      CNN  ...  NaN\n",
              "2      CNN  ...  NaN\n",
              "3      CNN  ...  NaN\n",
              "4      CNN  ...  NaN\n",
              "..     ...  ...  ...\n",
              "87    CNBC  ...  NaN\n",
              "88    CNBC  ...  NaN\n",
              "89    CNBC  ...  NaN\n",
              "90    CNBC  ...  NaN\n",
              "91    CNBC  ...  NaN\n",
              "\n",
              "[92 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    }
  ]
}