webscrape-reviews-nltk-beta.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1cc9b70c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import selenium\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options as ChromeOptions\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "import re\n",
    "import requests\n",
    "# from newspaper import Article\n",
    "import spacy\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "from string import punctuation\n",
    "from heapq import nlargest\n",
    "import textwrap as tr\n",
    "import urllib.request\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize import sent_tokenize\n",
    "import heapq\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cbabe096",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "please input total number pages for google search results (assuming 10 links per page)5\n",
      "please enter google search keywordslisi+block dental review\n",
      "please enter filename to save the resultslisi15\n"
     ]
    }
   ],
   "source": [
    "# questions\n",
    "\n",
    "n_pages = (input(\"please input total number pages for google search results (assuming 10 links per page)\"))\n",
    "                 \n",
    "query =(input(\"please enter google search keywords\"))\n",
    "                 \n",
    "filename = (input(\"please enter filename to save the results\"))\n",
    "                 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e69f6db1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert(url):\n",
    "    scraped_data = urllib.request.urlopen(url)\n",
    "    article = scraped_data.read()\n",
    "\n",
    "    parsed_article = BeautifulSoup(article,'lxml')\n",
    "\n",
    "    paragraphs = parsed_article.find_all('p')\n",
    "\n",
    "    article_text = \"\"\n",
    "\n",
    "    for p in paragraphs:\n",
    "        article_text += p.text\n",
    "\n",
    "    # Removing Square Brackets and Extra Spaces\n",
    "    article_text = re.sub(r'\\[[0-9]*\\]', ' ', article_text)\n",
    "    article_text = re.sub(r'\\s+', ' ', article_text)\n",
    "\n",
    "    # Removing special characters and digits\n",
    "    formatted_article_text = re.sub('[^a-zA-Z]', ' ', article_text )\n",
    "    formatted_article_text = re.sub(r'\\s+', ' ', formatted_article_text)\n",
    "\n",
    "    sentence_list = sent_tokenize(article_text)\n",
    "\n",
    "    stopwords = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "    word_frequencies = {}\n",
    "    for word in nltk.word_tokenize(formatted_article_text):\n",
    "        if word not in stopwords:\n",
    "            if word not in word_frequencies.keys():\n",
    "                word_frequencies[word] = 1\n",
    "            else:\n",
    "                word_frequencies[word] += 1\n",
    "\n",
    "    maximum_frequncy = max(word_frequencies.values())\n",
    "\n",
    "    for word in word_frequencies.keys():\n",
    "        word_frequencies[word] = (word_frequencies[word]/maximum_frequncy)\n",
    "\n",
    "    sentence_scores = {}\n",
    "    for sent in sentence_list:\n",
    "        for word in nltk.word_tokenize(sent.lower()):\n",
    "            if word in word_frequencies.keys():\n",
    "                if len(sent.split(' ')) < 30:\n",
    "                    if sent not in sentence_scores.keys():\n",
    "                        sentence_scores[sent] = word_frequencies[word]\n",
    "                    else:\n",
    "                        sentence_scores[sent] += word_frequencies[word]\n",
    "\n",
    "    summary_sentences = heapq.nlargest(7, sentence_scores, key=sentence_scores.get)\n",
    "\n",
    "    summary = ' '.join(summary_sentences)\n",
    "\n",
    "    return summary, len(formatted_article_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "08de1580",
   "metadata": {},
   "outputs": [],
   "source": [
    "# path = r'C:\\Users\\StephenHaynesAus\\OneDrive - GC Dental\\webscraping\\chromedriver.exe'\n",
    "path = r'chromedriver.exe'\n",
    "\n",
    "s = Service(path)\n",
    "options=Options()\n",
    "options.add_argument('--headless')\n",
    "\n",
    "driver = webdriver.Chrome(service=s, options=options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dbd67539",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "on it........\n"
     ]
    }
   ],
   "source": [
    "# query to obtain links\n",
    "\n",
    "print(\"on it........\")\n",
    "\n",
    "# initiate empty list to append results\n",
    "links = []\n",
    "\n",
    "# query = 'intext:lisi+block research'\n",
    "\n",
    "# filename = 'lisi11'\n",
    "\n",
    "# specify number of pages to load on SERP\n",
    "n_pages = int(n_pages)\n",
    "# n_pages = 5\n",
    "\n",
    "for page in range(1, n_pages):\n",
    "    # capture 10 links per page\n",
    "    url = \"http://www.google.com/search?q=\" + query + \"&start=\" + str((page - 1)*10)\n",
    "    driver.get(url)\n",
    "    # for each entire page\n",
    "    soup = BeautifulSoup(driver.page_source,'html.parser')\n",
    "    # look for the specifc class name for \"Lisi Block\"\n",
    "    search = soup.find_all('div',class_ = \"yuRUbf\")\n",
    "    # get every links in that page\n",
    "    for g in search:\n",
    "        links.append(g.a.get('href'))\n",
    "\n",
    "## convert to 2D list:\n",
    "n=1\n",
    "links = [links[i:i+n] for i in range(0,len(links),n)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "024234b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.dentaleconomics.com/science-tech/article/14277628/pearls-for-your-practice-gc-initial-lisi-block not ok\n",
      "\n",
      "length of article is 623, link: https://www.dentalproductsreport.com/view/5ws-gc-initial-lisi-block\n",
      "\n",
      "length of article is 551, link: https://www.nature.com/articles/s41415-021-3601-z\n",
      "\n",
      "length of article is 894, link: https://www.nature.com/articles/s41415-022-4287-6\n",
      "\n",
      "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6902521/ not ok\n",
      "\n",
      "https://www.sciencedirect.com/science/article/pii/S1013905220307458 not ok\n",
      "\n",
      "length of article is 1010, link: https://www.dental-tribune.com/c/gc-international/news/interview-cad-cam-materials-play-a-key-role-in-our-research-because-they-are-the-future/\n",
      "\n",
      "https://www.researchgate.net/figure/Mechanical-properties-of-IPS-emax-press-and-GC-Initial-LiSi-press-materials_tbl1_337852941 not ok\n",
      "\n",
      "length of article is 116, link: https://www.semanticscholar.org/paper/Fracture-behavior-of-CAD%2FCAM-ceramic-crowns-before-Garoushi-S%C3%A4ilynoja/9656af1dee1d81e61642153f7e78c7d4caa857ec\n",
      "\n",
      "https://www.dentalcompare.com/Restorative-Dentistry/4871-CAD-CAM-Blocks/Compare/?compare=3400150,3228854,42436,41212,41214&catid=4871 not ok\n",
      "\n",
      "length of article is 1013, link: https://www.ivoclar.com/makeitemax\n",
      "\n",
      "length of article is 1261, link: https://www.oceanicdentallab.com.au/lithium-discilicate-emax-lisi-press/\n",
      "\n",
      "length of article is 1308, link: https://link.springer.com/article/10.1007/s00784-022-04611-w\n",
      "\n",
      "length of article is 952, link: https://pocketdentistry.com/flexural-strength-fracture-toughness-three-body-wear-and-martens-parameters-of-pressable-lithium-x-silicate-ceramics/\n",
      "\n",
      "https://www.mdpi.com/2073-4360/13/7/1130/htm not ok\n",
      "\n",
      "length of article is 133, link: https://www.dme-medical.com/gc-initial-lisi-press.html\n",
      "\n",
      "https://www.zerodonto.com/en/2017/11/shake-stone-new-lisi-sinterized-blocks/ not ok\n",
      "\n",
      "done!\n"
     ]
    }
   ],
   "source": [
    "# parse every function and summarize the article\n",
    "\n",
    "h=1\n",
    "all_text=[]\n",
    "\n",
    "\n",
    "for url in links:\n",
    "    url = ','.join(url)\n",
    "    if (re.search(r'.pdf',url) or re.search(r'gcamerica',url) or re.search(r'gcaustralasia',url) \n",
    "        or re.search(r'europe.gc',url) or re.search(r'gcasia',url) or re.search('gceurope',url) or\n",
    "       re.search(r'dentalsky',url) or re.search(r'henryschein',url) or re.search(r'kentexpress', url) or\n",
    "       re.search(r'onlinelibrary',url) or re.search(r'insourceonline',url) or re.search(r'inqmobility',url) or\n",
    "       re.search(r'dentaltix',url) or re.search(r'issuu',url) or re.search(r'dentalpathway',url) \n",
    "        or re.search(r'bfmulholland',url) or re.search(r'dentistryshow',url) or re.search(r'accessdata.fda.gov',url) or\n",
    "       re.search(r'dentaleducationhub',url) or re.search(r'3di-dental-digital',url) or re.search(r'ur.booksc.eu',url) or\n",
    "       re.search(r'dentinaltubules',url) or re.search(r'tdsc',url) or re.search(r'pattersondenta',url) or\n",
    "       re.search(r'eudent',url) or re.search('dentex',url) or re.search(r'ddgroup',url) or re.search(r'basiqdental',url) or\n",
    "       re.search(r'unident',url) or re.search(r'gacd',url) or re.search(r'pearsondental',url) or\n",
    "       re.search(r'laboratoire.dentanor',url) or re.search(r'proclinic.es',url) or re.search(r'journalofosseo',url) or\n",
    "       re.search(r'e-meditrans.pl',url) or re.search(r'unidentdanmark',url) or re.search(r'goodreads',url) or\n",
    "       re.search(r'alphabond',url) or re.search(r'amazon',url) or re.search(r'dme-medical',url)):\n",
    "        \n",
    "        continue\n",
    "    else:\n",
    "        try:\n",
    "            article, length = convert(url)\n",
    "            if len(article) < 30:\n",
    "                continue\n",
    "            else:\n",
    "                with open(filename+\".txt\",'a',encoding='utf-8') as f:\n",
    "                    f.write(f\"{h}) Link of article: {url}, {length} words before summary \\n\\n{tr.fill(article,width=120)}\\n\\n\\n\")\n",
    "                    h+=1\n",
    "                print(f\"length of article is {len(article)}, link: {url}\")\n",
    "                print()\n",
    "        except:\n",
    "            print(f'{url} not ok')\n",
    "            print()\n",
    "        \n",
    "                \n",
    "print(\"done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "194e6c26",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0435241a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc1bdb16",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76b2db8d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2aea3101",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fdae166",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}