Completed task 4

MdManzerAlam_Task4/crawler/Crawler.py

@@ -0,0 +1,84 @@
+from urllib.parse import *
+from urllib import robotparser as rb
+from urllib.robotparser import RobotFileParser
+import requests
+from bs4 import BeautifulSoup
+from pymongo import MongoClient
+from models.query import Query
+class Crawler:
+    def __init__(self) -> None:
+        self.client = MongoClient('localhost',27017)
+        pass
+    def start_crawl(self,url):
+        robo_file_parser = rb.RobotFileParser()
+        robo_file_parser.set_url(url+'/robots.txt')
+        robo_file_parser.read()
+        self.url = url
+        self.crawl(url=url,depth=2,rf_parser=robo_file_parser)
+
+    def crawl(self,url,depth,rf_parser:RobotFileParser):
+        print(url+" | "+str(depth))
+
+        if depth == 0:
+            # todo parse single page
+            if rf_parser.can_fetch("*",url=url):
+
+                try:
+                    request = requests.get(url)
+                    soup = BeautifulSoup(request.text,"lxml")
+                    desc = ""
+                    try:
+                        for p_tag in soup.find_all("p"):
+                            desc+=str(p_tag.text)
+                    except:
+                        desc = "empty"
+                    title = ""
+                    try:
+                        title = soup.find('title').text
+                    except:
+                        title = "empty"
+                    query = Query(title=title,url=url,description=desc)
+                    query.save(self.client)
+                except:
+                    pass
+            pass
+        else :
+            # todo parse multiple pages
+            if rf_parser.can_fetch("*",url=url): 
+                try:
+                    request = requests.get(url)
+                    soup = BeautifulSoup(request.text,"lxml")
+                    desc = ""
+                    try:
+                        for p_tag in soup.find_all("p"):
+                            desc+=str(p_tag)
+                    except:
+                        desc = "empty"
+                    title = ""
+                    try:
+                        title = soup.find('title').text
+                    except:
+                        title = "empty"
+                    query = Query(title=title,url=url,description=desc)
+                    query.save(self.client)
+                    in_page_links_raw = soup.find_all("a")
+                    in_page_links = []
+                    for a_tag in in_page_links_raw:
+                        try:
+                            param = str(a_tag["href"])
+                        except:
+                            param = ""
+                            continue
+                        if param.startswith(" "):
+                            param.removeprefix(" ")
+                        if param.startswith("http"):
+                            in_page_links.append(param)
+                        elif param.startswith("//www"):
+                            in_page_links.append("https://"+param.removeprefix("//"))
+                        else:
+                            in_page_links.append(self.url+param)
+                    for link in in_page_links:
+                        self.crawl(link,depth-1,rf_parser)
+                except:
+                    pass
+            pass

MdManzerAlam_Task4/hello.py

@@ -0,0 +1,2 @@
+import nltk
+nltk.download()

MdManzerAlam_Task4/main.py

@@ -0,0 +1,3 @@
+from crawler.Crawler import Crawler
+obj = Crawler()
+obj.start_crawl("https://www.rottentomatoes.com")

MdManzerAlam_Task4/models/query.py

@@ -0,0 +1,33 @@
+import json
+from pymongo import MongoClient
+from pymongo import TEXT
+class Query:
+    def __init__(self,title,url,description) -> None:
+        self.title = title
+        self.url = url
+        self.description = description
+
+    def __str__(self) -> str:
+        obj = {
+            'url' : self.url,
+            'title' : self.title,
+            'description' : self.description,
+        }
+        return json.dumps(obj)
+
+    def save(self,client):
+        obj = {
+            'url' : self.url,
+            'title' : self.title,
+            'description' : self.description,
+        }
+        db = client.gluggle
+        collection = db.queries
+        collection.insert_one(obj)
+        collection.create_index([
+            ('url', TEXT),
+            ('title', TEXT),
+            ('description', TEXT),
+            ], name='search_results', default_language='english')
+        client.close()
+

MdManzerAlam_Task4/static/css/index.css

@@ -0,0 +1,81 @@
+@import url('https://fonts.googleapis.com/css2?family=Comfortaa:wght@300;400;500;600;700&family=Righteous&display=swap');
+*{
+    font-family: 'Comfortaa',sans-serif;
+    border: none;
+    outline: none;
+    text-decoration: none;
+    list-style: none;
+    margin: 0px;
+    padding: 0px;
+}
+html{
+    font-size: 16px;
+}
+h1{
+    font-size: 3rem;
+    font-weight: 400;
+    letter-spacing: 0.2rem;
+    font-family: 'Righteous', cursive;
+}
+h3{
+    font-size: 1.5rem;
+    font-weight: 400;
+}
+@media screen and (max-width: 500px) {
+    html{
+        font-size: 14px;
+    }
+}
+body{
+    background-color: rgb(32, 32, 32);
+    color: hsl(0, 0%, 98%);
+}
+header{
+    position: fixed;
+    top: 50%;
+    left: 50%;
+    -webkit-transform: translate(-50%, -50%);
+    transform: translate(-50%, -50%);
+    display: flex;
+    justify-content: center;
+    flex-direction: column;
+    align-items: center;
+    text-align: center;
+    width: 80%;
+    /* max-width: 1000px; */
+    gap: 2em;
+}
+header form{
+    width: 100%;
+}
+header fieldset{
+    display: flex;
+    gap: 1em;
+    width: 100%;
+    max-width: 900px;
+    margin: 0px auto;
+}
+input,button{
+    width: 80%;
+    font-size: 1.25rem;
+    padding: 1em 1em;
+    border-radius: 100px;
+    color: hsl(0, 0%, 98%);
+    background-color: hsl(0, 0%, 4%);
+    margin: 0.5em auto;
+}
+#search-btn{
+    background-color: #4fa;
+    font-family: 'Righteous', cursive;
+    color: black;
+    font-weight: 700;
+    padding: 0.5em 1em;
+    font-weight: 400;
+    letter-spacing: 0.2rem;
+    width: auto;
+}
+@media screen and (max-width: 500px) {
+    fieldset{
+        flex-direction: column;
+    }
+}

MdManzerAlam_Task4/static/css/results.css

@@ -0,0 +1,145 @@
+@import url('https://fonts.googleapis.com/css2?family=Comfortaa:wght@300;400;500;600;700&family=Righteous&display=swap');
+*{
+    font-family: 'Comfortaa',sans-serif;
+    border: none;
+    outline: none;
+    text-decoration: none;
+    list-style: none;
+    margin: 0px;
+    padding: 0px;
+    text-decoration: none;
+    color: inherit;
+}
+html{
+    font-size: 16px;
+}
+h1{
+    font-size: 3rem;
+    font-weight: 400;
+    margin: auto auto;
+    letter-spacing: 0.2rem;
+    font-family: 'Righteous', cursive;
+}
+h3{
+    font-size: 1.5rem;
+    font-weight: 400;
+}
+@media screen and (max-width: 500px) {
+    html{
+        font-size: 14px;
+    }
+}
+body{
+    background-color: rgb(32, 32, 32);
+    color: hsl(0, 0%, 98%);
+}
+header{
+    position: sticky;
+    margin: 1rem auto;
+    display: flex;
+    justify-content: center;
+    flex-direction: column;
+    align-items: center;
+    text-align: center;
+    width: 80%;
+    gap: 2em;
+}
+header form{
+    width: 100%;
+    display: flex;
+    justify-content: center;
+    gap: 1em;
+}
+header fieldset{
+    display: flex;
+    gap: 1em;
+    width: 100%;
+    max-width: 1000px;
+    margin: 0px auto;
+    text-align: center;
+}
+input,button{
+    width: 80%;
+    font-size: 1.25rem;
+    padding: 1em 1em;
+    border-radius: 100px;
+    color: hsl(0, 0%, 98%);
+    background-color: hsl(0, 0%, 4%);
+    margin: 0.5em auto;
+}
+#search-btn{
+    background-color: #4fa;
+    font-family: 'Righteous', cursive;
+    color: black;
+    font-weight: 700;
+    padding: 0.5em 1em;
+    font-weight: 400;
+    letter-spacing: 0.2rem;
+    width: auto;
+}
+main{
+    padding: 0.5em 10vw;
+}
+.results{
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    gap: 1em;
+    margin-top: 1em;
+}
+.results .card{
+    border-radius: 1em;
+    padding: 0.75em 1em;
+    background-color: hsl(0, 0%, 20%);
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+.card a{
+    color: #4fa;
+}
+@media screen and (max-width: 500px) {
+    header form{
+        flex-direction: column;
+    }
+    header form h1{
+        font-size: 2rem;
+    }
+    header fieldset{
+        flex-direction: row;
+        gap: 0.5em;
+    }
+    header{
+        width: 90%;
+    }
+    main{
+        padding: 0.5em 5vw;
+    }
+}
+.pagination{
+    margin: 1em auto;
+    display: flex;
+    justify-content: center;
+    gap: 2em;
+}
+.pagination li{
+    margin: auto 0px;
+}
+.pagination span,li{
+    background-color: hsl(0, 0%, 20%);
+    padding: 0.5em;
+    border-radius: 0.5rem;
+}
+.pagination li.active{
+    background-color: hsl(0, 0%, 50%);
+    padding: 0.5em;
+    border-radius: 0.5rem;
+}
+.pagination li.active span{
+    background-color: hsl(0, 0%, 50%);
+    padding: 0.5em;
+    border-radius: 0.5rem;
+}
+.disabled{
+    display: none;
+}

MdManzerAlam_Task4/templates/index.html

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Gluggle</title>
+    <link rel="stylesheet" href="{{url_for('static',filename='css/index.css')}}">
+</head>
+<body>
+    <header>
+        <h1>Gluggle</h1>
+        <h3>Website Crawler made with Python and ❤️</h3>
+        <form action="/results" method="get">
+            <fieldset>
+                <input type="text" name="query" placeholder="Enter what you want to search" id="id_query">
+
+                <button type="submit" id="search-btn"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"><path d="M23.809 21.646l-6.205-6.205c1.167-1.605 1.857-3.579 1.857-5.711 0-5.365-4.365-9.73-9.731-9.73-5.365 0-9.73 4.365-9.73 9.73 0 5.366 4.365 9.73 9.73 9.73 2.034 0 3.923-.627 5.487-1.698l6.238 6.238 2.354-2.354zm-20.955-11.916c0-3.792 3.085-6.877 6.877-6.877s6.877 3.085 6.877 6.877-3.085 6.877-6.877 6.877c-3.793 0-6.877-3.085-6.877-6.877z"/></svg></button>
+            </fieldset>
+        </form>
+    </header>
+</body>
+</html>