OpenInnovation

prashant/README.md.txt

@@ -0,0 +1,3 @@
+I was learning through project . It was tough for me to implement at this stage. 
+
+I can contribute by giving idea that we can have "User-LogIn" by which search results recommendation can vary from person to person.

prashant/Task4/README.md.txt

@@ -0,0 +1,3 @@
+I was learning through project . It was tough for me to implement at this stage. 
+
+I can contribute by giving idea that we can have "User-LogIn" by which search results recommendation can vary from person to person.

prashant/Task4/app.py

@@ -0,0 +1,78 @@
+from flask import Flask, render_template, request
+import pymongo
+import os
+from flask_paginate import Pagination, get_page_args
+from ranking import Ranking
+from query_processing import QueryProcessing
+import time
+
+
+app = Flask(__name__)
+
+
+@app.route('/')
+def entry_point():
+    return render_template('home.html')
+
+
+@app.route('/search_results')
+def search_results():
+    connect_url = 'mongodb://127.0.0.1:27017/'
+
+    client = pymongo.MongoClient(connect_url, connect=False)
+
+    db = client.results
+
+    search_string = request.args.get('search')
+
+    processor = QueryProcessing(search_string)
+    keywords = processor.processor()
+
+    query = []
+
+    start = time.time()
+
+    for keyword in keywords:
+        query.extend(db.search_results.find(
+            {'$text': {'$search': keyword, '$caseSensitive': False}}))
+
+    end = time.time()
+    print(f"time to execute: {end-start}")
+
+    search_result = []
+
+    for doc in query:
+        exist = False
+        for result in search_result:
+            if result['title'] == doc['title'] or result['url'] == doc['url']:
+                exist = True
+                break
+
+        if exist == False:
+            search_result.append(doc)
+
+    rank = Ranking(search_result, search_string)
+
+    ranked_result = rank.sorted_results()
+
+    client.close()
+
+    page, per_page, offset = get_page_args(page_parameter='page',
+                                           per_page_parameter='per_page')
+
+    total = len(ranked_result)
+
+    pagination = Pagination(page=page, per_page=per_page, total=total,
+                            css_framework='bootstrap4')
+
+    return render_template('search.html',
+                           search_result=ranked_result[offset:offset+per_page],
+                           page=page,
+                           per_page=per_page,
+                           pagination=pagination,
+                           search_string=search_string
+                           )
+
+
+if __name__ == '__main__':
+    app.run(debug=True, port=8000)

prashant/Task4/crawler/crawler.py

@@ -0,0 +1,118 @@
+from bs4 import BeautifulSoup
+import requests
+import pymongo
+import os
+import urllib.parse
+from popular_links import Popularity
+import sys
+
+
+class Crawler():
+    connect_url = 'mongodb://127.0.0.1:27017/'
+
+    client = pymongo.MongoClient(connect_url)
+
+    db = client.results
+
+    search_results = []
+
+    url_count = 1
+
+    def start_crawl(self, url, depth):
+        robot_url = urllib.parse.urljoin(url, '/robots.txt')
+        try:
+            robots = requests.get(robot_url)
+        except BaseException:
+            print("robots not found")
+            self.crawl(url, depth)
+
+        soup = BeautifulSoup(robots.text, 'lxml')
+
+        sauce = soup.find('p').text
+
+        content = sauce.split()
+
+        disallowed_links = []
+
+        for word in content:
+            if word[0] == '/':
+                disallowed_links.append(urllib.parse.urljoin(url, word))
+            elif 'http' in word:
+                disallowed_links.append(word)
+        print("got rebots!!!")
+
+        self.crawl(url, depth, disallowed_links)
+
+    def crawl(self, url, depth, *disallowed_links):
+
+        try:
+            print(f'Crawling url {self.url_count}: {url} at depth: {depth}')
+            self.url_count += 1
+            response = requests.get(url)
+
+        except BaseException:
+            print(f'Failed to perform HTTP GET request on {url}')
+            return
+
+        soup = BeautifulSoup(response.text, 'lxml')
+
+        try:
+            title = soup.find('title').text
+            description = ''
+
+            for tag in soup.findAll():
+                if tag.name == 'p':
+                    description += tag.text.strip().replace('\n', '')
+
+        except BaseException:
+            print("Failed to retrieve title and description\n")
+            return
+
+        popularity = Popularity(url)
+        popularity_score = popularity.popularity_score()
+
+        query = {
+            'url': url,
+            'title': title,
+            'description': description,
+            'score': 0,
+            'popularity': popularity_score,
+        }
+
+        search_results = self.db.search_results
+
+        search_results.insert_one(query)
+
+        search_results.create_index([
+            ('url', pymongo.TEXT),
+            ('title', pymongo.TEXT),
+            ('description', pymongo.TEXT),
+            ('score', 1),
+            ('popularity', 1)
+        ], name='search_results', default_language='english')
+
+        if depth == 0:
+            return
+
+        links = soup.findAll('a')
+
+        for link in links:
+            try:
+                if link['href'] not in disallowed_links[0]:
+                    if 'http' in link['href']:
+                        self.crawl(link['href'], depth -
+                                   1, disallowed_links[0])
+                    else:
+                        link['href'] = urllib.parse.urljoin(url, link['href'])
+                        self.crawl(link['href'], depth-1, disallowed_links[0])
+            except KeyError:
+                print("no links to retrieve in the website entered!!!")
+                pass
+
+        self.client.close()
+
+
+crawler = Crawler()
+
+crawler.start_crawl(
+    sys.argv[1], int(sys.argv[2]))

prashant/Task4/crawler/popular_links.py

@@ -0,0 +1,18 @@
+class Popularity():
+    popular_domains = ['https://en.wikipedia.org/', 'https://www.python.org/', 'https://www.rottentomatoes.com/',
+                       'https://pypi.org/', 'https://www.indiatoday.in/', 'https://www.geeksforgeeks.org/',
+                       'https://stackoverflow.com/']
+
+    ps = 0
+
+    def __init__(self, url):
+        self.url = url
+
+    def popularity_score(self):
+        for domain in self.popular_domains:
+            if domain == self.url:
+                self.ps += 100/len(self.popular_domains)
+            if domain in self.url:
+                self.ps += 100/len(self.popular_domains)
+
+        return self.ps

prashant/Task4/query_processing.py

@@ -0,0 +1,34 @@
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+from nltk.stem.porter import PorterStemmer
+from spellchecker import SpellChecker
+import string
+import re
+
+class QueryProcessing():
+    def __init__(self, search_string):
+        self.search_string = search_string
+
+    def processor(self):
+        old_query = self.search_string
+
+        self.search_string = self.search_string.lower()
+
+        translator = str.maketrans('', '', string.punctuation)
+        self.search_string = self.search_string.translate(translator)
+
+        self.search_string = " ".join(self.search_string.split())
+
+        stop_words = set(stopwords.words("english"))
+        word_tokens = word_tokenize(self.search_string)
+        tokens = [word for word in word_tokens if word not in stop_words]
+
+        stemmer = PorterStemmer()
+        tokens = [stemmer.stem(word) for word in tokens]
+
+        spell = SpellChecker()
+        for i in range(len(tokens)):
+            tokens[i] = spell.correction(tokens[i])
+
+        return tokens

prashant/Task4/ranking.py

@@ -0,0 +1,57 @@
+from operator import itemgetter
+import string
+
+class Ranking:
+    def __init__(self, results, query):
+        self.results = results
+        self.query = query
+
+    def search(self):
+        res = []
+        filtered = []
+        if '"' in self.query:
+            x = '"'
+            y = ' '
+            z = ' " '
+            mytable = self.query.maketrans(x, y, z)
+            res.insert(0, self.query.translate(mytable))
+        else:
+            if ':' in self.query:
+                key = self.query.split(':')[0]
+                fil = self.query.split(':')[1]
+                print(key)
+                print(fil)
+                for result in self.results:
+                    if fil.lower() in result['url'].lower():
+                        filtered.append(result)
+                    self.results = filtered
+            elif '-' in self.query:
+                key = self.query.split('-')[0]
+                fil = self.query.split('-')[1]
+                for result in self.results:
+                    if fil.lower() not in result['title'].lower() or fil.lower() not in result['description'].lower():
+                        filtered.append(result)
+                self.result = filtered
+            else:
+                key = self.query
+
+            res = key.split()
+        return res
+    def ranked_results(self):
+        keywords = self.search()
+        for key in keywords:
+            for result in self.results:
+                if key.lower() in result['title'].lower():
+                    result['score'] +=2
+                if key.lower() in result['description'].lower():
+                    result['score'] +=1
+
+        return self.results
+
+    def sorted_results(self):
+        ranked_searches = self.ranked_results()
+
+        sorted_searches = sorted(
+            ranked_searches, key=itemgetter('popularity', 'score'), reverse=True)
+
+        return sorted_searches

prashant/Task4/templates/base.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width-device, initial-scale=1">
+
+    <link href= "https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet"
+    integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous">
+
+    <style>
+        .form-control:focus {
+            border-color: #43971b;
+            box-shadow:0px 1px 1px rgba(0,0,0,0.075)inset, 0px 0px 8px rgba(255, 100,255, 0.5);
+        }
+
+        .dropdown{
+            float: left;
+        }
+    </style>
+
+    <title>GLUGLE</title>
+</head>
+
+<body style="background-color: rgb(224,226,226);">
+    {% block content %}{% endblock %}
+
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"
+        integrity="sha384-/bQdsTh/da6pkI1MST/rWKFNjaCP5gBSY4sEBT38Q/9RBh9AH40zEOg7Hlq2THRZ"
+        crossorigin="anonymous"></script>
+
+</body>
+</html>

prashant/Task4/templates/home.html

@@ -0,0 +1,18 @@
+{% extends 'base.html' %}
+
+{% block content %}
+
+<div class="col">
+    <div class="col mt-5">
+        <img src="{{url_for('static', filename='../static/images/ask1.png')}}" class="mx-auto d-block" style="width:25%; height:25%;">
+        <h1 style="color: rgb(91, 94, 92); text-align:center; font-family:'Times New Roman', Times, sherif; margin-top:15px;padding:15px;" class="mt-2"><b>Glugle Search</b></h1>
+    </div>
+
+    <form class="mt-5 container" name="search" style="width:50%" action="/search_results">
+        <div class="col-6 mx-auto input-group">
+            <input type="text" class="form-control" name="search" placeholder="search..">
+            <button type="submit" class="btn btn-success">SEARCH</button>
+        </div>
+    </form>
+</div>
+{% endblock %}