Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Solves Google Block HTTP 503 #5

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Added cookie support for better handling Google 503 Errors
limonkufu committed Aug 8, 2018
commit 6427e3d6407e299b97cce790e6f9de8fac72df39
20 changes: 15 additions & 5 deletions mendeley_add_citations.py
Original file line number Diff line number Diff line change
@@ -23,9 +23,12 @@
# skip documents already processed
skip_documents = True

# save cookie in init
save_cookie = True

# min and max sleep time in seconds between Scholar requests
min_sleep_time_sec = 5
max_sleep_time_sec = 20
min_sleep_time_sec = 10
max_sleep_time_sec = 30

# minimal difflib.SequenceMatcher ratio for
# Mendeley x Scholar title matching
@@ -101,9 +104,17 @@ def has_citation_tag(tags, patterns):
return False


def process(document):
def process(document):
scholar = ScholarQuerier()
query = SearchScholarQuery()

# save cookie at first paper
global save_cookie
if save_cookie:
query.set_phrase("quantum theory")
scholar.send_query(query)
save_cookie = False

query.set_phrase(document.title)
scholar.send_query(query)
scholar_articles = scholar.articles
@@ -167,8 +178,7 @@ def add_citations():
mendeley_session = get_session_from_cookies()
if 'process_id' not in session or 'ids' not in user_data:
session['process_id'] = 0
user_data['ids'] = [doc.id for doc in mendeley_session.documents.iter()]

user_data['ids'] = [doc.id for doc in mendeley_session.documents.iter()]
if skip_documents:
while True:
document = mendeley_session.documents.get(
6 changes: 3 additions & 3 deletions scholar/scholar.py
Original file line number Diff line number Diff line change
@@ -249,7 +249,7 @@ class ScholarConf(object):

# If set, we will use this file to read/save cookies to enable
# cookie use across sessions.
COOKIE_JAR_FILE = None
COOKIE_JAR_FILE = 'scholar-cookies.txt'

class ScholarUtils(object):
"""A wrapper for various utensils that come in handy."""
@@ -1307,8 +1307,8 @@ def main():
else:
txt(querier, with_globals=options.txt_globals)

if options.cookie_file:
querier.save_cookies()
# if options.cookie_file:
querier.save_cookies()

return 0