-
Notifications
You must be signed in to change notification settings - Fork 93
/
hp_download.py
118 lines (91 loc) · 4.24 KB
/
hp_download.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
#!/usr/bin/env python3
# --------------
# Load modules
# --------------
from bs4 import BeautifulSoup
from collections import defaultdict
from random import choice
from time import sleep
import urllib.request
import csv
import logging
# --------------------------------
# Scraping and parsing functions
# --------------------------------
def get_url(url):
user_agents = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.1.17 (KHTML, like Gecko) Version/7.1 Safari/537.85.10',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36'
]
agent = choice(user_agents)
wait = choice(range(10, 17))
response = urllib.request.Request(url, headers={'User-Agent': agent})
handler = urllib.request.urlopen(response).read()
sleep(wait)
return(handler)
def parse_chapter(url):
print("Parsing {0}".format(url))
try:
soup = BeautifulSoup(get_url(url))
except:
logging.warning("Error with {0}".format(url))
print("Error. Moving on.")
return(None)
content_section = soup.select('.ContentCss')
content_clean = [x.text.replace('\u3000', '') for x in content_section][0]
content = {}
content['content'] = content_clean.split('\n')
content['url'] = url
split_url = url.split('/')[-1].replace('.html', '').split('_')
content['book'] = ' '.join(split_url[0:-1])
content['chapter'] = split_url[-1]
return(content)
def convert_to_long(chapter):
long_results = defaultdict(dict)
entry_num = -1
for x in chapter['content']:
entry_num += 1
long_results[entry_num]['book'] = chapter['book']
long_results[entry_num]['chapter'] = chapter['chapter']
long_results[entry_num]['paragraph'] = entry_num + 1
long_results[entry_num]['text'] = x
return(long_results)
# --------------------------------
# Build a list of URLs to scrape
# --------------------------------
url_base = 'http://www.readfreeonline.net/OnlineBooks/'
specific_books = ['Harry_Potter_and_the_Sorcerers_Stone/Harry_Potter_and_the_Sorcerers_Stone_CHAPTER.html',
'Harry_Potter_and_The_Chamber_Of_Secrets/Harry_Potter_and_The_Chamber_Of_Secrets_CHAPTER.html',
'Harry_Potter_and_the_Prisoner_of_Azkaban/Harry_Potter_and_the_Prisoner_of_Azkaban_CHAPTER.html',
'Harry_Potter_and_the_Goblet_of_Fire/Harry_Potter_and_the_Goblet_of_Fire_CHAPTER.html',
'Harry_Potter_and_the_Order_of_the_Phoenix/Harry_Potter_and_the_Order_of_the_Phoenix_CHAPTER.html',
'Harry_Potter_and_the_Half-Blood_Prince/Harry_Potter_and_the_Half-Blood_Prince_CHAPTER.html',
'Harry_Potter_and_the_Deathly_Hallows/Harry_Potter_and_the_Deathly_Hallows_CHAPTER.html']
chapters = [17, 19, 22, 37, 38, 30, 37]
urls_to_parse = []
for i in range(0, len(specific_books)):
for j in range(1, chapters[i]+1):
final_url = url_base + specific_books[i].replace('CHAPTER', str(j))
urls_to_parse.append(final_url)
print(urls_to_parse.index('http://www.readfreeonline.net/OnlineBooks/Harry_Potter_and_the_Prisoner_of_Azkaban/Harry_Potter_and_the_Prisoner_of_Azkaban_20.html'))
# ---------------------------------------
# Scrape all those URLs and save to CSV
# ---------------------------------------
# # Set up log
# logging.basicConfig(filename='errors.log', filemode='w', level=logging.DEBUG,
# format='%(levelname)s %(asctime)s: %(message)s')
# logging.captureWarnings(True)
# csv_started = False
# fieldnames = ['book', 'chapter', 'paragraph', 'text']
# for url in urls_to_parse[0:2]:
# contents = convert_to_long(parse_chapter(url))
# for key, value in contents.items():
# w = csv.DictWriter(open('hp.csv', 'a'), fieldnames)
# if csv_started is False:
# w.writeheader()
# csv_started = True
# w.writerow(value)