forked from dcaribou/transfermarkt-scraper
-
Notifications
You must be signed in to change notification settings - Fork 0
/
settings.py
32 lines (24 loc) · 915 Bytes
/
settings.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
# -*- coding: utf-8 -*-
BOT_NAME = 'tfmkt'
SPIDER_MODULES = ['tfmkt.spiders']
NEWSPIDER_MODULE = 'tfmkt.spiders'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
FEED_FORMAT = 'jsonlines'
FEED_URI = 'stdout:'
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
EXTENSIONS = {
'scrapy.extensions.closespider.CloseSpider': 500
}
DOWNLOADER_MIDDLEWARES = {
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 500
}
CLOSESPIDER_PAGECOUNT = 0
LOG_LEVEL = 'ERROR'
# HttpCacheMiddleware settings
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
HTTPCACHE_ENABLED = True
HTTPCACHE_DIR = 'httpcache'
# https://docs.scrapy.org/en/latest/topics/request-response.html?highlight=REQUEST_FINGERPRINTER_IMPLEMENTATION#std-setting-REQUEST_FINGERPRINTER_IMPLEMENTATION
REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'