-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
31 lines (28 loc) · 816 Bytes
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
var Crawler = require("crawler");
var fs = require("fs");
var c = new Crawler({
maxConnections : 10,
// This will be called for each crawled page
callback : function (error, res, done) {
if(error){
console.log(error);
}else{
var $ = res.$;
// $ is Cheerio by default
//a lean implementation of core jQuery designed specifically for the server
console.log(res.body);
fs.createWriteStream(res.options.filename).write(res.body);
}
done();
}
});
c.queue({
uri: 'http://www.google.com',
// this will override the 'preRequest' defined in crawler
preRequest: function(options, done) {
setTimeout(function() {
console.log(options);
done();
}, 1000)
}
});