From cccec373ac0c1fcc0ca856be3d0aed10cbcaeb93 Mon Sep 17 00:00:00 2001 From: Sander Pick Date: Sat, 18 Feb 2017 13:07:48 -0800 Subject: [PATCH] update deps, fix getTicks --- lib/27crags.js | 30 +++++++++++++++++------------- package.json | 27 +++++++++++++++------------ scripts/crag_add_missing.js | 10 +++++----- scripts/crag_loader.js | 2 +- scripts/crag_map.js | 2 +- scripts/crag_scraper.js | 1 - test/test.js | 28 ++++++++++++++++++++++++++++ tests/test.js | 15 --------------- 8 files changed, 67 insertions(+), 48 deletions(-) create mode 100755 test/test.js delete mode 100755 tests/test.js diff --git a/lib/27crags.js b/lib/27crags.js index 533d439..cf16ca1 100644 --- a/lib/27crags.js +++ b/lib/27crags.js @@ -1,7 +1,7 @@ // Functionality for indexing content for search. var _ = require('underscore'); -_.mixin(require('underscore.string')); +var _s = require('underscore.string'); var util = require('util'); var Step = require('step'); var cheerio = require('cheerio'); @@ -45,7 +45,7 @@ function format(str) { str = str.replace(/\\/g, '').trim(); if (str.length > 3) str = str.toLowerCase(); return _.map(str.split(' '), function (w) { - return _.capitalize(w); + return _s.capitalize(w); }).join(' '); } @@ -131,24 +131,29 @@ var boulderMap = [ ]; var createTickObj = function($, els) { + + var grade = _s.trim($(els.get(5)).find('.grade').text()) + if (grade === '') { + grade = _s.trim($(els.get(5)).text()) + } + var obj = { date: moment($(els.get(0)), 'YYYY-MM-DD'), sent: true, - style: styleMap[$(els.get(8)).text()], - ascent: _.trim(format($(els.get(1)).find('a').text())), + style: styleMap[_s.trim($(els.get(6)).find('.ascent-type').text())], + ascent: _s.trim(format($($(els.get(2)).find('a').get(0)).text())), recommended: false, - crag: _.trim(format($(els.get(2)).find('a').text())), + crag: _s.trim(format($(els.get(3)).find('a').text())), cragCountry: null, cragCity: null, ascentSector: null, - first: $(els.get(7)).text().indexOf('FA') !== -1, + first: $(els.get(6)).text().indexOf('FA') !== -1, feel: null, secondGo: null, - note: $(els.get(9)).text(), - rating: $(els.get(6)).find('.full').length, - type: $(els.get(3)).text().indexOf('Boulder') !== -1 ? 'b' : 'r', - // Remove + and - from the V grades - grade: $(els.get(4)).text().replace(/[+-]/g, '') + note: _s.trim($(els.get(2)).find('.ascent-details').text()), + rating: $(els.get(2)).find('.stars').children().length, + type: $(els.get(4)).text().indexOf('Boulder') !== -1 ? 'b' : 'r', + grade: grade } // Try to map to Island's crag names @@ -177,7 +182,6 @@ var createTickObj = function($, els) { } return obj; - } // Searches for a username, and if exists, returns the username with geo @@ -294,7 +298,7 @@ exports.scrapeCrags = function(start, end, cb) { var $ = cheerio.load(body); - // I had to serialize all the crag requests to not overwhelm 27crags, + // I had to serialize all the crag requests to not overwhelm 27crags, // so this is quick and dirty way to do that var rows = $('table').find('tr:not(:has(th))'); var len = rows.length; diff --git a/package.json b/package.json index 52dd17d..9c4699e 100644 --- a/package.json +++ b/package.json @@ -2,24 +2,27 @@ "private": true, "author": "The Island (git://github.com/The-Island)", "name": "island-lib27crags", - "version": "0.0.1", + "version": "0.0.2", "repository": { "type": "git", "url": "git://github.com/Islandio/island-lib27crags.git" }, "main": "lib/27crags", "dependencies": { - "async": "^1.3.0", - "cheerio": "^0.18.0", - "csv-parse": "^0.1.3", - "json2csv": "^2.4.1", - "moment": "^2.8.4", - "request": "^2.58.0", - "step": "0.0.5", - "underscore": "^1.4.4", - "underscore.string": "~2.3.1" + "async": "~2.1.4", + "csv-parse": "^1.2.0", + "json2csv": "^3.7.3", + "cheerio": "^0.22.0", + "moment": "^2.17.1", + "request": "~2.79.0", + "step": "1.0.0", + "underscore": "~1.8.3", + "underscore.string": "~3.3.4" }, - "engines": { - "node": "~0.10.26" + "devDependencies": { + "mocha": "^3.2.0" + }, + "scripts": { + "test": "mocha" } } diff --git a/scripts/crag_add_missing.js b/scripts/crag_add_missing.js index 1a29157..a77527c 100755 --- a/scripts/crag_add_missing.js +++ b/scripts/crag_add_missing.js @@ -4,7 +4,7 @@ var lib27crags = require('../lib/27crags'); var fs = require('fs'); var request = require('request'); var _ = require('underscore'); -_.mixin(require('underscore.string')); +var _s = require('underscore.string'); var Step = require('Step'); var csvParse = require('csv-parse'); var json2csv = require('json2csv'); @@ -29,7 +29,7 @@ Step( fs.readFile(file, this); }, // Parse CSV - function (err, csv) { + function (err, csv) { if (err) return this(err); csvParse(csv, this); }, @@ -107,18 +107,18 @@ Step( }, this); }, function (err, data) { - if (err) return this(err); + if (err) return this(err); var newFile = file.split('.csv')[0] + '_new.csv'; var csv = _.map(data, function(d) { d = _.map(d, function(el) { if (el && el.indexOf(',') !== -1) { el = '\"' + el + '\"'; - } + } return el; }); return d.toString(); }); - csv = csv.join('\n'); + csv = csv.join('\n'); console.log('Added ' + counter + ' crags to Island'); console.log('Writing to ' + newFile); fs.writeFile(newFile, csv, this); diff --git a/scripts/crag_loader.js b/scripts/crag_loader.js index 418839b..44592ec 100755 --- a/scripts/crag_loader.js +++ b/scripts/crag_loader.js @@ -4,7 +4,7 @@ var lib27crags = require('../lib/27crags'); var fs = require('fs'); var request = require('request'); var _ = require('underscore'); -_.mixin(require('underscore.string')); +var _s = require('underscore.string'); var Step = require('Step'); var file = process.argv[2]; diff --git a/scripts/crag_map.js b/scripts/crag_map.js index 5f985f6..c32480a 100755 --- a/scripts/crag_map.js +++ b/scripts/crag_map.js @@ -4,7 +4,7 @@ var lib27crags = require('../lib/27crags'); var fs = require('fs'); var request = require('request'); var _ = require('underscore'); -_.mixin(require('underscore.string')); +var _s = require('underscore.string'); var Step = require('Step'); var json2csv = require('json2csv'); diff --git a/scripts/crag_scraper.js b/scripts/crag_scraper.js index eae0696..270c27d 100755 --- a/scripts/crag_scraper.js +++ b/scripts/crag_scraper.js @@ -20,4 +20,3 @@ lib27crags.scrapeCrags(start, end, function(err, res) { }); }); }); - diff --git a/test/test.js b/test/test.js new file mode 100755 index 0000000..fbcaa83 --- /dev/null +++ b/test/test.js @@ -0,0 +1,28 @@ +var lib27crags = require('../lib/27crags'); +var assert = require('assert'); + +describe('island-27crags', function() { + this.timeout(10000); + var userId; + describe('#searchUser()', function() { + it('should find the user', function(done) { + lib27crags.searchUser('Paul Robinson', function(err, res) { + if (err) done(err); + assert.ok(res.length > 0) + assert.equal(res[0].name, 'Paul Robinson') + userId = res[0].userId + done() + }); + }); + }); + + describe('#getTicks()', function() { + it('should get the user\'s ticks', function(done) { + lib27crags.getTicks(userId, function(err, res) { + if (err) done(err); + assert.ok(res.length > 0) + done() + }); + }); + }); +}); diff --git a/tests/test.js b/tests/test.js deleted file mode 100755 index b9e905d..0000000 --- a/tests/test.js +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env node - -var lib27crags = require('../lib/27crags'); - -/* -lib27crags.searchUser('bill', function(err, res) { - console.log(err, res); -}); -*/ - -lib27crags.getTicks('wmmurray', function(err, res) { - //console.log(err, res.length); - console.log(res[0]); -}); -