From 27bcef900e46d306f71035a647ec08c3a95b7cc8 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Dec 2016 14:08:38 -0500 Subject: [PATCH 1/2] Set JShint JS version to ES6 --- .jshintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.jshintrc b/.jshintrc index f9df9b9d..8a7f48c0 100644 --- a/.jshintrc +++ b/.jshintrc @@ -1,4 +1,5 @@ { + "esversion": 6, "node": true, "curly": true, "eqeqeq": true, From 6a4dcc3e74270d135cf36615caf722fabc373e5c Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Tue, 13 Dec 2016 14:08:59 -0500 Subject: [PATCH 2/2] Rewrite downloader The downloader is now similar to the WOF downloader in that it uses child_process.exec and curl to download data. This seems to be more reliable than using request and piping to a file. There were also issues with the progress bar package, so rather than sort them out, this allows us to simply remove them. This change probably will help fix https://github.com/pelias/geonames/issues/26, but it's not 100% certain. --- lib/tasks/download.js | 40 +++++++++++++++++++++------------------- package.json | 2 -- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/tasks/download.js b/lib/tasks/download.js index b84046cf..0ed40bf7 100644 --- a/lib/tasks/download.js +++ b/lib/tasks/download.js @@ -1,30 +1,32 @@ +'use strict'; -var fs = require('fs-extra'), - util = require('util'), - request = require('request'), - _ = require('lodash'), - progress = require('../../util/streamProgressBar'), - logger = require( 'pelias-logger' ).get( 'geonames' ); +const child_process = require('child_process'); +const logger = require( 'pelias-logger' ).get( 'geonames' ); // use datapath setting from your config file -var settings = require('pelias-config').generate(); -var basepath = settings.imports.geonames.datapath; +const settings = require('pelias-config').generate(); +const basepath = settings.imports.geonames.datapath; module.exports = function (filename) { + const remoteFilePath = `http://download.geonames.org/export/dump/${filename}.zip`; + const localFileName = `${basepath}/${filename}.zip`; - var remoteFilePath = util.format( 'http://download.geonames.org/export/dump/%s.zip', filename ); - var localFileName = util.format( '%s/%s.zip', basepath, filename ); + logger.info( 'downloading datafile from:', remoteFilePath ); - fs.mkdirs('data', function(error) { - if( error ){ - logger.error( error ); - return; - } - logger.info( 'downloading datafile from:', remoteFilePath ); + const command = `curl ${remoteFilePath} > ${localFileName}`; - request.get( remoteFilePath ) - .pipe( progress( _.padEnd( localFileName, 30 ) ) ) - .pipe( fs.createWriteStream( localFileName ) ); + const job = child_process.exec(command); + + job.stdout.on('data', (data) => { + process.stdout.write(data); + }); + + job.stderr.on('data', (data) => { + process.stderr.write(data); }); + job.on('close', (code) => { + console.log(`Geonames download finished with exit code ${code}`); + process.exitCode = code; + }); }; diff --git a/package.json b/package.json index d1ced8cb..33f1d6fd 100644 --- a/package.json +++ b/package.json @@ -52,8 +52,6 @@ "pelias-logger": "0.1.0", "pelias-model": "4.4.0", "pelias-wof-admin-lookup": "2.8.0", - "progress": "^1.1.5", - "progress-stream": "^1.2.0", "request": "^2.34.0", "through2": "^2.0.1", "through2-filter": "^2.0.0"