perf: Merge pull request #143 from pelias/refactor-for-layer-fallbacks

BREAKING CHANGE: Refactor for layer fallbacks, no longer calls every layer on lookup

.gitignore

@@ -28,3 +28,5 @@ node_modules
 
 # Webstorm IDE files
 .idea
+
+wof-*-data.json

index.js

@@ -5,10 +5,10 @@ const _ = require('lodash');
 const os = require('os');
 
 module.exports = {
-  create: () => {
+  create: (layers) => {
     if (peliasConfig.imports.adminLookup.enabled) {
       const datapath = peliasConfig.imports.whosonfirst.datapath;
-      const resolver = require('./src/localPipResolver')(datapath);
+      const resolver = require('./src/localPipResolver')(datapath, layers);
 
       return require('./src/lookupStream')(resolver,
         peliasConfig.imports.adminLookup.maxConcurrentReqs);

package.json

@@ -28,13 +28,14 @@
   "dependencies": {
     "async": "^2.1.4",
     "joi": "^11.0.1",
-    "lodash": "^4.6.0",
+    "lodash": "^4.17.4",
     "pelias-config": "2.12.1",
     "pelias-logger": "0.2.0",
     "pelias-parallel-stream": "0.1.0",
-    "pelias-whosonfirst": "2.19.10",
+    "pelias-whosonfirst": "2.20.0",
     "polygon-lookup": "^2.1.0",
     "simplify-js": "^1.2.1",
+    "temp": "^0.8.3",
     "through2": "^2.0.0",
     "through2-filter": "^2.0.0",
     "through2-map": "^3.0.0",
@@ -52,8 +53,7 @@
     "proxyquire": "^1.7.10",
     "semantic-release": "^8.0.0",
     "tap-dot": "^1.0.1",
-    "tape": "^4.2.2",
-    "temp": "^0.8.3"
+    "tape": "^4.2.2"
   },
   "pre-commit": [
     "lint",

src/localPipResolver.js

@@ -2,17 +2,18 @@
 
 const logger = require('pelias-logger').get('wof-admin-lookup');
 const createPipService = require('./pip/index').create;
+const _ = require('lodash');
 
 /**
  * LocalPIPService class
  *
  * @param {object} [pipService] optional, primarily used for testing
  * @constructor
  */
-function LocalPipService(datapath) {
+function LocalPipService(datapath, layers) {
   const self = this;
 
-  createPipService(datapath, [], false, (err, service) => {
+  createPipService(datapath, _.defaultTo(layers, []), false, (err, service) => {
      self.pipService = service;
   });
 
@@ -82,6 +83,6 @@ LocalPipService.prototype.end = function end() {
  * @param {string} [datapath]
  * @returns {LocalPIPService}
  */
-module.exports = (datapath) => {
-  return new LocalPipService(datapath);
+module.exports = (datapath, layers) => {
+  return new LocalPipService(datapath, layers);
 };

src/pip/components/extractFields.js

@@ -24,7 +24,6 @@ module.exports.create = function(enableLocalizedNames) {
         Id: wofData.properties['wof:id'],
         Name: getName(wofData, enableLocalizedNames),
         Placetype: wofData.properties['wof:placetype'],
-        Hierarchy: wofData.properties['wof:hierarchy'],
         Centroid: {
           lat: wofData.properties['geom:latitude'],
           lon: wofData.properties['geom:longitude']
@@ -34,6 +33,14 @@ module.exports.create = function(enableLocalizedNames) {
       geometry: wofData.geometry
     };
 
+    if (!_.isEmpty(wofData.properties['wof:hierarchy'])) {
+      // if there's a wof:hierarchy, condense down to just the ids
+      res.properties.Hierarchy = _.map(wofData.properties['wof:hierarchy'], hierarchy => _.values(hierarchy));
+    } else {
+      // otherwise, synthesize a hierarchy from the records' id
+      res.properties.Hierarchy = [ [ res.properties.Id ] ];
+    }
+
     // use different abbreviation field for country
     if (res.properties.Placetype === 'country') {
       res.properties.Abbrev = wofData.properties['wof:country_alpha3'];

src/pip/components/filterOutCitylessNeighbourhoods.js

@@ -0,0 +1,24 @@
+const filter = require('through2-filter');
+const _ = require('lodash');
+const logger = require('pelias-logger').get('wof-pip-service:filterOutCitylessNeighbourhoods');
+
+// This filter stream returns false when wof:placetype = neighbourhood AND
+// wof:hierarchy lacks both locality_id AND localadmin_id
+// returns true otherwise
+//
+// It's needed because there are some unfortunate important WOF neighbourhoods
+// that are currently incomplete
+module.exports.create = function create() {
+  return filter.obj(wofData => {
+    // in this case, conformsTo cleans up a somewhat messy if statement
+    if (_.conformsTo(wofData.properties, {
+      'wof:placetype': placetype => placetype === 'neighbourhood',
+      'wof:hierarchy': hierarchy => !_.has(hierarchy[0], 'locality_id') && !_.has(hierarchy[0], 'localadmin_id')
+    })) {
+      logger.debug(`skipping ${wofData.properties['wof:id']}: neighbourhood without locality or localadmin`);
+      return false;
+    }
+
+    return true;
+  });
+};

src/pip/components/filterOutHierarchylessNeighbourhoods.js

@@ -0,0 +1,24 @@
+const filter = require('through2-filter');
+const _ = require('lodash');
+const logger = require('pelias-logger').get('wof-pip-service:filterOutHierarchylessNeighbourhoods');
+
+// This filter stream returns false when wof:placetype = neighbourhood AND
+// wof:hierarchy is empty
+// returns true otherwise
+//
+// It's needed because there are some unfortunate important WOF neighbourhoods
+// that are currently incomplete
+module.exports.create = function create() {
+  return filter.obj(wofData => {
+    // in this case, conformsTo cleans up a somewhat messy if statement
+    if (_.conformsTo(wofData.properties, {
+      'wof:placetype': placetype => placetype === 'neighbourhood',
+      'wof:hierarchy': hierarchy => _.isEmpty(hierarchy)
+    })) {
+      logger.debug(`skipping ${wofData.properties['wof:id']}: neighbourhood with empty hierarchy`);
+      return false;
+    }
+
+    return true;
+  });
+};