v0.54.5 Fix node 8 promise compatibility (#1291)

* Update docker-compose-js docs * Change cluster services to be less dependent on bluebird * Add pRace function to promises utils * Fix build cleanup * Convert assets storage to async await vs bluebird to avoid node 8 compatibility * Update worker-shutdown to avoid using bluebird when possible * Update to spec to avoid using bluebird specific APIs when possible * Improvements to autoloading assets and race conditions when multiple nodes come up immediately. * bump packages * make linting errors not fail the build * Fixes to asset loading
terascope · Aug 1, 2019 · 86dd713 · 86dd713
1 parent b32ac12
commit 86dd713
Show file tree

Hide file tree

Showing 38 changed files with 676 additions and 692 deletions.
diff --git a/docs/packages/docker-compose-js/overview.md b/docs/packages/docker-compose-js/overview.md
@@ -29,7 +29,7 @@ The examples below show the common patterns for calls. Since this is a wrapper a
 Note: Each function takes an object for keyword parameters as the last argument.
 
 ```js
-var compose = require('@terascope/docker-compose-js')('docker-compose.yaml');
+const compose = require('@terascope/docker-compose-js')('docker-compose.yaml');
 
 compose.up()
     .then(function() {
@@ -43,17 +43,17 @@ compose.up()
     })
     .then(console.log)
     .catch(function(error) {
-        console.log(error);
-    })
-    .finally(function() {
-        compose.down();
+        console.error(error);
     })
+    .then(function() {
+        return compose.down();
+    });
 ```
 
 Example of scaling a particular task
 
 ```js
-var compose = require('@terascope/docker-compose-js')('docker-compose.yaml');
+const compose = require('@terascope/docker-compose-js')('docker-compose.yaml');
 
 compose.up()
     .then(function(result) {
@@ -66,7 +66,7 @@ compose.up()
     .catch(function(error) {
         console.log(error);
     })
-    .finally(function() {
+    .then(function() {
         return compose.down();
     })
 ```
diff --git a/package.json b/package.json
@@ -20,7 +20,7 @@
         "test:watch": "jest --coverage=false --notify --onlyChanged --watch",
         "test:e2e": "yarn --cwd ./e2e test",
         "lint": "env FORCE_COLOR=1 lerna run lint --no-bail && yarn lint:docs",
-        "lint:docs": "markdownlint docs/*.md docs/**/*.md docs/**/**/*.md packages/*/*.md README.md",
+        "lint:docs": "markdownlint docs/*.md docs/**/*.md docs/**/**/*.md packages/*/*.md README.md || echo '* lint:docs failed but it's okay",
         "lint:fix": "env FORCE_COLOR=1 lerna run lint:fix --no-bail"
     },
     "displayName": "Teraslice",

diff --git a/packages/chunked-file-reader/package.json b/packages/chunked-file-reader/package.json
@@ -22,7 +22,7 @@
         "test:watch": "jest --coverage=false --notify --watch --onlyChanged"
     },
     "dependencies": {
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "bluebird": "^3.5.5",
         "csvtojson": "^2.0.8",
         "lodash": "^4.17.11"

diff --git a/packages/data-access-plugin/package.json b/packages/data-access-plugin/package.json
@@ -31,7 +31,7 @@
         "@terascope/data-access": "^0.12.5",
         "@terascope/data-types": "^0.5.0",
         "@terascope/elasticsearch-api": "^2.1.0",
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "apollo-server-express": "^2.6.5",
         "graphql": "^14.3.1",
         "graphql-iso-date": "^3.6.1",

diff --git a/packages/data-access/package.json b/packages/data-access/package.json
@@ -27,7 +27,7 @@
     },
     "dependencies": {
         "@terascope/data-types": "^0.5.0",
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "elasticsearch-store": "^0.10.2",
         "xlucene-evaluator": "^0.9.5"
     },

diff --git a/packages/data-types/package.json b/packages/data-types/package.json
@@ -28,7 +28,7 @@
         "test:watch": "jest --coverage=false --notify --watch --onlyChanged"
     },
     "dependencies": {
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "graphql": "^14.3.1",
         "lodash.defaultsdeep": "^4.6.1",
         "lodash.set": "^4.3.2",

diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
@@ -19,7 +19,7 @@
         "test:watch": "jest --coverage=false --notify --watch --onlyChanged"
     },
     "dependencies": {
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "bluebird": "^3.5.5",
         "lodash": "^4.17.11"
     },

diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
@@ -26,7 +26,7 @@
     },
     "dependencies": {
         "@terascope/data-types": "^0.5.0",
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "ajv": "^6.10.0",
         "nanoid": "^2.0.3",
         "rambda": "^2.11.1",

diff --git a/packages/job-components/package.json b/packages/job-components/package.json
@@ -34,7 +34,7 @@
     },
     "dependencies": {
         "@terascope/queue": "^1.1.6",
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "convict": "^4.4.1",
         "datemath-parser": "^1.0.6",
         "uuid": "^3.3.2"

diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json
@@ -24,7 +24,7 @@
     },
     "dependencies": {
         "@terascope/elasticsearch-api": "^2.1.0",
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "agentkeepalive": "^4.0.2",
         "aws-sdk": "^2.496.0",
         "bluebird": "^3.5.5",

diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json
@@ -33,7 +33,7 @@
     },
     "dependencies": {
         "@terascope/queue": "^1.1.6",
-        "@terascope/utils": "^0.13.0",
+        "@terascope/utils": "^0.14.0",
         "nanoid": "^2.0.3",
         "p-event": "^4.1.0",
         "porty": "^3.1.1",

diff --git a/packages/teraslice/lib/cluster/cluster_master.js b/packages/teraslice/lib/cluster/cluster_master.js
@@ -4,6 +4,7 @@ const Promise = require('bluebird');
 const _ = require('lodash');
 const express = require('express');
 const request = require('request');
+const { pDelay } = require('@terascope/utils');
 const { ClusterMaster } = require('@terascope/teraslice-messaging');
 const { makeLogger } = require('../workers/helpers/terafoundation');
 const ExecutionService = require('./services/execution');
@@ -64,45 +65,44 @@ module.exports = function _clusterMaster(context) {
         }
         return isAssetServiceUp().then((isUp) => {
             if (isUp) return Promise.resolve();
-            return Promise.delay(1000).then(() => waitForAssetsService(timeoutAt));
+            return pDelay(1000).then(() => waitForAssetsService(timeoutAt));
         });
     }
 
     return {
-        initialize() {
-            return Promise.resolve()
-                .then(() => clusterMasterServer.start())
-                .then(() => {
-                    logger.info(`cluster master listening on port ${clusterConfig.port}`);
-                    return ExecutionService(context, { clusterMasterServer });
-                })
-                .then((executionService) => {
-                    logger.debug('execution service has been instantiated');
-                    context.services.execution = executionService;
-                    return JobService(context);
-                })
-                .then((jobsService) => {
-                    logger.debug('job service has been instantiated');
-                    context.services.jobs = jobsService;
-                    // give the assets service a bit to come up
-                    const fiveMinutes = 5 * 60 * 1000;
-                    return waitForAssetsService(Date.now() + fiveMinutes);
-                })
-                .then(() => APIService(context, app, { assetsUrl, clusterMasterServer }))
-                .then((apiService) => {
-                    logger.debug('api service has been instantiated');
-                    context.services.api = apiService;
-                    return makeLogs(context);
-                })
-                .then(() => {
-                    logger.info('cluster master is ready!');
-                    running = true;
-                })
-                .catch((err) => {
-                    logger.error(err, 'error during service initialization');
-                    running = false;
-                    return Promise.reject(err);
+        async initialize() {
+            try {
+                await clusterMasterServer.start();
+                logger.info(`cluster master listening on port ${clusterConfig.port}`);
+
+                const executionService = await ExecutionService(context, { clusterMasterServer });
+                logger.debug('execution service has been instantiated');
+                context.services.execution = executionService;
+
+                const jobsService = await JobService(context);
+                logger.debug('job service has been instantiated');
+                context.services.jobs = jobsService;
+
+                // give the assets service a bit to come up
+                const fiveMinutes = 5 * 60 * 1000;
+                waitForAssetsService(Date.now() + fiveMinutes);
+
+                const apiService = await APIService(context, app, {
+                    assetsUrl,
+                    clusterMasterServer
                 });
+                logger.debug('api service has been instantiated');
+
+                context.services.api = apiService;
+                await makeLogs(context);
+
+                logger.info('cluster master is ready!');
+                running = true;
+            } catch (err) {
+                logger.error(err, 'error during service initialization');
+                running = false;
+                throw err;
+            }
         },
         run() {
             return new Promise((resolve) => {
@@ -118,16 +118,14 @@ module.exports = function _clusterMaster(context) {
                 }, 1000);
             });
         },
-        shutdown() {
+        async shutdown() {
             running = false;
             logger.info('cluster_master is shutting down');
             clusterMasterServer.isShuttingDown = true;
-            return Promise.resolve()
-                .then(() => {
-                    const services = _.values(context.services);
-                    return Promise.map(services, service => service.shutdown());
-                })
-                .then(() => clusterMasterServer.shutdown());
+
+            const promises = Object.values(context.services).map(service => service.shutdown());
+            await Promise.all(promises);
+            return clusterMasterServer.shutdown();
         },
     };
 };
diff --git a/packages/teraslice/lib/cluster/services/cluster/backends/native/index.js b/packages/teraslice/lib/cluster/services/cluster/backends/native/index.js
@@ -3,7 +3,7 @@
 const _ = require('lodash');
 const Promise = require('bluebird');
 const Queue = require('@terascope/queue');
-const { TSError, parseError } = require('@terascope/utils');
+const { TSError, getFullErrorStack, pDelay } = require('@terascope/utils');
 const { makeLogger } = require('../../../../../workers/helpers/terafoundation');
 const stateUtils = require('../state-utils');
 const Messaging = require('./messaging');
@@ -369,10 +369,12 @@ module.exports = function nativeClustering(context, clusterMasterServer, executi
                     response: true
                 })
                     .catch((err) => {
-                        const errMsg = parseError(err, true);
-                        logger.error(err, `failed to allocate execution_controller to ${slicerNodeID}`);
-                        errorNodes[slicerNodeID] = errMsg;
-                        return Promise.reject(errMsg);
+                        const error = new TSError(err, {
+                            reason: `failed to allocate execution_controller to ${slicerNodeID}`
+                        });
+                        logger.error(error);
+                        errorNodes[slicerNodeID] = getFullErrorStack(error);
+                        return Promise.reject(error);
                     });
             });
     }
@@ -424,7 +426,7 @@ module.exports = function nativeClustering(context, clusterMasterServer, executi
         if (messaging) {
             return messaging.shutdown();
         }
-        return Promise.delay(100);
+        return pDelay(100);
     }
 
     function addWorkers(execution, workerNum) {
@@ -531,7 +533,7 @@ module.exports = function nativeClustering(context, clusterMasterServer, executi
                 return;
             }
 
-            Promise.map(nodes, (node) => {
+            const promises = nodes.map((node) => {
                 const sendingMsg = Object.assign({}, messageData, {
                     to: 'node_master',
                     address: node.node_id,
@@ -542,7 +544,9 @@ module.exports = function nativeClustering(context, clusterMasterServer, executi
                 logger.trace(`notifying node ${node.node_id} to stop execution ${exId}`, sendingMsg);
 
                 return messaging.send(sendingMsg);
-            })
+            });
+
+            Promise.all(promises)
                 .then(() => {
                     resolve(true);
                 })

diff --git a/packages/teraslice/lib/cluster/services/cluster/backends/native/messaging.js b/packages/teraslice/lib/cluster/services/cluster/backends/native/messaging.js
@@ -4,6 +4,7 @@ const _ = require('lodash');
 const shortid = require('shortid');
 const Promise = require('bluebird');
 const Queue = require('@terascope/queue');
+const { pDelay } = require('@terascope/utils');
 
 // messages send to cluster_master
 const clusterMasterMessages = {
@@ -538,7 +539,7 @@ module.exports = function messaging(context, logger) {
     function shutdown() {
         if (io && _.isFunction(io.close)) {
             io.close();
-            return Promise.delay(100);
+            return pDelay(100);
         }
         return Promise.resolve();
     }