diff --git a/CHANGELOG.yaml b/CHANGELOG.yaml index f77adb74..a5341bcc 100644 --- a/CHANGELOG.yaml +++ b/CHANGELOG.yaml @@ -1,11 +1,14 @@ unreleased: breaking changes: - GH-1052 Dropped support for Node < v18 + new features: + - GH-1032 Enhanced performance when operating on buffers in Node environment fixed bugs: - GH-1036 Fixed `uncaughtException` event listener not being removed - GH-1034 Fixed an issue where sandbox crashes for large response body chores: - GH-1033 Update ci.yml to run coverage on latest node version + - GH-1032 Add support for configuring module resolver based on environment 5.1.2: date: 2024-09-04 diff --git a/lib/bundle/index.js b/lib/bundle/index.js index 6a289a3b..7a2088ca 100644 --- a/lib/bundle/index.js +++ b/lib/bundle/index.js @@ -57,11 +57,18 @@ class Bundle { */ this.compress = options.compress; + this.preferBrowserResolver = options.preferBrowserResolver; + // process any list of modules externally required and also accommodate the use of built-ins if needed _.forEach(options.require, (options, resolve) => { // allow resolution override where the required module is resolved // from a different name than the one provided in options - options.resolve && (resolve = options.resolve); + if (this.preferBrowserResolver && options.resolveBrowser) { + resolve = options.resolveBrowser; + } + else if (options.resolve) { + resolve = options.resolve; + } // set the name using which the module is exported to the one marked as the module name (only in case when // one is not explicitly provided in options.) @@ -88,10 +95,25 @@ class Bundle { this.bundler.bundle((err, bundle) => { if (err) { return done(err); } + // Expose only the required node built-ins to be used in vendor modules + // These will be cleared out by sandbox when the bundle is loaded + const safeExposeBuiltIns = ` + if (typeof require === 'function') { + globalThis._nodeRequires = { + buffer: require('buffer') + }; + + // prevent access to node's require function + require = null; + } + `, + + bundleString = safeExposeBuiltIns + bundle.toString(); + // bail out if compression is disabled - if (!this.compress) { return done(null, bundle.toString()); } + if (!this.compress) { return done(null, bundleString); } - minify(bundle.toString(), { + minify(bundleString, { compress: { drop_console: true // discard calls to console.* functions }, diff --git a/lib/environment.js b/lib/environment.js index add774ff..756937ed 100644 --- a/lib/environment.js +++ b/lib/environment.js @@ -11,7 +11,12 @@ module.exports = { util: { preferBuiltin: true, glob: true }, stream: { preferBuiltin: true, glob: true }, string_decoder: { preferBuiltin: true, glob: true }, - buffer: { resolve: 'buffer/index.js', expose: 'buffer', glob: true }, + buffer: { + resolve: '../vendor/buffer/index.js', + resolveBrowser: '../vendor/buffer/index.browser.js', + expose: 'buffer', + glob: true + }, url: { preferBuiltin: true, glob: true }, punycode: { preferBuiltin: true, glob: true }, querystring: { preferBuiltin: true, glob: true }, diff --git a/lib/sandbox/index.js b/lib/sandbox/index.js index cde7c047..fa152b21 100644 --- a/lib/sandbox/index.js +++ b/lib/sandbox/index.js @@ -22,6 +22,13 @@ // Setup Timerz before we delete the global timers require('./timers'); +// Require buffer to make sure it's available in the sandbox +// Browserify statically analyses the usage of buffers and only +// injects Buffer into the scope if it's used. `Buffer` injected +// by browserify is part of the functional scope and does not get +// deleted when we mutate the global scope below. +require('buffer'); + // Although we execute the user code in a well-defined scope using the uniscope // module but still to cutoff the reference to the globally available properties // we sanitize the global scope by deleting the forbidden properties in this UVM diff --git a/lib/vendor/buffer/buffer.js b/lib/vendor/buffer/buffer.js new file mode 100644 index 00000000..41ea6da8 --- /dev/null +++ b/lib/vendor/buffer/buffer.js @@ -0,0 +1,10 @@ +function getBufferModule (buffer) { + return { + Buffer: buffer.Buffer, + SlowBuffer: buffer.SlowBuffer, + INSPECT_MAX_BYTES: buffer.INSPECT_MAX_BYTES, + kMaxLength: buffer.kMaxLength + } +} + +module.exports = getBufferModule; diff --git a/lib/vendor/buffer/index.browser.js b/lib/vendor/buffer/index.browser.js new file mode 100644 index 00000000..f07c967a --- /dev/null +++ b/lib/vendor/buffer/index.browser.js @@ -0,0 +1,8 @@ +const getBufferModule = require('./buffer'); +const SpecificBuffer = require('./specific-buffer'); +const buffer = require('buffer/'); + +module.exports = getBufferModule({ + ...buffer, + Buffer: SpecificBuffer(buffer.Buffer) +}) diff --git a/lib/vendor/buffer/index.js b/lib/vendor/buffer/index.js new file mode 100644 index 00000000..9accc807 --- /dev/null +++ b/lib/vendor/buffer/index.js @@ -0,0 +1,8 @@ +const getBufferModule = require('./buffer'); +const SpecificBuffer = require('./specific-buffer'); +const buffer = globalThis._nodeRequires.buffer; + +module.exports = getBufferModule({ + ...buffer, + Buffer: SpecificBuffer(globalThis.Buffer), +}); diff --git a/lib/vendor/buffer/specific-buffer.js b/lib/vendor/buffer/specific-buffer.js new file mode 100644 index 00000000..d2968066 --- /dev/null +++ b/lib/vendor/buffer/specific-buffer.js @@ -0,0 +1,57 @@ +function SpecificBuffer (_Buffer) { + const Buffer = function () { + if (typeof arguments[0] === 'number') { + return _Buffer.alloc(...arguments); + } + + return _Buffer.from(...arguments); + } + + Buffer.poolSize = _Buffer.poolSize; + + Object.defineProperty(Buffer, Symbol.hasInstance, { + value: function (instance) { + return instance instanceof _Buffer; + } + }); + + Buffer.isBuffer = function () { + return _Buffer.isBuffer(...arguments); + }; + + Buffer.alloc = function () { + return _Buffer.alloc(...arguments); + }; + + Buffer.allocUnsafe = function () { + return _Buffer.allocUnsafe(...arguments); + }; + + Buffer.allocUnsafeSlow = function () { + return _Buffer.allocUnsafeSlow(...arguments); + }; + + Buffer.from = function () { + return _Buffer.from(...arguments); + }; + + Buffer.compare = function () { + return _Buffer.compare(...arguments); + }; + + Buffer.isEncoding = function () { + return _Buffer.isEncoding(...arguments); + }; + + Buffer.concat = function () { + return _Buffer.concat(...arguments); + }; + + Buffer.byteLength = function () { + return _Buffer.byteLength(...arguments); + }; + + return Buffer; +} + +module.exports = SpecificBuffer; diff --git a/npm/cache.js b/npm/cache.js index c0ad9ffe..1e5b93d7 100755 --- a/npm/cache.js +++ b/npm/cache.js @@ -18,10 +18,7 @@ function createBundle (options, file, done) { }, function (codeString, next) { - // @note: we are appending "require=null;" here to avoid access to - // node's require function when running sandbox in worker_threads. - // This does not affect the require function injected by browserify. - fs.writeFile(file, `module.exports=c=>c(null,${JSON.stringify('require=null;' + codeString)})`, next); + fs.writeFile(file, `module.exports=c=>c(null,${JSON.stringify(codeString)})`, next); }, function (next) { @@ -48,10 +45,12 @@ module.exports = function (exit) { async.parallel([ async.apply(createBundle, _.merge({ compress: true, + preferBrowserResolver: false, bundler: { browserField: false } }, options), './.cache/bootcode.js'), async.apply(createBundle, _.merge({ compress: true, + preferBrowserResolver: true, bundler: { browserField: true } }, options), './.cache/bootcode.browser.js') ], function (err) { diff --git a/test/system/bootcode-dependencies.test.js b/test/system/bootcode-dependencies.test.js index abbc84a1..2c5b93d3 100644 --- a/test/system/bootcode-dependencies.test.js +++ b/test/system/bootcode-dependencies.test.js @@ -1,139 +1,268 @@ -describe('bootcode dependencies', function () { - this.timeout(60 * 1000); +const expect = require('chai').expect, + env = require('../../lib/environment'), + Bundle = require('../../lib/bundle'), - var expect = require('chai').expect, - env = require('../../lib/environment'), - Bundle = require('../../lib/bundle'), - currentDependencies; + expectedDependenciesBrowser = [ + '@faker-js/faker', + '@postman/tough-cookie', + 'ajv', + 'array-filter', + 'assert', + 'assertion-error', + 'atob', + 'available-typed-arrays', + 'backbone', + 'base64-js', + 'boolbase', + 'browserify', + 'btoa', + 'buffer', + 'call-bind', + 'chai', + 'chai-postman', + 'charset', + 'check-error', + 'cheerio', + 'core-util-is', + 'crypto-js', + 'css-select', + 'css-what', + 'csv-parse', + 'deep-eql', + 'define-properties', + 'dom-serializer', + 'domelementtype', + 'domhandler', + 'domutils', + 'entities', + 'es-abstract', + 'es6-object-assign', + 'events', + 'fast-deep-equal', + 'fast-json-stable-stringify', + 'file-type', + 'foreach', + 'function-bind', + 'get-func-name', + 'get-intrinsic', + 'has', + 'has-symbols', + 'htmlparser2', + 'http-reasons', + 'iconv-lite', + 'ieee754', + 'inherits', + 'is-arguments', + 'is-buffer', + 'is-generator-function', + 'is-nan', + 'is-typed-array', + 'isarray', + 'jquery', + 'json-schema-traverse', + 'liquid-json', + 'lodash', + 'lodash.assignin', + 'lodash.bind', + 'lodash.defaults', + 'lodash.filter', + 'lodash.flatten', + 'lodash.foreach', + 'lodash.map', + 'lodash.merge', + 'lodash.pick', + 'lodash.reduce', + 'lodash.reject', + 'lodash.some', + 'lodash3', + 'loupe', + 'mime-db', + 'mime-format', + 'mime-types', + 'moment', + 'nth-check', + 'object-is', + 'object-keys', + 'os-browserify', + 'path-browserify', + 'pathval', + 'postman-collection', + 'postman-sandbox', + 'postman-url-encoder', + 'process', + 'process-nextick-args', + 'psl', + 'punycode', + 'querystring-es3', + 'querystringify', + 'readable-stream', + 'requires-port', + 'safe-buffer', + 'safer-buffer', + 'sax', + 'semver', + 'stream-browserify', + 'string_decoder', + 'teleport-javascript', + 'timers-browserify', + 'tv4', + 'type-detect', + 'underscore', + 'uniscope', + 'universalify', + 'uri-js', + 'url', + 'url-parse', + 'util', + 'util-deprecate', + 'uuid', + 'which-typed-array', + 'xml2js', + 'xmlbuilder' + ], + expectedDependenciesNode = [ + '@faker-js/faker', + '@postman/tough-cookie', + 'ajv', + 'array-filter', + 'assert', + 'assertion-error', + 'atob', + 'available-typed-arrays', + 'backbone', + 'boolbase', + 'browserify', + 'btoa', + 'call-bind', + 'chai', + 'chai-postman', + 'charset', + 'check-error', + 'cheerio', + 'core-util-is', + 'crypto-js', + 'css-select', + 'css-what', + 'csv-parse', + 'deep-eql', + 'define-properties', + 'dom-serializer', + 'domelementtype', + 'domhandler', + 'domutils', + 'entities', + 'es-abstract', + 'es6-object-assign', + 'events', + 'fast-deep-equal', + 'fast-json-stable-stringify', + 'file-type', + 'foreach', + 'function-bind', + 'get-func-name', + 'get-intrinsic', + 'has', + 'has-symbols', + 'htmlparser2', + 'http-reasons', + 'iconv-lite', + 'inherits', + 'is-arguments', + 'is-buffer', + 'is-generator-function', + 'is-nan', + 'is-typed-array', + 'isarray', + 'jquery', + 'json-schema-traverse', + 'liquid-json', + 'lodash', + 'lodash.assignin', + 'lodash.bind', + 'lodash.defaults', + 'lodash.filter', + 'lodash.flatten', + 'lodash.foreach', + 'lodash.map', + 'lodash.merge', + 'lodash.pick', + 'lodash.reduce', + 'lodash.reject', + 'lodash.some', + 'lodash3', + 'loupe', + 'mime-db', + 'mime-format', + 'mime-types', + 'moment', + 'nth-check', + 'object-is', + 'object-keys', + 'os-browserify', + 'path-browserify', + 'pathval', + 'postman-collection', + 'postman-sandbox', + 'postman-url-encoder', + 'process', + 'process-nextick-args', + 'psl', + 'punycode', + 'querystring-es3', + 'querystringify', + 'readable-stream', + 'requires-port', + 'safe-buffer', + 'safer-buffer', + 'sax', + 'semver', + 'stream-browserify', + 'string_decoder', + 'teleport-javascript', + 'timers-browserify', + 'tv4', + 'type-detect', + 'underscore', + 'uniscope', + 'universalify', + 'uri-js', + 'url', + 'url-parse', + 'util', + 'util-deprecate', + 'uuid', + 'which-typed-array', + 'xml2js', + 'xmlbuilder' + ]; - before(function (done) { - process && process.stdout.write(' -- building dependencies, please wait... '); - Bundle.load(env).listDependencies(function (err, dependencies) { - currentDependencies = dependencies; +function getCurrentDependencies (isBrowser, cb) { + Bundle.load({ ...env, preferBrowserResolver: isBrowser }) + .listDependencies(function (err, dependencies) { console.info(err ? 'failed' : 'done'); - return done(err); + return cb(err, dependencies); + }); +} + +describe('bootcode dependencies', function () { + this.timeout(60 * 1000); + + it('should not change on Node', function (done) { + getCurrentDependencies(false, function (err, currentDependencies) { + if (err) { return done(err); } + + expect(currentDependencies).to.eql(expectedDependenciesNode); + done(); }); }); - it('should not change', function () { - expect(currentDependencies).to.eql([ - '@faker-js/faker', - '@postman/tough-cookie', - 'ajv', - 'array-filter', - 'assert', - 'assertion-error', - 'atob', - 'available-typed-arrays', - 'backbone', - 'base64-js', - 'boolbase', - 'browserify', - 'btoa', - 'buffer', - 'call-bind', - 'chai', - 'chai-postman', - 'charset', - 'check-error', - 'cheerio', - 'core-util-is', - 'crypto-js', - 'css-select', - 'css-what', - 'csv-parse', - 'deep-eql', - 'define-properties', - 'dom-serializer', - 'domelementtype', - 'domhandler', - 'domutils', - 'entities', - 'es-abstract', - 'es6-object-assign', - 'events', - 'fast-deep-equal', - 'fast-json-stable-stringify', - 'file-type', - 'foreach', - 'function-bind', - 'get-func-name', - 'get-intrinsic', - 'has', - 'has-symbols', - 'htmlparser2', - 'http-reasons', - 'iconv-lite', - 'ieee754', - 'inherits', - 'is-arguments', - 'is-buffer', - 'is-generator-function', - 'is-nan', - 'is-typed-array', - 'isarray', - 'jquery', - 'json-schema-traverse', - 'liquid-json', - 'lodash', - 'lodash.assignin', - 'lodash.bind', - 'lodash.defaults', - 'lodash.filter', - 'lodash.flatten', - 'lodash.foreach', - 'lodash.map', - 'lodash.merge', - 'lodash.pick', - 'lodash.reduce', - 'lodash.reject', - 'lodash.some', - 'lodash3', - 'loupe', - 'mime-db', - 'mime-format', - 'mime-types', - 'moment', - 'nth-check', - 'object-is', - 'object-keys', - 'os-browserify', - 'path-browserify', - 'pathval', - 'postman-collection', - 'postman-sandbox', - 'postman-url-encoder', - 'process', - 'process-nextick-args', - 'psl', - 'punycode', - 'querystring-es3', - 'querystringify', - 'readable-stream', - 'requires-port', - 'safe-buffer', - 'safer-buffer', - 'sax', - 'semver', - 'stream-browserify', - 'string_decoder', - 'teleport-javascript', - 'timers-browserify', - 'tv4', - 'type-detect', - 'underscore', - 'uniscope', - 'universalify', - 'uri-js', - 'url', - 'url-parse', - 'util', - 'util-deprecate', - 'uuid', - 'which-typed-array', - 'xml2js', - 'xmlbuilder' - ]); + it('should not change on Browser', function (done) { + getCurrentDependencies(true, function (err, currentDependencies) { + if (err) { return done(err); } + + expect(currentDependencies).to.eql(expectedDependenciesBrowser); + done(); + }); }); }); diff --git a/test/unit/sandbox-libraries/buffer.test.js b/test/unit/sandbox-libraries/buffer.test.js index 6ee0baa7..07599d47 100644 --- a/test/unit/sandbox-libraries/buffer.test.js +++ b/test/unit/sandbox-libraries/buffer.test.js @@ -114,12 +114,137 @@ describe('sandbox library - buffer', function () { context.execute(` var assert = require('assert'), buf1 = new Buffer('buffer'), - buf2 = new Buffer(buf1); + buf2 = new Buffer(buf1), + buf3 = Buffer(1); buf1[0] = 0x61; + buf3[0] = 0x61; assert.strictEqual(buf1.toString(), 'auffer'); assert.strictEqual(buf2.toString(), 'buffer'); + assert.strictEqual(buf3.toString(), 'a'); `, done); }); + + it('should be able to detect Buffer instances using isBuffer', function (done) { + context.execute(` + const assert = require('assert'), + + bufUsingFrom = Buffer.from('test'), + bufUsingNew = new Buffer('test'), + buf = Buffer(1); + + assert.strictEqual(Buffer.isBuffer(bufUsingFrom), true); + assert.strictEqual(Buffer.isBuffer(bufUsingNew), true); + assert.strictEqual(Buffer.isBuffer(buf), true); + `, done); + }); + + it('should be able to detect Buffer instances using Symbol.hasInstance', function (done) { + context.execute(` + const assert = require('assert'), + + bufUsingFrom = Buffer.from('test'), + bufUsingNew = new Buffer('test'); + buf = Buffer(1); + + assert.strictEqual(bufUsingFrom instanceof Buffer, true); + assert.strictEqual(bufUsingNew instanceof Buffer, true); + assert.strictEqual(buf instanceof Buffer, true); + `, done); + }); + + it('should be able to convert large buffer to string', function (done) { + // For native buffer, the max string length is ~512MB + // For browser buffer, the max string length is ~100MB + const SIZE = (typeof window === 'undefined' ? 511 : 100) * 1024 * 1024; + + context.execute(` + const assert = require('assert'), + buf = Buffer.alloc(${SIZE}, 'a'); + + assert.strictEqual(buf.toString().length, ${SIZE}); + `, done); + }); + + it('should implement Buffer.compare', function (done) { + context.execute(` + const assert = require('assert'), + + buf1 = Buffer.from('abc'), + buf2 = Buffer.from('abc'), + buf3 = Buffer.from('abd'); + + assert.strictEqual(Buffer.compare(buf1, buf2), 0); + assert.strictEqual(Buffer.compare(buf1, buf3), -1); + assert.strictEqual(Buffer.compare(buf3, buf1), 1); + `, done); + }); + + it('should implement Buffer.byteLength', function (done) { + context.execute(` + const assert = require('assert'), + buf = Buffer.from('abc'); + + assert.strictEqual(Buffer.byteLength(buf), 3); + `, done); + }); + + it('should implement Buffer.concat', function (done) { + context.execute(` + const assert = require('assert'), + + buf1 = Buffer.from('abc'), + buf2 = Buffer.from('def'); + + assert.strictEqual(Buffer.concat([buf1, buf2]).toString(), 'abcdef'); + `, done); + }); + + it('should implement Buffer.isEncoding', function (done) { + context.execute(` + const assert = require('assert'); + + assert.strictEqual(Buffer.isEncoding('utf8'), true); + assert.strictEqual(Buffer.isEncoding('hex'), true); + assert.strictEqual(Buffer.isEncoding('ascii'), true); + assert.strictEqual(Buffer.isEncoding('utf16le'), true); + assert.strictEqual(Buffer.isEncoding('ucs2'), true); + assert.strictEqual(Buffer.isEncoding('base64'), true); + assert.strictEqual(Buffer.isEncoding('binary'), true); + + assert.strictEqual(Buffer.isEncoding('utf-8'), true); + assert.strictEqual(Buffer.isEncoding('utf/8'), false); + assert.strictEqual(Buffer.isEncoding(''), false); + `, done); + }); + + it('should expose Buffer.poolSize', function (done) { + context.execute(` + const assert = require('assert'); + + assert.strictEqual(typeof Buffer.poolSize, 'number'); + `, done); + }); + + it('should expose SlowBuffer', function (done) { + context.execute(` + const assert = require('assert'), + buffer = require('buffer'); + + const buf = new buffer.SlowBuffer(10); + assert.strictEqual(buf.length, 10); + `, done); + }); + + it('should expose constants', function (done) { + context.execute(` + const assert = require('assert'), + buffer = require('buffer'); + + assert.strictEqual(typeof buffer.kMaxLength, 'number'); + assert.strictEqual(typeof buffer.INSPECT_MAX_BYTES, 'number'); + + `, done); + }); });