diff --git a/.eslintrc.json b/.eslintrc.json index 37ed09aaa..596ebd5bb 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -127,7 +127,7 @@ } }, { - "files": ["test/es2018/**"], + "files": ["test/benchmark/**", "test/es2018/**"], "env": { "es2017": true }, diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 087250c21..d9df1fffe 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -92,6 +92,7 @@ jobs: # ``` # root@ubuntu-tmp/qunit$ apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y libmozjs-68-dev # root@ubuntu-tmp/qunit$ js68 test/mozjs.js + # root@ubuntu-tmp/qunit$ js68 test/benchmark/index-mozjs.js # ``` sm-test: name: SpiderMonkey diff --git a/.gitignore b/.gitignore index 2eac8aecb..82942b8be 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ /docs/.jekyll-cache/ /docs/_site/ /docs/Gemfile.lock +/test/benchmark/package-lock.json +/test/benchmark/node_modules/ /test/integration/*/package-lock.json /test/integration/*/node_modules/ /node_modules/ diff --git a/package-lock.json b/package-lock.json index ecc3a18a0..cb3a2bf48 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "@rollup/plugin-commonjs": "^20.0.0", "@rollup/plugin-node-resolve": "^13.0.4", "@rollup/plugin-replace": "^3.0.0", + "benchmark": "2.1.4", "eslint": "7.32.0", "eslint-config-semistandard": "^16.0.0", "eslint-config-standard": "^16.0.3", @@ -2528,6 +2529,16 @@ "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", "dev": true }, + "node_modules/benchmark": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/benchmark/-/benchmark-2.1.4.tgz", + "integrity": "sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ==", + "dev": true, + "dependencies": { + "lodash": "^4.17.4", + "platform": "^1.3.3" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -7032,6 +7043,12 @@ "node": ">=8" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "dev": true + }, "node_modules/portscanner": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/portscanner/-/portscanner-2.2.0.tgz", @@ -10527,6 +10544,16 @@ "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", "dev": true }, + "benchmark": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/benchmark/-/benchmark-2.1.4.tgz", + "integrity": "sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ==", + "dev": true, + "requires": { + "lodash": "^4.17.4", + "platform": "^1.3.3" + } + }, "bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -13879,6 +13906,12 @@ } } }, + "platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "dev": true + }, "portscanner": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/portscanner/-/portscanner-2.2.0.tgz", diff --git a/package.json b/package.json index db0659e7c..2c4603450 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "@rollup/plugin-commonjs": "^20.0.0", "@rollup/plugin-node-resolve": "^13.0.4", "@rollup/plugin-replace": "^3.0.0", + "benchmark": "2.1.4", "eslint": "7.32.0", "eslint-config-semistandard": "^16.0.0", "eslint-config-standard": "^16.0.3", diff --git a/test/benchmark/README.md b/test/benchmark/README.md new file mode 100644 index 000000000..5658f4be8 --- /dev/null +++ b/test/benchmark/README.md @@ -0,0 +1,26 @@ +# Benchmark for QUnit internals + +## Usage + +The default is to benchmark the local development version of QUnit. + +1. Install QUnit for development and generate the release artefact: + * `qunit$ npm ci` + * `qunit$ npm run build` +2. Link benchmark to local artefact. + NOTE: Alternatively, you can edit benchmark/package.json + and change `file:../..` to something like `2.19.1` to + benchmark older versions of QUnit. + * `qunit/test/benchmark$ npm install` +3. Run the benchmark + * In Node.js: + `qunit/test/benchmark$ node index-node.js` + * In a browser: + * Start a static web server, e.g. using Python + `qunit$ python3 -m http.server 4000` + or PHP: + `php -S localhost:4000` + * Open + * Check the console output. + +Powered by [Benchmark.js](https://benchmarkjs.com/). diff --git a/test/benchmark/bench.js b/test/benchmark/bench.js new file mode 100644 index 000000000..308e2967c --- /dev/null +++ b/test/benchmark/bench.js @@ -0,0 +1,41 @@ +/* global require, globalThis, QUnitFixtureEquiv, QUnit, console */ + +const Benchmark = typeof require === 'function' ? require('benchmark') : globalThis.Benchmark; +const suite = new Benchmark.Suite(); + +// Check for correctness first, mainly for the return value, +// but also for any unexpected exceptions as Benchmark will tolerate +// uncaught exceptions as being benchmarkable behaviour. +for (const group of QUnitFixtureEquiv) { + group.pairs.forEach((pair, i) => { + const res = QUnit.equiv(pair.a, pair.b); + if (res !== pair.equal) { + throw new Error(`Unexpected return value in "${group.name}" at pairs[${i}]\n Expected: ${pair.equal}\n Actual: ${res}`); + } + }); +} + +suite.add('equiv', function () { + for (const group of QUnitFixtureEquiv) { + for (const pair of group.pairs) { + QUnit.equiv(pair.a, pair.b); + } + } +}); + +for (const group of QUnitFixtureEquiv) { + suite.add(`equiv (${group.name})`, function () { + for (const pair of group.pairs) { + QUnit.equiv(pair.a, pair.b); + } + }); +} + +console.log('Running benchmark...'); +suite.on('cycle', function (event) { + console.log(String(event.target)); +}); +suite.on('complete', function () { + console.log('Done!'); +}); +suite.run({ async: true }); diff --git a/test/benchmark/fixture-equiv.js b/test/benchmark/fixture-equiv.js new file mode 100644 index 000000000..daa27efe2 --- /dev/null +++ b/test/benchmark/fixture-equiv.js @@ -0,0 +1,522 @@ +function createSet (values) { + const x = new Set(); + for (const value of values) { + x.add(value); + } + return x; +} + +function createEntries (list) { + const entries = []; + let i = 0; + while (i + 1 < list.length) { + entries.push([ + list[i], + list[i + 1] + ]); + i += 2; + } + return entries; +} + +function createObject (list) { + return Object.fromEntries(createEntries(list)); +} + +function createObjectWithValue (list, value) { + const x = {}; + for (const key of list) { + x[key] = value; + } + return x; +} + +function createMap (list) { + return new Map(createEntries(list)); +} + +function createArrayDeep (list) { + const top = []; + let current = top; + let i = 0; + while (i < list.length) { + current.push(list[i]); + const next = []; + current.push(next); + current = next; + i++; + } + + return top; +} + +function createObjectDeep (list) { + const top = {}; + let current = top; + let keys = list.slice(); + while (keys.length >= 3) { + current[keys.pop()] = 1; + current[keys.pop()] = true; + const next = {}; + current[keys.pop()] = next; + current = next; + } + + return top; +} + +function X (val) { + this.val = val; +} +X.prototype.getVal = function () { + return this.val; +}; +function Y (val) { + this.val = val; +} +Y.prototype.getVal = function () { + return this.val; +}; +function Point (x, y) { + this.x = x; + this.y = y; +} +function Line (points) { + this.setPoints(points); +} +Line.prototype.setPoints = function (points) { + this.head = points[0]; + this.points = points; +}; +function NamedLine (title, points) { + Line.call(this, points); + this.setTitle(title); +} +NamedLine.prototype = Object.create(Line.prototype); +NamedLine.prototype.constructor = NamedLine; +NamedLine.prototype.setTitle = function (title) { + this.title = title; +}; + +function createClassTree (list) { + const points = []; + let i = 0; + while (i + 1 < list.length) { + points.push(new Point( + new X(list[i].length), + new Y(list[i + 1].length) + )); + i += 2; + } + + return new NamedLine(list[0], points); +} + +const listEdible = [ + 'Allahabadi Surkha', + 'Annona longiflora', + 'Annona nutans', + 'Annona paludosa', + 'Annonaceae', + 'Apricot', + 'Arbutus unedo', + 'Aristotelia serrata', + 'Asimina parviflora', + 'Asimina triloba', + 'Atriplex semibaccata', + 'Averrhoa bilimbi', + 'Banana', + 'Banana melon', + 'Banana passionfruit', + 'Breadfruit', + 'Burchellia', + 'Bush tomato', + 'Cherimoya', + 'Chrysophyllum cainito', + 'Citrus australasica', + 'Citrus australis', + 'Clausena lansium', + 'Clymenia (plant)', + 'Cornus canadensis', + 'Cornus mas', + 'Couepia polyandra', + 'Crataegus crus-galli', + 'Crataegus phaenopyrum', + 'Crataegus succulenta', + 'Crataegus tanacetifolia', + 'Cucumis prophetarum', + 'Date-plum', + 'Diospyros blancoi', + 'Eugenia calycina', + 'Eugenia pyriformis', + 'Eugenia reinwardtiana', + 'Ficus carica', + 'Ficus pumila', + 'Fig', + 'Fruit hat', + 'Gac', + 'Garcinia assamica', + 'Garcinia cowa', + 'Garcinia forbesii', + 'Garcinia humilis', + 'Garcinia lanceifolia', + 'Garcinia magnifolia', + 'Garcinia pseudoguttifera', + 'Gaya melon', + 'Glycosmis parviflora', + 'Glycosmis pentaphylla', + 'Grape', + 'Guava', + 'Haruka (citrus)', + 'Honeydew (melon)', + 'Irvingia', + 'Kajari melon', + 'Kanpei', + 'Kawachi Bankan', + 'Kinkoji unshiu', + 'Kiwifruit', + 'Kobayashi mikan', + 'Koji orange', + 'Kolkhoznitsa melon', + 'Kuchinotsu No. 37', + 'Leycesteria formosa', + 'Licania platypus', + 'Litsea garciae', + 'Longan', + 'Lonicera caerulea', + 'Lucuma', + 'Lychee', + 'Mahonia fremontii', + 'Mango', + 'Mangosteen', + 'Mayhaw', + 'Mespilus germanica', + 'Mirza melon', + 'Morinda citrifolia', + 'Morus nigra', + 'Nectaplum', + 'Nephelium chryseum', + 'Nephelium xerospermoides', + 'Pandanus tectorius', + 'Passiflora', + 'Passiflora ligularis', + 'Passiflora tarminiana', + 'Persimmon', + 'Physalis pubescens', + 'Pineapple', + 'Pitaya', + 'Pomegranate', + 'Pompia', + 'Produce', + 'Prunus americana', + 'Psidium cattleyanum', + 'Pulasan', + 'Quince', + 'Rambutan', + 'Rhus typhina', + 'Ribes americanum', + 'Ribes montigenum', + 'Rubus spectabilis', + 'Rubus tricolor', + 'Rubus ulmifolius', + 'Sambucus nigra', + 'Sandoricum koetjape', + 'Seedless fruit', + 'Sonneratia caseolaris', + 'Sorbus americana', + 'Stone fruits', + 'Sugar-apple', + 'Sunflower seed', + 'Sycamine', + 'Tamarillo', + 'Viburnum edule', + 'Volkamer lemon', + 'Watermelon', + 'White sapote', + 'Xocotl' +]; + +const listDishes = [ + 'Açaí na tigela', + 'Ashure', + 'Asinan', + 'Avakaya', + 'Banbury cake', + 'Black bun', + 'Boerenjongens', + 'Bombe glacée', + 'Candle salad', + 'Chakka prathaman', + 'Chakkavaratti', + 'Charlotte (cake)', + 'Chorley cake', + 'Clafoutis', + 'Cobbler (food)', + 'Compote', + 'Cranachan', + 'Crema de fruta', + 'Crisp (dessert)', + 'Crumble', + 'Cumberland rum nicky', + 'Daechu-gom', + 'Dolma', + 'Donauwelle', + 'Duff (dessert)', + 'Eccles cake', + 'Es buah', + 'Es campur', + 'Es teler', + 'Eton mess', + 'Flaugnarde', + 'Flies graveyard', + 'Frogeye salad', + 'Fruit butter', + 'Fruit curd', + 'Fruit fool', + 'Fruit hat (pudding)', + 'Fruit ketchup', + 'Fruit salad', + 'Fruit whip', + 'Fruitcake', + 'Grape syrup', + 'Güllaç', + 'Gwapyeon', + 'Hagebuttenmark', + 'Hakuto jelly', + 'Hwachae', + 'Kirschenmichel', + 'Kompot', + 'Krentjebrij', + 'Lörtsy', + 'Malvern pudding', + 'Manchester tart', + 'Mango cake', + 'Mango float', + 'Mango pudding', + 'Marillenknödel', + 'Mincemeat', + 'Multekrem', + 'Murabba', + 'Peach Melba', + 'Peaches and cream', + 'Persimmon pudding', + 'Pestil', + 'Poi', + 'Pork chops and applesauce', + 'Qubani-ka-Meetha', + 'Relish', + 'Rødgrød', + 'Rojak', + 'Rose hip soup', + 'Rumtopf', + 'Seafoam salad', + 'Spotted dick', + 'Summer pudding', + 'Takihi', + 'Tanghulu', + 'Tarte des Alpes', + 'Tilslørte bondepiker', + 'Tomato jam', + 'Tutti frutti', + 'Vispipuuro', + 'Xi gua lao' +]; + +// eslint-disable-next-line no-undef +globalThis.QUnitFixtureEquiv = [ + { + name: 'primitives', + pairs: listEdible.concat(listDishes).map(value => { + return { + a: value, + b: value, + equal: true + }; + }) + }, + { + name: 'small arrays', + pairs: [ + ...listEdible.slice(0, 5).map(value => { + return { + a: [value], + b: [value], + equal: true + }; + }), + { + a: listEdible.slice(0, 10), + b: listEdible.slice(0, 10), + equal: true + }, + { + a: listDishes.slice(0, 10), + b: listDishes.slice(0, 10), + equal: true + } + ] + }, + { + name: 'deep arrays', + pairs: [ + { + a: createArrayDeep(listEdible), + b: createArrayDeep(listEdible), + equal: true + } + ] + }, + { + name: 'arrays', + pairs: [ + { + a: listEdible, + b: listEdible.slice(), + equal: true + }, + { + a: listEdible.concat(listDishes), + b: listEdible.concat(listDishes), + equal: true + }, + { + a: listEdible.concat(listDishes).concat(['foo']), + b: listEdible.concat(listDishes).concat(['bar']), + equal: false + } + ] + }, + { + name: 'small objects', + pairs: [ + ...listEdible.slice(0, 5).map(value => { + const a = {}; + const b = {}; + a[value] = 1; + b[value] = 1; + return { + a, + b, + equal: true + }; + }) + ] + }, + { + name: 'deep objects', + pairs: [ + { + a: createObjectDeep(listEdible), + b: createObjectDeep(listEdible), + equal: true + } + ] + }, + { + name: 'objects', + pairs: [ + { + a: createObjectWithValue(listEdible, true), + b: createObjectWithValue(listEdible, true), + equal: true + }, + { + a: createObjectWithValue(listDishes, 1), + b: createObjectWithValue(listDishes, 1), + equal: true + }, + { + a: createObject(listDishes), + b: createObject(listDishes.slice()), + equal: true + }, + { + a: createObject(listEdible), + b: createObject(listEdible.slice(0, -5)), + equal: false + } + ] + }, + { + name: 'small sets', + pairs: [ + ...listEdible.slice(0, 5).map(value => { + return { + a: createSet([value]), + b: createSet([value]), + equal: true + }; + }), + { + a: createSet(listEdible.slice(0, 10)), + b: createSet(listEdible.slice(0, 10).reverse()), + equal: true + }, + { + a: createSet(listDishes.slice(0, 10)), + b: createSet(listDishes.slice(0, 10).reverse()), + equal: true + } + ] + }, + { + name: 'sets', + pairs: [ + { + a: createSet(listEdible), + b: createSet(listEdible.slice().reverse()), + equal: true + }, + { + a: createSet(listEdible), + b: createSet(listEdible.slice(0, -1)), + equal: false + }, + { + a: createSet(listEdible.concat(listDishes).sort()), + b: createSet(listEdible.concat(listDishes).sort().reverse()), + equal: true + }, + { + a: createSet(listEdible.concat(listDishes).sort()), + b: createSet(listDishes.slice(0, -1).concat(listEdible.slice(0, -1)).sort()), + equal: false + }, + { + a: listEdible.concat(listDishes).concat(['foo']), + b: listDishes.concat(listEdible).concat(['bar']), + equal: false + } + ] + }, + { + name: 'maps', + pairs: [ + { + a: createMap(listEdible), + b: createMap(listEdible.slice()), + equal: true + }, + { + a: createMap(listDishes), + b: createMap(listDishes.slice(0, -5)), + equal: false + } + ] + }, + { + name: 'class trees', + pairs: [ + { + a: createClassTree(listEdible), + b: createClassTree(listEdible.slice()), + equal: true + }, + { + a: createClassTree(listEdible), + b: createClassTree(listEdible.slice(0, -5)), + equal: false + } + ] + } +]; diff --git a/test/benchmark/index-browser.html b/test/benchmark/index-browser.html new file mode 100644 index 000000000..fc2a8d9ee --- /dev/null +++ b/test/benchmark/index-browser.html @@ -0,0 +1,14 @@ + + + + + Benchmark + + + + + + + + + diff --git a/test/benchmark/index-mozjs.js b/test/benchmark/index-mozjs.js new file mode 100644 index 000000000..04e676293 --- /dev/null +++ b/test/benchmark/index-mozjs.js @@ -0,0 +1,7 @@ +/* global loadRelativeToScript */ + +loadRelativeToScript('../../node_modules/lodash/lodash.js'); +loadRelativeToScript('../../node_modules/benchmark/benchmark.js'); +loadRelativeToScript('./node_modules/qunit/qunit/qunit.js'); +loadRelativeToScript('./fixture-equiv.js'); +loadRelativeToScript('./bench.js'); diff --git a/test/benchmark/index-node.js b/test/benchmark/index-node.js new file mode 100644 index 000000000..66db6550b --- /dev/null +++ b/test/benchmark/index-node.js @@ -0,0 +1,5 @@ +/* eslint-env node */ + +global.QUnit = require('qunit'); +require('./fixture-equiv.js'); +require('./bench.js'); diff --git a/test/benchmark/package.json b/test/benchmark/package.json new file mode 100644 index 000000000..145fe4ce0 --- /dev/null +++ b/test/benchmark/package.json @@ -0,0 +1,6 @@ +{ + "private": true, + "devDependencies": { + "qunit": "file:../.." + } +} diff --git a/test/cli/structure.js b/test/cli/structure.js index 73d272162..ce05d564f 100644 --- a/test/cli/structure.js +++ b/test/cli/structure.js @@ -61,8 +61,9 @@ QUnit.module('structure', () => { // The expected HTML paths use Unix-style line ending, as per HTTP. .map(file => file.replace(/\\/g, '/')) // Ignore file names containing "--", which are subresources (e.g. iframes). - // Ignore integration/grunt-contrib-qunit, which is managed separately. - .filter(file => !file.includes('--') && !file.includes('integration')) + // Ignore test/benchmark, which is unrelated. + // Ignore test/integration/grunt-contrib-qunit, which we manage separately. + .filter(file => !file.includes('--') && !file.includes('benchmark') && !file.includes('integration')) .map(file => `test/${file}`); QUnit.test('files', assert => {