diff --git a/test/ExampleWorker.js b/test/ExampleWorker.js index 0e39f08f..30c9ea1a 100644 --- a/test/ExampleWorker.js +++ b/test/ExampleWorker.js @@ -27,9 +27,9 @@ const runExample = options => { let render; let extrinsicCapture; - const bodyOverlap = (bodyA, bodyB) => { - const collision = Matter.Collision.collides(bodyA, bodyB); - return collision ? collision.depth : 0; + const pairOverlap = (pair) => { + const collision = Matter.Collision.collides(pair.bodyA, pair.bodyB); + return collision ? Math.max(collision.depth - pair.slop, 0) : -1; }; for (i = 0; i < options.repeats; i += 1) { @@ -66,20 +66,30 @@ const runExample = options => { timeDeltaAverage = smoothExp(timeDeltaAverage, timeDelta); } - if (j === 1) { - const pairsList = engine.pairs.list; - const pairsListLength = engine.pairs.list.length; + let overlapTotalUpdate = 0; + let overlapCountUpdate = 0; - for (let p = 0; p < pairsListLength; p += 1) { - const pair = pairsList[p]; + const pairsList = engine.pairs.list; + const pairsListLength = engine.pairs.list.length; - if (pair.isActive && !pair.isSensor) { - overlapTotal += bodyOverlap(pair.bodyA, pair.bodyB); - overlapCount += 1; + for (let p = 0; p < pairsListLength; p += 1) { + const pair = pairsList[p]; + + if (pair.isActive && !pair.isSensor){ + const overlap = pairOverlap(pair); + + if (overlap >= 0) { + overlapTotalUpdate += overlap; + overlapCountUpdate += 1; } } } + if (overlapCountUpdate > 0) { + overlapTotal += overlapTotalUpdate / overlapCountUpdate; + overlapCount += 1; + } + if (!extrinsicCapture && engine.timing.timestamp >= 1000) { extrinsicCapture = captureExtrinsics(engine, Matter); extrinsicCapture.updates = j; diff --git a/test/TestTools.js b/test/TestTools.js index 183d93dd..6e7108bc 100644 --- a/test/TestTools.js +++ b/test/TestTools.js @@ -12,23 +12,22 @@ const equalityThreshold = 1; const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, BrightCyan: 36 }; const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save, benchmark) => { - const performanceDev = capturePerformanceTotals(capturesDev); - const performanceBuild = capturePerformanceTotals(capturesBuild); + const { + durationChange, + memoryChange, + overlapChange + } = captureBenchmark(capturesDev, capturesBuild); - const perfChange = noiseThreshold(1 - (performanceDev.duration / performanceBuild.duration), 0.01); - const memoryChange = noiseThreshold((performanceDev.memory / performanceBuild.memory) - 1, 0.01); - const overlapChange = (performanceDev.overlap / (performanceBuild.overlap || 1)) - 1; const filesizeChange = (devSize / buildSize) - 1; - const similaritys = extrinsicSimilarity(capturesDev, capturesBuild); - const similarityAverage = extrinsicSimilarityAverage(similaritys); - const similarityEntries = Object.entries(similaritys); - similarityEntries.sort((a, b) => a[1] - b[1]); - const firstCapture = Object.entries(capturesDev)[0][1]; const updates = firstCapture.extrinsic.updates; + const similaritys = extrinsicSimilarity(capturesDev, capturesBuild); + const similarityAverage = extrinsicSimilarityAverage(similaritys); const similarityAveragePerUpdate = Math.pow(1, -1 / updates) * Math.pow(similarityAverage, 1 / updates); + const similarityEntries = Object.entries(similaritys); + similarityEntries.sort((a, b) => a[1] - b[1]); const devIntrinsicsChanged = {}; const buildIntrinsicsChanged = {}; @@ -51,8 +50,6 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV }) .sort((a, b) => a.name.localeCompare(b.name)); - const formatColor = green => green ? colors.Green : colors.Yellow; - const report = (breakEvery, format) => [ [`Output sample comparison estimates of ${similarityEntries.length} examples`, `against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}:` @@ -70,7 +67,7 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV ...(benchmark ? [ `\n${format('Performance', colors.White)}`, - ` ${format(formatPercent(perfChange), formatColor(perfChange >= 0))}%`, + ` ${format(formatPercent(durationChange), formatColor(durationChange >= 0))}%`, ` ${format('Memory', colors.White)} `, ` ${format(formatPercent(memoryChange), formatColor(memoryChange <= 0))}%`, @@ -102,9 +99,13 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV }; const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·'; + const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·'; + const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text; +const formatColor = isGreen => isGreen ? colors.Green : colors.Yellow; + const formatPercent = (val, showSign=true, showFractional=false, padStart=6) => { let fractionalSign = ''; @@ -127,10 +128,35 @@ const noiseThreshold = (val, threshold) => { return sign * Math.max(0, magnitude - threshold) / (1 - threshold); }; +const median = (values, lower, upper) => { + const valuesSorted = values.slice(0).sort(); + + return mean(valuesSorted.slice( + Math.floor(valuesSorted.length * lower), + Math.floor(valuesSorted.length * upper) + )); +}; + +const mean = (values) => { + const valuesLength = values.length; + let result = 0; + + for (let i = 0; i < valuesLength; i += 1) { + result += values[i]; + } + + return (result / valuesLength) || 0; +}; + const smoothExp = (last, current) => { const delta = current - last; const sign = delta < 0 ? -1 : 1; const magnitude = Math.abs(delta); + + if (magnitude < 1) { + return last + 0.01 * delta; + } + return last + Math.sqrt(magnitude) * sign; }; @@ -143,20 +169,35 @@ const equals = (a, b) => { return true; }; -const capturePerformanceTotals = (captures) => { - const totals = { - duration: 0, - overlap: 0, - memory: 0 - }; +const captureBenchmark = (capturesDev, capturesBuild) => { + const overlapChanges = []; + + let durationDev = 0; + let durationBuild = 0; + let memoryDev = 0; + let memoryBuild = 0; + + for (const name in capturesDev) { + durationDev += capturesDev[name].duration; + durationBuild += capturesBuild[name].duration; + + memoryDev += capturesDev[name].memory; + memoryBuild += capturesBuild[name].memory; - for (const [ name ] of Object.entries(captures)) { - totals.duration += captures[name].duration; - totals.overlap += captures[name].overlap; - totals.memory += captures[name].memory; + if (capturesBuild[name].overlap > 0.1 && capturesDev[name].overlap > 0.1){ + overlapChanges.push(capturesDev[name].overlap / capturesBuild[name].overlap); + } }; - return totals; + const durationChange = 1 - noiseThreshold(durationDev / durationBuild, 0.02); + const memoryChange = noiseThreshold(memoryDev / memoryBuild, 0.02) - 1; + const overlapChange = noiseThreshold(median(overlapChanges, 0.45, 0.55), 0.001) - 1; + + return { + durationChange, + memoryChange, + overlapChange + }; }; const extrinsicSimilarity = (currentCaptures, referenceCaptures, key='extrinsic') => {