diff --git a/resources/style.css b/resources/style.css index 3759b5ea..c68f519a 100644 --- a/resources/style.css +++ b/resources/style.css @@ -228,6 +228,13 @@ table.benchmark-details .btn-sm { padding: 0 !important; } +.benchmark-details .alert { + display: inline-block; + padding: 0rem 0.2rem; + margin: 0.1rem 0.5rem; + line-height: 1rem; +} + .warmup-benchmark { padding-right: 1.5ex; font-weight: bold; diff --git a/src/backend/compare/html/index.html b/src/backend/compare/html/index.html index d5a366db..4ec9c475 100644 --- a/src/backend/compare/html/index.html +++ b/src/backend/compare/html/index.html @@ -11,6 +11,11 @@ + + + + +
diff --git a/src/backend/compare/html/stats-row.html b/src/backend/compare/html/stats-row.html index d34207e8..8687cef0 100644 --- a/src/backend/compare/html/stats-row.html +++ b/src/backend/compare/html/stats-row.html @@ -39,8 +39,16 @@ if (hasTotal) { + let inconsistent = ''; + if (stats.inconsistentRunIds) { + inconsistent = ``; + } %} -{%= stats.benchId.b %}{%- args %} +{%= stats.benchId.b %}{%- args %}{%- inconsistent %} {% if (stats.exeStats) { %}{%- include('stats-row-across-exes.html', { diff --git a/src/backend/compare/prep-data.ts b/src/backend/compare/prep-data.ts index 85f7e324..9c285744 100644 --- a/src/backend/compare/prep-data.ts +++ b/src/backend/compare/prep-data.ts @@ -69,7 +69,12 @@ export function compareToSortForSinglePassChangeStats( a: Measurements, b: Measurements ): number { - const r = compareToSortForSinglePassChangeStatsWithoutCommitId(a, b); + let r = compareToSortForSinglePassChangeStatsNonStrict(a, b); + if (r !== 0) { + return r; + } + + r = a.runId - b.runId; if (r !== 0) { return r; } @@ -77,16 +82,11 @@ export function compareToSortForSinglePassChangeStats( return a.commitId.localeCompare(b.commitId); } -export function compareToSortForSinglePassChangeStatsWithoutCommitId( +export function compareToSortForSinglePassChangeStatsNonStrict( a: Measurements, b: Measurements ): number { - let r = a.runId - b.runId; - if (r !== 0) { - return r; - } - - r = a.envId - b.envId; + let r = a.envId - b.envId; if (r !== 0) { return r; } @@ -374,9 +374,7 @@ export function countVariantsAndDropMissing( const change = measurements[i + 1]; - if ( - compareToSortForSinglePassChangeStatsWithoutCommitId(base, change) !== 0 - ) { + if (compareToSortForSinglePassChangeStatsNonStrict(base, change) !== 0) { dropAsMissing(i); } else { i += 2; @@ -499,6 +497,9 @@ async function computeStatisticsAndInlinePlot( // add the various details row.details.hasWarmup = siteConfig.canShowWarmup(change.values); + if (row.inconsistentRunIds === undefined || row.inconsistentRunIds) { + row.inconsistentRunIds = base.runId !== change.runId; + } if (!row.versionStats) { row.versionStats = {}; diff --git a/src/shared/view-types.ts b/src/shared/view-types.ts index 1a1f970b..c9966ddb 100644 --- a/src/shared/view-types.ts +++ b/src/shared/view-types.ts @@ -113,6 +113,8 @@ export interface CompareStatsRow { inlinePlot?: string; + inconsistentRunIds?: boolean; + /** Statistics per criterion, comparing base and change. */ versionStats?: CompareStatsRowAcrossVersions; exeStats?: CompareStatsRowAcrossExes[]; diff --git a/tests/backend/compare/prep-data.test.ts b/tests/backend/compare/prep-data.test.ts index 6d9dcfdc..cd50b555 100644 --- a/tests/backend/compare/prep-data.test.ts +++ b/tests/backend/compare/prep-data.test.ts @@ -82,14 +82,20 @@ const runSettings: RunSettings = { simplifiedCmdline: 'Exec TestBenchmark1' }; -function makeM(criterion, unit, envId, commitId) { +function makeM( + criterion: string, + unit: string, + envId: number, + commitId: string, + runId = 1 +) { return { criterion: { name: criterion, unit }, values: [[]], envId, commitId, runSettings, - runId: 1, + runId, trialId: 1, expId: 1 }; @@ -210,6 +216,31 @@ describe('compareToSortForSinglePassChangeStats()', () => { expect(data[5].envId).toBe(2); expect(data[5].criterion.name).toBe('total'); }); + + it( + 'should place same criteria next to each other, ' + + 'even when runId differs', + () => { + const data: Measurements[] = [ + makeM('total', 'ms', 1, 'a', 1), + makeM('alloc', 'by', 1, 'b', 2), + makeM('total', 'ms', 1, 'b', 2), + makeM('alloc', 'by', 1, 'a', 1) + ]; + + data.sort(compareToSortForSinglePassChangeStats); + + expect(data[0].commitId).toBe('a'); + expect(data[0].criterion.name).toBe('alloc'); + expect(data[1].commitId).toBe('b'); + expect(data[1].criterion.name).toBe('alloc'); + + expect(data[2].commitId).toBe('a'); + expect(data[2].criterion.name).toBe('total'); + expect(data[3].commitId).toBe('b'); + expect(data[3].criterion.name).toBe('total'); + } + ); }); describe('countVariantsAndDropMissing()', () => { @@ -318,19 +349,17 @@ describe('countVariantsAndDropMissing()', () => { expect(mc2[0].criterion.name).toEqual('total'); }); - it('should consider different runIds as incompatible', () => { + it('should not drop different runIds', () => { const data: Measurements[] = [ - makeM('total', 'ms', 1, 'a'), - makeM('total', 'ms', 1, 'a') + makeM('total', 'ms', 1, 'a', 1), + makeM('total', 'ms', 1, 'a', 2) ]; - data[0].runId = 2; const result = countVariantsAndDropMissing(makeProRes(data), 'a', 'b'); - expect(data).toHaveLength(0); + expect(data).toHaveLength(2); expect(result.missing).toBeDefined(); - expect(result.missing.size).toEqual(1); - expect([...result.missing.values()][0].missing).toHaveLength(2); + expect(result.missing.size).toEqual(0); }); }); diff --git a/tests/data/expected-results/stats-data-prep/compare-view-jssom.html b/tests/data/expected-results/stats-data-prep/compare-view-jssom.html index 21a822a2..6d1b6535 100644 --- a/tests/data/expected-results/stats-data-prep/compare-view-jssom.html +++ b/tests/data/expected-results/stats-data-prep/compare-view-jssom.html @@ -22,6 +22,11 @@ + + + + +
diff --git a/tests/data/expected-results/stats-data-prep/compare-view-tsom.html b/tests/data/expected-results/stats-data-prep/compare-view-tsom.html index bf059401..48f2d64e 100644 --- a/tests/data/expected-results/stats-data-prep/compare-view-tsom.html +++ b/tests/data/expected-results/stats-data-prep/compare-view-tsom.html @@ -22,6 +22,11 @@ + + + + +