From 497715a0433c7df5a41653143994943923dcef97 Mon Sep 17 00:00:00 2001 From: Robert Kieffer Date: Mon, 11 Dec 2023 17:25:39 -0800 Subject: [PATCH 1/6] feat!: resolve extension conflicts with mime-score, close #116 --- index.js | 7 ++++--- mimeScore.js | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ test/test.js | 19 +++++++++++++++++-- 3 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 mimeScore.js diff --git a/index.js b/index.js index 9d09a11..88ccbab 100644 --- a/index.js +++ b/index.js @@ -14,6 +14,7 @@ var db = require('mime-db') var extname = require('path').extname +var mimeScore = require('./mimeScore') /** * Module variables. @@ -171,11 +172,11 @@ function populateMaps (extensions, types) { var extension = exts[i] if (types[extension]) { - var from = preference.indexOf(db[types[extension]].source) - var to = preference.indexOf(mime.source) + var from = mimeScore(types[extension]) + var to = mimeScore(type) if (types[extension] !== 'application/octet-stream' && - (from > to || (from === to && types[extension].slice(0, 12) === 'application/'))) { + from > to) { // skip the remapping continue } diff --git a/mimeScore.js b/mimeScore.js new file mode 100644 index 0000000..7a67b7d --- /dev/null +++ b/mimeScore.js @@ -0,0 +1,48 @@ +// 'mime-score' back-ported to CommonJS + +// Score RFC facets (see https://tools.ietf.org/html/rfc6838#section-3) +var FACET_SCORES = { + 'prs.': 100, + 'x-': 200, + 'x.': 300, + 'vnd.': 400, + default: 900, +} + +// Score mime source (Logic originally from `jshttp/mime-types` module) +var SOURCE_SCORES = { + nginx: 10, + apache: 20, + iana: 40, + default: 30, // definitions added by `jshttp/mime-db` project? +} + +var TYPE_SCORES = { + // prefer application/xml over text/xml + // prefer application/rtf over text/rtf + application: 1, + + // prefer font/woff over application/font-woff + font: 2, + + default: 0, +} + +/** + * Get each component of the score for a mime type. The sum of these is the + * total score. The higher the score, the more "official" the type. + */ +module.exports = function mimeScore(mimeType, source = 'default') { + let [type, subtype] = mimeType.split('/') + + const facet = subtype.replace(/(\.|x-).*/, '$1') + + const facetScore = FACET_SCORES[facet] || FACET_SCORES.default + const sourceScore = SOURCE_SCORES[source] || SOURCE_SCORES.default + const typeScore = TYPE_SCORES[type] || TYPE_SCORES.default + + // All else being equal prefer shorter types + const lengthScore = 1 - mimeType.length / 100 + + return facetScore + sourceScore + typeScore + lengthScore +} diff --git a/test/test.js b/test/test.js index 0423b8f..9b482ba 100644 --- a/test/test.js +++ b/test/test.js @@ -1,4 +1,3 @@ - var assert = require('assert') var mimeTypes = require('..') @@ -220,7 +219,23 @@ describe('mimeTypes', function () { }) it('should return mime type when there is extension, but no path', function () { - assert.strictEqual(mimeTypes.lookup('.config.json'), 'application/json') + assert.strictEqual( + mimeTypes.lookup('.config.json'), + 'application/json' + ) + }) + }) + + describe('extension conflicts', function () { + it('should use mime-score', function () { + // Test extension conflicts where the lookup has changed as a result of the switch to mime-score + assert.strictEqual(mimeTypes.lookup('exe'), 'application/x-msdownload') // was application/x-msdos-program + assert.strictEqual(mimeTypes.lookup('prc'), 'application/x-pilot') // was application/x-mobipocket-ebook + assert.strictEqual(mimeTypes.lookup('mp3'), 'audio/mp3') // was audio/mpeg + assert.strictEqual(mimeTypes.lookup('wav'), 'audio/wav') // was audio/wave + assert.strictEqual(mimeTypes.lookup('ra'), 'audio/x-realaudio') // was audio/x-pn-realaudio + assert.strictEqual(mimeTypes.lookup('x3db'), 'model/x3d+binary') // was model/x3d+fastinfoset + assert.strictEqual(mimeTypes.lookup('jpm'), 'video/jpm') // was image/jpm }) }) }) From 64326bc45700f77dcee9c006a5739995d2cd9974 Mon Sep 17 00:00:00 2001 From: Robert Kieffer Date: Mon, 11 Dec 2023 17:25:39 -0800 Subject: [PATCH 2/6] feat!: resolve extension conflicts with mime-score, close #116 --- index.js | 7 ++++--- mimeScore.js | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ test/test.js | 19 +++++++++++++++++-- 3 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 mimeScore.js diff --git a/index.js b/index.js index 9d09a11..88ccbab 100644 --- a/index.js +++ b/index.js @@ -14,6 +14,7 @@ var db = require('mime-db') var extname = require('path').extname +var mimeScore = require('./mimeScore') /** * Module variables. @@ -171,11 +172,11 @@ function populateMaps (extensions, types) { var extension = exts[i] if (types[extension]) { - var from = preference.indexOf(db[types[extension]].source) - var to = preference.indexOf(mime.source) + var from = mimeScore(types[extension]) + var to = mimeScore(type) if (types[extension] !== 'application/octet-stream' && - (from > to || (from === to && types[extension].slice(0, 12) === 'application/'))) { + from > to) { // skip the remapping continue } diff --git a/mimeScore.js b/mimeScore.js new file mode 100644 index 0000000..7a67b7d --- /dev/null +++ b/mimeScore.js @@ -0,0 +1,48 @@ +// 'mime-score' back-ported to CommonJS + +// Score RFC facets (see https://tools.ietf.org/html/rfc6838#section-3) +var FACET_SCORES = { + 'prs.': 100, + 'x-': 200, + 'x.': 300, + 'vnd.': 400, + default: 900, +} + +// Score mime source (Logic originally from `jshttp/mime-types` module) +var SOURCE_SCORES = { + nginx: 10, + apache: 20, + iana: 40, + default: 30, // definitions added by `jshttp/mime-db` project? +} + +var TYPE_SCORES = { + // prefer application/xml over text/xml + // prefer application/rtf over text/rtf + application: 1, + + // prefer font/woff over application/font-woff + font: 2, + + default: 0, +} + +/** + * Get each component of the score for a mime type. The sum of these is the + * total score. The higher the score, the more "official" the type. + */ +module.exports = function mimeScore(mimeType, source = 'default') { + let [type, subtype] = mimeType.split('/') + + const facet = subtype.replace(/(\.|x-).*/, '$1') + + const facetScore = FACET_SCORES[facet] || FACET_SCORES.default + const sourceScore = SOURCE_SCORES[source] || SOURCE_SCORES.default + const typeScore = TYPE_SCORES[type] || TYPE_SCORES.default + + // All else being equal prefer shorter types + const lengthScore = 1 - mimeType.length / 100 + + return facetScore + sourceScore + typeScore + lengthScore +} diff --git a/test/test.js b/test/test.js index 0423b8f..9b482ba 100644 --- a/test/test.js +++ b/test/test.js @@ -1,4 +1,3 @@ - var assert = require('assert') var mimeTypes = require('..') @@ -220,7 +219,23 @@ describe('mimeTypes', function () { }) it('should return mime type when there is extension, but no path', function () { - assert.strictEqual(mimeTypes.lookup('.config.json'), 'application/json') + assert.strictEqual( + mimeTypes.lookup('.config.json'), + 'application/json' + ) + }) + }) + + describe('extension conflicts', function () { + it('should use mime-score', function () { + // Test extension conflicts where the lookup has changed as a result of the switch to mime-score + assert.strictEqual(mimeTypes.lookup('exe'), 'application/x-msdownload') // was application/x-msdos-program + assert.strictEqual(mimeTypes.lookup('prc'), 'application/x-pilot') // was application/x-mobipocket-ebook + assert.strictEqual(mimeTypes.lookup('mp3'), 'audio/mp3') // was audio/mpeg + assert.strictEqual(mimeTypes.lookup('wav'), 'audio/wav') // was audio/wave + assert.strictEqual(mimeTypes.lookup('ra'), 'audio/x-realaudio') // was audio/x-pn-realaudio + assert.strictEqual(mimeTypes.lookup('x3db'), 'model/x3d+binary') // was model/x3d+fastinfoset + assert.strictEqual(mimeTypes.lookup('jpm'), 'video/jpm') // was image/jpm }) }) }) From 0ba40b09b13b45a91a80a725c667d4c879cac9dc Mon Sep 17 00:00:00 2001 From: Robert Kieffer Date: Mon, 26 Aug 2024 06:42:46 -0700 Subject: [PATCH 3/6] chore: fix lint issues --- index.js | 3 --- mimeScore.js | 10 +++++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 88ccbab..a8fd828 100644 --- a/index.js +++ b/index.js @@ -153,9 +153,6 @@ function lookup (path) { */ function populateMaps (extensions, types) { - // source preference (least -> most) - var preference = ['nginx', 'apache', undefined, 'iana'] - Object.keys(db).forEach(function forEachMimeType (type) { var mime = db[type] var exts = mime.extensions diff --git a/mimeScore.js b/mimeScore.js index 7a67b7d..f6c39e8 100644 --- a/mimeScore.js +++ b/mimeScore.js @@ -6,7 +6,7 @@ var FACET_SCORES = { 'x-': 200, 'x.': 300, 'vnd.': 400, - default: 900, + default: 900 } // Score mime source (Logic originally from `jshttp/mime-types` module) @@ -14,7 +14,7 @@ var SOURCE_SCORES = { nginx: 10, apache: 20, iana: 40, - default: 30, // definitions added by `jshttp/mime-db` project? + default: 30 // definitions added by `jshttp/mime-db` project? } var TYPE_SCORES = { @@ -25,15 +25,15 @@ var TYPE_SCORES = { // prefer font/woff over application/font-woff font: 2, - default: 0, + default: 0 } /** * Get each component of the score for a mime type. The sum of these is the * total score. The higher the score, the more "official" the type. */ -module.exports = function mimeScore(mimeType, source = 'default') { - let [type, subtype] = mimeType.split('/') +module.exports = function mimeScore (mimeType, source = 'default') { + const [type, subtype] = mimeType.split('/') const facet = subtype.replace(/(\.|x-).*/, '$1') From aede407ff2acc259b022d2fe857db3ce29471ff8 Mon Sep 17 00:00:00 2001 From: Robert Kieffer Date: Mon, 26 Aug 2024 11:12:53 -0700 Subject: [PATCH 4/6] wip: lint, logic, and test fixup --- index.js | 57 +++++++++++++++++++++++++++++++++++++--------------- mimeScore.js | 4 ++++ test/test.js | 19 +++++++----------- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/index.js b/index.js index a8fd828..c348cb6 100644 --- a/index.js +++ b/index.js @@ -36,6 +36,7 @@ exports.extension = extension exports.extensions = Object.create(null) exports.lookup = lookup exports.types = Object.create(null) +exports._extensionConflicts = [] // Populate the extensions/types maps populateMaps(exports.extensions, exports.types) @@ -81,9 +82,7 @@ function contentType (str) { return false } - var mime = str.indexOf('/') === -1 - ? exports.lookup(str) - : str + var mime = str.indexOf('/') === -1 ? exports.lookup(str) : str if (!mime) { return false @@ -167,20 +166,46 @@ function populateMaps (extensions, types) { // extension -> mime for (var i = 0; i < exts.length; i++) { var extension = exts[i] - - if (types[extension]) { - var from = mimeScore(types[extension]) - var to = mimeScore(type) - - if (types[extension] !== 'application/octet-stream' && - from > to) { - // skip the remapping - continue - } + types[extension] = _preferredType(extension, types[extension], type) + + // DELETE (eventually): Capture extension->type maps that change as a + // result of switching to mime-score. This is just to help make reviewing + // PR #119 easier, and can be removed once that PR is approved. + const legacyType = _preferredTypeLegacy( + extension, + types[extension], + type + ) + if (legacyType !== types[extension]) { + exports._extensionConflicts.push([extension, legacyType, types[extension]]) } - - // set the extension -> mime - types[extension] = type } }) } + +// Resolve type conflict using mime-score +function _preferredType (ext, type0, type1) { + var score0 = type0 ? mimeScore(type0, db[type0].source) : 0 + var score1 = type1 ? mimeScore(type1, db[type1].source) : 0 + + return score0 > score1 ? type0 : type1 +} + +// Resolve type conflict using pre-mime-score logic +function _preferredTypeLegacy (ext, type0, type1) { + var SOURCE_RANK = ['nginx', 'apache', undefined, 'iana'] + + var score0 = type0 ? SOURCE_RANK.indexOf(db[type0].source) : 0 + var score1 = type1 ? SOURCE_RANK.indexOf(db[type1].source) : 0 + + if ( + exports.types[extension] !== 'application/octet-stream' && + (score0 > score1 || + (score0 === score1 && + exports.types[extension]?.slice(0, 12) === 'application/')) + ) { + return type0 + } + + return score0 > score1 ? type0 : type1 +} diff --git a/mimeScore.js b/mimeScore.js index f6c39e8..e2ecc66 100644 --- a/mimeScore.js +++ b/mimeScore.js @@ -33,6 +33,10 @@ var TYPE_SCORES = { * total score. The higher the score, the more "official" the type. */ module.exports = function mimeScore (mimeType, source = 'default') { + if (mimeType === 'application/octet-stream') { + return 0 + } + const [type, subtype] = mimeType.split('/') const facet = subtype.replace(/(\.|x-).*/, '$1') diff --git a/test/test.js b/test/test.js index 9b482ba..e5a1a6d 100644 --- a/test/test.js +++ b/test/test.js @@ -225,18 +225,13 @@ describe('mimeTypes', function () { ) }) }) + }) - describe('extension conflicts', function () { - it('should use mime-score', function () { - // Test extension conflicts where the lookup has changed as a result of the switch to mime-score - assert.strictEqual(mimeTypes.lookup('exe'), 'application/x-msdownload') // was application/x-msdos-program - assert.strictEqual(mimeTypes.lookup('prc'), 'application/x-pilot') // was application/x-mobipocket-ebook - assert.strictEqual(mimeTypes.lookup('mp3'), 'audio/mp3') // was audio/mpeg - assert.strictEqual(mimeTypes.lookup('wav'), 'audio/wav') // was audio/wave - assert.strictEqual(mimeTypes.lookup('ra'), 'audio/x-realaudio') // was audio/x-pn-realaudio - assert.strictEqual(mimeTypes.lookup('x3db'), 'model/x3d+binary') // was model/x3d+fastinfoset - assert.strictEqual(mimeTypes.lookup('jpm'), 'video/jpm') // was image/jpm - }) - }) + // Note the changes in extension->type mapping that result from using mime-score + describe('extension conflicts', function () { + console.warn('Mime-score logic changes extension->type mappings for the following:') + for (var [extension, legacy, current] of mimeTypes._extensionConflicts) { + console.warn(` ${extension} -> ${legacy} is now ${current}`) + } }) }) From ca6ddb3eeb5ac4ce9c42bcfb6fa49dbcc1f11bc5 Mon Sep 17 00:00:00 2001 From: Robert Kieffer Date: Mon, 26 Aug 2024 11:50:20 -0700 Subject: [PATCH 5/6] test: tweak log output --- test/test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.js b/test/test.js index e5a1a6d..0dcf5b8 100644 --- a/test/test.js +++ b/test/test.js @@ -231,7 +231,7 @@ describe('mimeTypes', function () { describe('extension conflicts', function () { console.warn('Mime-score logic changes extension->type mappings for the following:') for (var [extension, legacy, current] of mimeTypes._extensionConflicts) { - console.warn(` ${extension} -> ${legacy} is now ${current}`) + console.warn(`* ${extension} -> ${legacy} is now ${current}`) } }) }) From 5215669d013eb59f60668368636528461e46f72d Mon Sep 17 00:00:00 2001 From: Wes Todd Date: Sat, 31 Aug 2024 08:33:37 -0500 Subject: [PATCH 6/6] fix: update history.md --- HISTORY.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index c5043b7..4384def 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,17 +1,18 @@ -2.1.35 / 2022-03-12 +unreleased =================== - * deps: mime-db@1.52.0 - - Add extensions from IANA for more `image/*` types - - Add extension `.asc` to `application/pgp-keys` - - Add extensions to various XML types - - Add new upstream MIME types - -2.1.34 / 2021-11-08 -=================== - - * deps: mime-db@1.51.0 - - Add new upstream MIME types +* resolve extension conflicts with mime-score (#119) + * asc -> application/pgp-signature is now application/pgp-keys + * mpp -> application/vnd.ms-project is now application/dash-patch+xml + * ac -> application/vnd.nokia.n-gage.ac+xml is now application/pkix-attr-cert + * bdoc -> application/x-bdoc is now application/bdoc + * wmz -> application/x-msmetafile is now application/x-ms-wmz + * xsl -> application/xslt+xml is now application/xml + * wav -> audio/wave is now audio/wav + * rtf -> text/rtf is now application/rtf + * xml -> text/xml is now application/xml + * mp4 -> video/mp4 is now application/mp4 + * mpg4 -> video/mp4 is now application/mp4 2.1.33 / 2021-10-01 ===================