Strip reasons from v4+ (#1325)

* Strip reasons from v4+ * Modify how reasons are stripped * Validate previous 2 versions of config * Generalize reason restoration
duckduckgo · Oct 9, 2023 · 5349d51 · 5349d51
1 parent 27e95e7
commit 5349d51
Show file tree

Hide file tree

Showing 6 changed files with 146 additions and 13 deletions.
diff --git a/compatibility.js b/compatibility.js
@@ -69,6 +69,47 @@ const compatFunctions = {
         }
 
         return v2Config
+    },
+    v3: (config, unmodifiedConfig) => {
+        // Breaking changes: none, reasons stripped starting in v4
+
+        const v3Config = JSON.parse(JSON.stringify(config))
+
+        /**
+         * This function will take reasons from the source array and assign them to the target array.
+         *
+         * @param {Array} target - the array to assign reasons to
+         * @param {Array} source - the array to pull reasons from
+         */
+        function assignReasons (target, source) {
+            target = target.map((exception, i) => {
+                const reason = source[i]?.reason || ''
+                return Object.assign(exception, { reason })
+            })
+        }
+
+        // Replace reasons
+        for (const key of Object.keys(unmodifiedConfig.features)) {
+            assignReasons(v3Config.features[key].exceptions, unmodifiedConfig?.features[key]?.exceptions)
+
+            if (key === 'trackerAllowlist') {
+                for (const domain of Object.keys(unmodifiedConfig.features[key].settings.allowlistedTrackers)) {
+                    const rules = v3Config.features[key].settings.allowlistedTrackers[domain].rules
+                    assignReasons(rules, unmodifiedConfig?.features[key]?.settings?.allowlistedTrackers[domain]?.rules)
+                }
+            }
+
+            if (key === 'customUserAgent') {
+                if (unmodifiedConfig.features[key].settings.omitApplicationSites) {
+                    assignReasons(v3Config.features[key].settings.omitApplicationSites, unmodifiedConfig.features[key].settings.omitApplicationSites)
+                }
+                if (unmodifiedConfig.features[key].settings.omitVersionSites) {
+                    assignReasons(v3Config.features[key].settings.omitVersionSites, unmodifiedConfig.features[key].settings.omitVersionSites)
+                }
+            }
+        }
+
+        return v3Config
     }
 }
 

diff --git a/constants.js b/constants.js
@@ -1,5 +1,5 @@
 module.exports = {
-    CURRENT_CONFIG_VERSION: 3,
+    CURRENT_CONFIG_VERSION: 4,
     OVERRIDE_DIR: 'overrides',
     GENERATED_DIR: 'generated',
     LISTS_DIR: 'features',

diff --git a/index.js b/index.js
@@ -2,7 +2,7 @@ const fs = require('fs')
 
 const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args))
 
-const { addCnameEntriesToAllowlist, inlineReasonArrays, mergeAllowlistedTrackers, addHashToFeatures } = require('./util')
+const { addCnameEntriesToAllowlist, inlineReasonArrays, mergeAllowlistedTrackers, addHashToFeatures, stripReasons } = require('./util')
 
 const { OVERRIDE_DIR, GENERATED_DIR, LISTS_DIR, BROWSERS_SUBDIR, CURRENT_CONFIG_VERSION } = require('./constants')
 
@@ -46,22 +46,28 @@ function writeConfigToDisk (platform, config) {
 
     // Write config and convert to backwards compatible versions
     let prevConfig = null
+    const unmodifiedConfig = JSON.parse(JSON.stringify(config))
     for (let i = CURRENT_CONFIG_VERSION; i > 0; i--) {
         const version = `v${i}`
         mkdirIfNeeded(`${GENERATED_DIR}/${version}`)
 
+        if (i === CURRENT_CONFIG_VERSION) {
+            stripReasons(config)
+        }
+
         if (!prevConfig) {
             prevConfig = config
         } else {
             if (!compatibility.compatFunctions[version]) {
                 throw new Error(`No compat function for config version ${version}`)
             }
 
-            prevConfig = compatibility.compatFunctions[version](prevConfig)
+            prevConfig = compatibility.compatFunctions[version](prevConfig, unmodifiedConfig)
         }
 
         const compatConfig = JSON.parse(JSON.stringify(prevConfig))
         addHashToFeatures(compatConfig)
+
         compatibility.removeEolFeatures(compatConfig, i)
         fs.writeFileSync(`${GENERATED_DIR}/${version}/${configName}-config.json`, JSON.stringify(compatConfig, null, 4))
     }
@@ -245,7 +251,9 @@ async function buildPlatforms () {
         platformConfig = inlineReasonArrays(platformConfig)
         platformConfigs[platform] = platformConfig
 
-        writeConfigToDisk(platform, platformConfig)
+        // Write config to disk
+        // Make a copy to avoid mutating the original object
+        writeConfigToDisk(platform, JSON.parse(JSON.stringify(platformConfig)))
     }
     return platformConfigs
 }

diff --git a/tests/config-tests.js b/tests/config-tests.js
@@ -12,17 +12,18 @@ function formatErrors (errors) {
     return errors.map(item => `${item.instancePath}: ${item.message}`).join(', ')
 }
 
-const v2configs = platforms.map((plat) => {
+// Test the latest 2 versions of each platform
+const latestConfigs = platforms.map((plat) => {
     return {
-        name: `${plat}-config.json`,
-        body: JSON.parse(fs.readFileSync(`./generated/v2/${plat}-config.json`))
+        name: `v4/${plat}-config.json`,
+        body: JSON.parse(fs.readFileSync(`./generated/v4/${plat}-config.json`))
     }
 })
 
-const v1configs = platforms.map((plat) => {
+const previousConfigs = platforms.map((plat) => {
     return {
-        name: `${plat}-config.json`,
-        body: JSON.parse(fs.readFileSync(`./generated/v1/${plat}-config.json`))
+        name: `v3/${plat}-config.json`,
+        body: JSON.parse(fs.readFileSync(`./generated/v3/${plat}-config.json`))
     }
 })
 
@@ -33,8 +34,42 @@ describe('Config schema tests', () => {
     const validateFeature = ajv.compile(featureSchema)
     const exceptionSchema = JSON.parse(fs.readFileSync('./tests/schemas/exception.json'))
     const validateException = ajv.compile(exceptionSchema)
+    const exceptionSchemav4 = JSON.parse(fs.readFileSync('./tests/schemas/exception-v4.json'))
+    const validateExceptionv4 = ajv.compile(exceptionSchemav4)
 
-    for (const config of v2configs.concat(v1configs)) {
+    for (const config of latestConfigs) {
+        describe(`${config.name}`, () => {
+            it('should have a valid root schema', () => {
+                expect(validateRoot(config.body)).to.be.equal(true, formatErrors(validateRoot.errors))
+            })
+
+            it('should have a vaild feature schema', () => {
+                for (const featureKey in config.body.features) {
+                    expect(validateFeature(config.body.features[featureKey])).to.be.equal(true, `Feature ${featureKey}: ` + formatErrors(validateFeature.errors))
+                }
+            })
+
+            it('should have valid exception lists', () => {
+                for (const featureKey in config.body.features) {
+                    for (const exception of config.body.features[featureKey].exceptions) {
+                        expect(validateExceptionv4(exception)).to.be.equal(true, `Feature ${featureKey}: ` + formatErrors(validateException.errors))
+                    }
+                }
+
+                for (const exception of config.body.unprotectedTemporary) {
+                    expect(validateExceptionv4(exception)).to.be.equal(true, 'unprotectedTemporary: ' + formatErrors(validateException.errors))
+                }
+            })
+
+            // appTrackerProtection should only be on the Android config since it is a large feature
+            const shouldContainAppTP = (config.name.split('/')[1] === 'android-config.json')
+            it('should contain appTrackerProtection or not', () => {
+                expect('appTrackerProtection' in config.body.features).to.be.equal(shouldContainAppTP, `appTrackerProtection expected: ${shouldContainAppTP}`)
+            })
+        })
+    }
+
+    for (const config of previousConfigs) {
         describe(`${config.name}`, () => {
             it('should have a valid root schema', () => {
                 expect(validateRoot(config.body)).to.be.equal(true, formatErrors(validateRoot.errors))
@@ -59,7 +94,7 @@ describe('Config schema tests', () => {
             })
 
             // appTrackerProtection should only be on the Android config since it is a large feature
-            const shouldContainAppTP = (config.name === 'android-config.json')
+            const shouldContainAppTP = (config.name.split('/')[1] === 'android-config.json')
             it('should contain appTrackerProtection or not', () => {
                 expect('appTrackerProtection' in config.body.features).to.be.equal(shouldContainAppTP, `appTrackerProtection expected: ${shouldContainAppTP}`)
             })

diff --git a/tests/schemas/exception-v4.json b/tests/schemas/exception-v4.json
@@ -0,0 +1,9 @@
+{
+    "type": "object",
+    "properties": {
+        "domain": {"type": "string"},
+        "reason": {"type": "string"}
+    },
+    "additionalProperties": false,
+    "required": ["domain"]
+}
diff --git a/util.js b/util.js
@@ -35,6 +35,11 @@ function addPathRule (rules, rule) {
     if (existing.domains.includes('<all>')) {
         existing.domains = ['<all>']
     }
+
+    if (existing.reason === undefined) {
+        return
+    }
+
     const reasons = existing.reason.split('; ')
     const newReason = rule.reason
     if (!reasons.includes(rule.reason)) {
@@ -125,10 +130,45 @@ function addHashToFeatures (config) {
     }
 }
 
+/**
+ * Removes reason fields from the config object
+ *
+ * @param {object} config - the config object to update
+ */
+function stripReasons (config) {
+    for (const key of Object.keys(config.features)) {
+        for (const exception of config.features[key].exceptions) {
+            delete exception.reason
+        }
+
+        if (key === 'trackerAllowlist') {
+            for (const domain of Object.keys(config.features[key].settings.allowlistedTrackers)) {
+                for (const rule of config.features[key].settings.allowlistedTrackers[domain].rules) {
+                    delete rule.reason
+                }
+            }
+        }
+
+        if (key === 'customUserAgent') {
+            if (config.features[key].settings.omitApplicationSites) {
+                for (const exception of config.features[key].settings.omitApplicationSites) {
+                    delete exception.reason
+                }
+            }
+            if (config.features[key].settings.omitVersionSites) {
+                for (const exception of config.features[key].settings.omitVersionSites) {
+                    delete exception.reason
+                }
+            }
+        }
+    }
+}
+
 module.exports = {
     addAllowlistRule: addAllowlistRule,
     addCnameEntriesToAllowlist: addCnameEntriesToAllowlist,
     inlineReasonArrays: inlineReasonArrays,
     mergeAllowlistedTrackers: mergeAllowlistedTrackers,
-    addHashToFeatures: addHashToFeatures
+    addHashToFeatures: addHashToFeatures,
+    stripReasons: stripReasons
 }