From cb798467b64becd4f2363b8a4df9f828262501f1 Mon Sep 17 00:00:00 2001 From: Pop Chunhapanya Date: Thu, 3 Feb 2022 04:12:34 +0800 Subject: [PATCH 1/5] Use only body params to identify issue request Since there is no more query param, we will use only the body param to identify when the user want to issue tokens. --- src/background/providers/cloudflare.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/background/providers/cloudflare.ts b/src/background/providers/cloudflare.ts index 2f9f1fe1..91d7d907 100644 --- a/src/background/providers/cloudflare.ts +++ b/src/background/providers/cloudflare.ts @@ -13,8 +13,7 @@ const ISSUANCE_BODY_PARAM_NAME = 'blinded-tokens'; const COMMITMENT_URL = 'https://raw.githubusercontent.com/privacypass/ec-commitments/master/commitments-p256.json'; -const QUALIFIED_QUERY_PARAMS = ['__cf_chl_captcha_tk__', '__cf_chl_managed_tk__']; -const QUALIFIED_BODY_PARAMS = ['g-recaptcha-response', 'h-captcha-response', 'cf_captcha_kind']; +const QUALIFIED_BODY_PARAMS = ['h-captcha-response', 'cf_captcha_kind']; const CHL_BYPASS_SUPPORT = 'cf-chl-bypass'; const DEFAULT_ISSUING_HOSTNAME = 'captcha.website'; @@ -237,13 +236,10 @@ export class CloudflareProvider implements Provider { return; } - const hasQueryParams = QUALIFIED_QUERY_PARAMS.some((param) => { - return url.searchParams.has(param); - }); - const hasBodyParams = QUALIFIED_BODY_PARAMS.some((param) => { + const hasBodyParams = QUALIFIED_BODY_PARAMS.every((param) => { return details.requestBody !== null && param in details.requestBody.formData!; }); - if (!hasQueryParams || !hasBodyParams) { + if (!hasBodyParams) { return; } From 01ad66d543221e11d1f98c69938633203e9be9f0 Mon Sep 17 00:00:00 2001 From: Pop Chunhapanya Date: Thu, 3 Feb 2022 04:13:40 +0800 Subject: [PATCH 2/5] Add extraHeaders into extraInfos for Chrome Chrome needs the extraHeaders flag in extraInfos to read the Referer header. --- src/background/index.ts | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/background/index.ts b/src/background/index.ts index 46c79005..9c279e4c 100644 --- a/src/background/index.ts +++ b/src/background/index.ts @@ -21,9 +21,28 @@ declare global { } } +declare let browser: any; + window.ACTIVE_TAB_ID = chrome.tabs.TAB_ID_NONE; window.TABS = new Map(); +const BROWSERS = { + CHROME: 'Chrome', + FIREFOX: 'Firefox', + EDGE: 'Edge', +} as const; +type BROWSERS = typeof BROWSERS[keyof typeof BROWSERS]; + +function getBrowser(): BROWSERS { + if (typeof chrome !== 'undefined') { + if (typeof browser !== 'undefined') { + return BROWSERS.FIREFOX; + } + return BROWSERS.CHROME; + } + return BROWSERS.EDGE; +} + /* Listeners for navigator */ chrome.tabs.onActivated.addListener(handleActivated); @@ -57,10 +76,14 @@ chrome.webRequest.onBeforeRequest.addListener(handleBeforeRequest, { urls: [''] }, - ['requestHeaders', 'blocking'], + extraInfos, ); chrome.webRequest.onHeadersReceived.addListener(handleHeadersReceived, { urls: [''] }, [ From 7c7b066d6e8314c6671822a198cfa95e5b1dd747 Mon Sep 17 00:00:00 2001 From: Pop Chunhapanya Date: Thu, 3 Feb 2022 04:16:29 +0800 Subject: [PATCH 3/5] Change the issue request flow for Cloudflare Previously we used axios to send an issuance request in handleBeforeRequest, but now we will send the request in handleBeforeSendHeaders because we need to read the Referer header first. After reading the Referer header, we extract the __cf_chl_tk query param in the Referer url and add the __cf_chl_f_tk query param with that token in the issuance request URL. We need a new "issueInfo" property to remember the request body we can read in handleBeforeRequest and then it will be read in handleBeforeSendHeaders so that in handleBeforeSendHeaders we will have both the body and the Referer header. --- src/background/providers/cloudflare.ts | 127 ++++++++++++++++--------- 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/src/background/providers/cloudflare.ts b/src/background/providers/cloudflare.ts index 91d7d907..5eb226a7 100644 --- a/src/background/providers/cloudflare.ts +++ b/src/background/providers/cloudflare.ts @@ -18,6 +18,9 @@ const QUALIFIED_BODY_PARAMS = ['h-captcha-response', 'cf_captcha_kind']; const CHL_BYPASS_SUPPORT = 'cf-chl-bypass'; const DEFAULT_ISSUING_HOSTNAME = 'captcha.website'; +const REFERER_QUERY_PARAM = '__cf_chl_tk'; +const QUERY_PARAM = '__cf_chl_f_tk'; + const VERIFICATION_KEY = `-----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAExf0AftemLr0YSz5odoj3eJv6SkOF VcH7NNb2xwdEz6Pxm44tvovEl/E+si8hdIDVg1Ys+cbaWwP0jYJW3ygv+Q== @@ -30,11 +33,17 @@ interface RedeemInfo { token: Token; } +interface IssueInfo { + requestId: string; + formData: { [key: string]: string[] | string }; +} + export class CloudflareProvider implements Provider { static readonly ID: number = 1; private callbacks: Callbacks; private storage: Storage; + private issueInfo: IssueInfo | null; private redeemInfo: RedeemInfo | null; constructor(storage: Storage, callbacks: Callbacks) { @@ -44,6 +53,7 @@ export class CloudflareProvider implements Provider { this.callbacks = callbacks; this.storage = storage; + this.issueInfo = null; this.redeemInfo = null; } @@ -190,44 +200,85 @@ export class CloudflareProvider implements Provider { handleBeforeSendHeaders( details: chrome.webRequest.WebRequestHeadersDetails, ): chrome.webRequest.BlockingResponse | void { - if (this.redeemInfo === null || details.requestId !== this.redeemInfo.requestId) { - return; + // If we suppose to redeem a token with this request + if (this.redeemInfo !== null && details.requestId === this.redeemInfo.requestId) { + const url = new URL(details.url); + + const token = this.redeemInfo.token; + // Clear the redeem info to indicate that we are already redeeming the token. + this.redeemInfo = null; + + const key = token.getMacKey(); + const binding = voprf.createRequestBinding(key, [ + voprf.getBytesFromString(url.hostname), + voprf.getBytesFromString(details.method + ' ' + url.pathname), + ]); + + const contents = [ + voprf.getBase64FromBytes(token.getInput()), + binding, + voprf.getBase64FromString(JSON.stringify(voprf.defaultECSettings)), + ]; + const redemption = btoa(JSON.stringify({ type: 'Redeem', contents })); + + const headers = details.requestHeaders ?? []; + headers.push({ name: 'challenge-bypass-token', value: redemption }); + + this.callbacks.updateIcon(this.getBadgeText()); + + return { + requestHeaders: headers, + }; } - const url = new URL(details.url); - - const token = this.redeemInfo!.token; - // Clear the redeem info to indicate that we are already redeeming the token. - this.redeemInfo = null; - - const key = token.getMacKey(); - const binding = voprf.createRequestBinding(key, [ - voprf.getBytesFromString(url.hostname), - voprf.getBytesFromString(details.method + ' ' + url.pathname), - ]); - - const contents = [ - voprf.getBase64FromBytes(token.getInput()), - binding, - voprf.getBase64FromString(JSON.stringify(voprf.defaultECSettings)), - ]; - const redemption = btoa(JSON.stringify({ type: 'Redeem', contents })); - - const headers = details.requestHeaders ?? []; - headers.push({ name: 'challenge-bypass-token', value: redemption }); - - this.callbacks.updateIcon(this.getBadgeText()); + // If we suppose to issue tokens with this request + if (this.issueInfo !== null && details.requestId === this.issueInfo.requestId) { + const formData = this.issueInfo.formData; + // Clear the issue info to indicate that we are already issuing tokens. + this.issueInfo = null; + + // We are supposed to also send a Referer header in the issuance request, if there is + // any in the original request. But the browsers don't allow us to send a Referer + // header according to https://xhr.spec.whatwg.org/#dom-xmlhttprequest-setrequestheader + // So we need to extract the token from the Referer header and send it in the query + // param __cf_chl_f_tk instead. (Note that this token is not a Privacy Pass token. + let token: string | null = null; + if (details.requestHeaders !== undefined) { + details.requestHeaders.forEach((header) => { + // Filter only for Referrer header. + if (header.name === 'Referer' && header.value !== undefined) { + const url = new URL(header.value); + token = url.searchParams.get(REFERER_QUERY_PARAM); + } + }); + } - return { - requestHeaders: headers, - }; + (async () => { + const url = new URL(details.url); + if (token !== null) { + url.searchParams.append(QUERY_PARAM, token); + } + + // Issue tokens. + const tokens = await this.issue(url.href, formData); + // Store tokens. + const cached = this.getStoredTokens(); + this.setStoredTokens(cached.concat(tokens)); + + this.callbacks.navigateUrl(`${url.origin}${url.pathname}`); + })(); + + // TODO I tried to use redirectUrl with data URL or text/html and text/plain but it didn't work, so I continue + // cancelling the request. However, it seems that we can use image/* except image/svg+html. Let's figure how to + // use image data URL later. + // https://blog.mozilla.org/security/2017/11/27/blocking-top-level-navigations-data-urls-firefox-59/ + return { cancel: true }; + } } handleBeforeRequest( details: chrome.webRequest.WebRequestBodyDetails, ): chrome.webRequest.BlockingResponse | void { - const url = new URL(details.url); - if ( details.requestBody === null || details.requestBody === undefined || @@ -253,21 +304,7 @@ export class CloudflareProvider implements Provider { } } - (async () => { - // Issue tokens. - const tokens = await this.issue(details.url, flattenFormData); - // Store tokens. - const cached = this.getStoredTokens(); - this.setStoredTokens(cached.concat(tokens)); - - this.callbacks.navigateUrl(`${url.origin}${url.pathname}`); - })(); - - // TODO I tried to use redirectUrl with data URL or text/html and text/plain but it didn't work, so I continue - // cancelling the request. However, it seems that we can use image/* except image/svg+html. Let's figure how to - // use image data URL later. - // https://blog.mozilla.org/security/2017/11/27/blocking-top-level-navigations-data-urls-firefox-59/ - return { cancel: true }; + this.issueInfo = { requestId: details.requestId, formData: flattenFormData }; } handleHeadersReceived( From 98f8a8d4427c986cbb032f3db340420654449cc4 Mon Sep 17 00:00:00 2001 From: Pop Chunhapanya Date: Thu, 3 Feb 2022 18:40:05 +0800 Subject: [PATCH 4/5] Test new Cloudflare issuance request flow --- src/background/providers/cloudflare.test.ts | 205 +++++++++++++++++--- 1 file changed, 175 insertions(+), 30 deletions(-) diff --git a/src/background/providers/cloudflare.test.ts b/src/background/providers/cloudflare.test.ts index 94b3de00..a5198bd9 100644 --- a/src/background/providers/cloudflare.test.ts +++ b/src/background/providers/cloudflare.test.ts @@ -57,17 +57,32 @@ test('getBadgeText', () => { }); /* - * The issuance involves handleBeforeRequest listener. + * The issuance involves handleBeforeRequest and handleBeforeSendHeaders + * listeners. In handleBeforeRequest listener, * 1. Firstly, the listener check if the request looks like the one that we * should send an issuance request. - * 2. If it passes the check, the listener returns the cancel command to - * cancel the request. If not, it returns nothing and let the request - * continue. - * 3. At the same time the listener returns, it calls a private method + * 2. If it passes the check, The listener sets "issueInfo" property which + * includes the request id and the form data of the request. The property + * will be used by handleBeforeSendHeaders again to issue the tokens. If not, + * it returns nothing and let the request continue. + * + * In handleBeforeSendHeaders, + * 1. The listener will check if the provided request id matches the + * request id in "issueInfo". If so, it means that we are issuing the tokens. + * If not, it returns nothing and let the request continue. + * 2. If it passes the check, the listener extract the form data from + * "issueInfo" clears the "issueInfo" property because "issueInfo" is used + * already. If not, it returns nothing and let the request continue. + * 3. The listener tries to look for the Referer header to get + * the (not PP) token from __cf_chl_tk query param in the Referer url. + * 4. The listener returns the cancel command to cancel the request. + * 5. At the same time the listener returns, it calls a private method * "issue" to send an issuance request to the server and the method return - * an array of issued tokens. - * 4. The listener stored the issued tokens in the storage. - * 5. The listener reloads the tab to get the proper web page for the tab.); + * an array of issued tokens. In the issuance request, the body will be the + * form data extracted from "issueInfo" earlier and also include the + * __cf_chl_f_tk query param with the token it got from Step 3 (if any). + * 6. The listener stored the issued tokens in the storage. + * 7. The listener reloads the tab to get the proper web page for the tab. */ describe('issuance', () => { describe('handleBeforeRequest', () => { @@ -77,12 +92,9 @@ describe('issuance', () => { const navigateUrl = jest.fn(); const provider = new CloudflareProvider(storage, { updateIcon, navigateUrl }); - const tokens = [new Token(), new Token(), new Token()]; - const issue = jest.fn(async () => { - return tokens; - }); + const issue = jest.fn(async () => []); provider['issue'] = issue; - const url = 'https://captcha.website/?__cf_chl_captcha_tk__=query-param'; + const url = 'https://captcha.website'; const details = { method: 'POST', url, @@ -92,6 +104,47 @@ describe('issuance', () => { tabId: 1, type: 'xmlhttprequest' as chrome.webRequest.ResourceType, timeStamp: 1, + requestBody: { + formData: { + ['h-captcha-response']: ['body-param'], + ['cf_captcha_kind']: ['body-param'], + }, + }, + }; + const result = await provider.handleBeforeRequest(details); + expect(result).toBeUndefined(); + expect(issue).not.toHaveBeenCalled(); + expect(navigateUrl).not.toHaveBeenCalled(); + + const issueInfo = provider['issueInfo']; + expect(issueInfo!.requestId).toEqual(details.requestId); + expect(issueInfo!.formData).toStrictEqual({ + ['h-captcha-response']: 'body-param', + ['cf_captcha_kind']: 'body-param', + }); + }); + + /* + * The request is invalid only if the body has both + * 'h-captcha-response' and 'cf_captcha_kind' params. + */ + test('invalid request', async () => { + const storage = new StorageMock(); + const updateIcon = jest.fn(); + const navigateUrl = jest.fn(); + + const provider = new CloudflareProvider(storage, { updateIcon, navigateUrl }); + const issue = jest.fn(async () => []); + provider['issue'] = issue; + const details = { + method: 'GET', + url: 'https://cloudflare.com/', + requestId: 'xxx', + frameId: 1, + parentFrameId: 1, + tabId: 1, + type: 'xmlhttprequest' as chrome.webRequest.ResourceType, + timeStamp: 1, requestBody: { formData: { ['h-captcha-response']: ['body-param'], @@ -99,11 +152,57 @@ describe('issuance', () => { }, }; const result = await provider.handleBeforeRequest(details); - expect(result).toEqual({ cancel: true }); + expect(result).toBeUndefined(); + expect(issue).not.toHaveBeenCalled(); + expect(navigateUrl).not.toHaveBeenCalled(); + }); + }); + + describe('handleBeforeSendHeaders', () => { + test('with issueInfo with Referer header', async () => { + const storage = new StorageMock(); + const updateIcon = jest.fn(); + const navigateUrl = jest.fn(); + + const provider = new CloudflareProvider(storage, { updateIcon, navigateUrl }); + const tokens = [new Token(), new Token(), new Token()]; + const issue = jest.fn(async () => { + return tokens; + }); + provider['issue'] = issue; + const issueInfo = { + requestId: 'xxx', + formData: { + ['h-captcha-response']: 'body-param', + ['cf_captcha_kind']: 'body-param', + }, + }; + provider['issueInfo'] = issueInfo; + const details = { + method: 'POST', + url: 'https://captcha.website', + requestId: 'xxx', + frameId: 1, + parentFrameId: 1, + tabId: 1, + type: 'xmlhttprequest' as chrome.webRequest.ResourceType, + timeStamp: 1, + requestHeaders: [ + { + name: 'Referer', + value: 'https://captcha.website/?__cf_chl_tk=token', + }, + ], + }; + const result = await provider.handleBeforeSendHeaders(details); + expect(result).toStrictEqual({ cancel: true }); + const newIssueInfo = provider['issueInfo']; + expect(newIssueInfo).toBeNull(); expect(issue.mock.calls.length).toBe(1); - expect(issue).toHaveBeenCalledWith(url, { + expect(issue).toHaveBeenCalledWith('https://captcha.website/?__cf_chl_f_tk=token', { ['h-captcha-response']: 'body-param', + ['cf_captcha_kind']: 'body-param', }); expect(navigateUrl.mock.calls.length).toBe(1); @@ -116,17 +215,58 @@ describe('issuance', () => { ); }); - /* - * The request is invalid if any of the followings is true: - * 1. It has no url param of any of the followings: - * a. '__cf_chl_captcha_tk__' - * b. '__cf_chl_managed_tk__' - * 2. It has no body param of any of the followings: - * a. 'g-recaptcha-response' - * b. 'h-captcha-response' - * c. 'cf_captcha_kind' - */ - test('invalid request', async () => { + test('with issueInfo without Referer header', async () => { + const storage = new StorageMock(); + const updateIcon = jest.fn(); + const navigateUrl = jest.fn(); + + const provider = new CloudflareProvider(storage, { updateIcon, navigateUrl }); + const tokens = [new Token(), new Token(), new Token()]; + const issue = jest.fn(async () => { + return tokens; + }); + provider['issue'] = issue; + const issueInfo = { + requestId: 'xxx', + formData: { + ['h-captcha-response']: 'body-param', + ['cf_captcha_kind']: 'body-param', + }, + }; + provider['issueInfo'] = issueInfo; + const details = { + method: 'POST', + url: 'https://captcha.website/?__cf_chl_f_tk=token', + requestId: 'xxx', + frameId: 1, + parentFrameId: 1, + tabId: 1, + type: 'xmlhttprequest' as chrome.webRequest.ResourceType, + timeStamp: 1, + requestHeaders: [], + }; + const result = await provider.handleBeforeSendHeaders(details); + expect(result).toStrictEqual({ cancel: true }); + const newIssueInfo = provider['issueInfo']; + expect(newIssueInfo).toBeNull(); + + expect(issue.mock.calls.length).toBe(1); + expect(issue).toHaveBeenCalledWith('https://captcha.website/?__cf_chl_f_tk=token', { + ['h-captcha-response']: 'body-param', + ['cf_captcha_kind']: 'body-param', + }); + + expect(navigateUrl.mock.calls.length).toBe(1); + expect(navigateUrl).toHaveBeenCalledWith('https://captcha.website/'); + + // Expect the tokens are added. + const storedTokens = provider['getStoredTokens'](); + expect(storedTokens.map((token) => token.toString())).toEqual( + tokens.map((token) => token.toString()), + ); + }); + + test('without issueInfo', async () => { const storage = new StorageMock(); const updateIcon = jest.fn(); const navigateUrl = jest.fn(); @@ -135,17 +275,22 @@ describe('issuance', () => { const issue = jest.fn(async () => []); provider['issue'] = issue; const details = { - method: 'GET', - url: 'https://cloudflare.com/', + method: 'POST', + url: 'https://captcha.website', requestId: 'xxx', frameId: 1, parentFrameId: 1, tabId: 1, type: 'xmlhttprequest' as chrome.webRequest.ResourceType, timeStamp: 1, - requestBody: {}, + requestHeaders: [ + { + name: 'Referer', + value: 'https://captcha.website/?__cf_chl_tk=token', + }, + ], }; - const result = await provider.handleBeforeRequest(details); + const result = await provider.handleBeforeSendHeaders(details); expect(result).toBeUndefined(); expect(issue).not.toHaveBeenCalled(); expect(navigateUrl).not.toHaveBeenCalled(); From ac60db3f5f49eb504e6a5fdf7f30e2705a0c1284 Mon Sep 17 00:00:00 2001 From: Pop Chunhapanya Date: Thu, 3 Feb 2022 18:41:19 +0800 Subject: [PATCH 5/5] Bump the version to 3.0.1 --- package.json | 2 +- public/manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index fb4fe06e..913baeb6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "privacy-pass", - "version": "3.0.0", + "version": "3.0.1", "contributors": [ "Suphanat Chunhapanya ", "Armando Faz " diff --git a/public/manifest.json b/public/manifest.json index 3a5edf12..a6676770 100644 --- a/public/manifest.json +++ b/public/manifest.json @@ -2,7 +2,7 @@ "name": "Privacy Pass", "manifest_version": 2, "description": "Client support for Privacy Pass anonymous authorization protocol.", - "version": "3.0.0", + "version": "3.0.1", "icons": { "32": "icons/32/gold.png", "48": "icons/48/gold.png",