From 42ed2dc421615711f02606b7b86cd0f7acd8d801 Mon Sep 17 00:00:00 2001 From: fabianospinelli Date: Wed, 3 Apr 2024 09:25:22 +0000 Subject: [PATCH 1/4] Added functions to API services in order to check private addresses --- src/pages/api/services/index.ts | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/pages/api/services/index.ts b/src/pages/api/services/index.ts index d312568d..b6f01238 100644 --- a/src/pages/api/services/index.ts +++ b/src/pages/api/services/index.ts @@ -13,6 +13,27 @@ import fs from 'fs'; import getConfig from 'next/config'; import { getLatestFailDate } from 'modules/Github/api'; + +function isPrivateAddress(checkUrl) { + var ipv4PrivatePattern = /^(10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3})$/; + var ipv6PrivatePattern = /^(?:[fF][cCdD]|[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2}|(?:2001:)?[dD][bB][aA][8-9]::|(?:2001:)?[dD][bB][cCdD]:|(?:(?:[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2})?::)?[0-9a-fA-F]{1,4}:(?:[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2})?)$/; + + // Extract the hostname from the URL + let hostname; + try { + const urlObj = new URL(checkUrl); + hostname = urlObj.hostname; + } catch (err) { + return false; + } + + if (ipv4PrivatePattern.test(hostname) || ipv6PrivatePattern.test(hostname) || hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1") { + return true; + } else { + return false; + } +} + const { serverRuntimeConfig } = getConfig(); const get = @@ -32,6 +53,20 @@ const get = return res; } + if (json.hasOwnProperty("fetch")) { + const url = json.fetch; + if (isPrivateAddress(url)) { + res.statusCode = HttpStatusCode.OK; + res.json({ + status: 'ko', + message: 'Could not download url', + url: '', + error: "FetchDocumentError: request to " + url + " failed, reason: URL is a private address", + }); + return res; + } + } + const downloadResult = await downloadUrl(json, { folderDirPath: serverRuntimeConfig.scrapedFilesFolder, newUrlDirPath: `${process.env.NEXT_PUBLIC_BASE_PATH || ''}${ From d9ea6c69069655fc8660cc4f8d2ba436099c3880 Mon Sep 17 00:00:00 2001 From: Fabiano Spinelli <53008742+fabianospinelli@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:09:51 +0200 Subject: [PATCH 2/4] Update src/pages/api/services/index.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Biron --- src/pages/api/services/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pages/api/services/index.ts b/src/pages/api/services/index.ts index b6f01238..e61bd854 100644 --- a/src/pages/api/services/index.ts +++ b/src/pages/api/services/index.ts @@ -61,7 +61,8 @@ const get = status: 'ko', message: 'Could not download url', url: '', - error: "FetchDocumentError: request to " + url + " failed, reason: URL is a private address", + error: "URL seems to be a private address and has been blocked", + ``` }); return res; } From 2d52168cf20529c8c1b471de01743db8b045ae79 Mon Sep 17 00:00:00 2001 From: Fabiano Spinelli <53008742+fabianospinelli@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:10:29 +0200 Subject: [PATCH 3/4] Update src/pages/api/services/index.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Biron --- src/pages/api/services/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pages/api/services/index.ts b/src/pages/api/services/index.ts index e61bd854..1d20a28b 100644 --- a/src/pages/api/services/index.ts +++ b/src/pages/api/services/index.ts @@ -27,11 +27,7 @@ function isPrivateAddress(checkUrl) { return false; } - if (ipv4PrivatePattern.test(hostname) || ipv6PrivatePattern.test(hostname) || hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1") { - return true; - } else { - return false; - } +return (ipv4PrivatePattern.test(hostname) || ipv6PrivatePattern.test(hostname) || hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1" || hostname === "0.0.0.0"); } const { serverRuntimeConfig } = getConfig(); From f7c7100ed2b041bb748c500522001505522a917b Mon Sep 17 00:00:00 2001 From: fabianospinelli Date: Thu, 18 Apr 2024 09:12:37 +0000 Subject: [PATCH 4/4] Added suggestion from OTA team --- src/pages/api/services/index.ts | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/pages/api/services/index.ts b/src/pages/api/services/index.ts index b6f01238..f994aafc 100644 --- a/src/pages/api/services/index.ts +++ b/src/pages/api/services/index.ts @@ -14,24 +14,28 @@ import getConfig from 'next/config'; import { getLatestFailDate } from 'modules/Github/api'; -function isPrivateAddress(checkUrl) { - var ipv4PrivatePattern = /^(10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3})$/; - var ipv6PrivatePattern = /^(?:[fF][cCdD]|[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2}|(?:2001:)?[dD][bB][aA][8-9]::|(?:2001:)?[dD][bB][cCdD]:|(?:(?:[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2})?::)?[0-9a-fA-F]{1,4}:(?:[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2})?)$/; +function isPrivateAddress(checkUrl: string): boolean { + // The ipv4PrivatePattern matches private IPv4 addresses. These ranges are coming from RFC 1918, which defines private IPv4 address ranges: + // - 10.0.0.0 - 10.255.255.255 + // - 172.16.0.0 - 172.31.255.255 + // - 192.168.0.0 - 192.168.255.255 + const ipv4PrivatePattern: RegExp = /^(10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3})$/; + // The ipv6PrivatePattern matches private IPv6 addresses. These ranges are coming from RFC 4193, which defines Unique Local IPv6 Unicast Addresses: + // - fc00::/7 + // - fd00::/8 + // Additionally, it includes the loopback and unspecified addresses (::1 and ::) and the link-local addresses (fe80::/10). + const ipv6PrivatePattern: RegExp = /^(?:[fF][cCdD]|[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2}|(?:2001:)?[dD][bB][aA][8-9]::|(?:2001:)?[dD][bB][cCdD]:|(?:(?:[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2})?::)?[0-9a-fA-F]{1,4}:(?:[fF]{3}(?::[0-9a-fA-F]{1,4}){1,2})?)$/; // Extract the hostname from the URL - let hostname; + let hostname: string; try { - const urlObj = new URL(checkUrl); + const urlObj: URL = new URL(checkUrl); hostname = urlObj.hostname; } catch (err) { return false; } - if (ipv4PrivatePattern.test(hostname) || ipv6PrivatePattern.test(hostname) || hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1") { - return true; - } else { - return false; - } + return (ipv4PrivatePattern.test(hostname) || ipv6PrivatePattern.test(hostname) || hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1" || hostname === "0.0.0.0"); } const { serverRuntimeConfig } = getConfig(); @@ -61,7 +65,7 @@ const get = status: 'ko', message: 'Could not download url', url: '', - error: "FetchDocumentError: request to " + url + " failed, reason: URL is a private address", + error: "URL seems to be a private address and has been blocked" }); return res; }