From c662b5ef7ca24dcdeeafc41593681b61801420e2 Mon Sep 17 00:00:00 2001 From: quidl <43738052+quidl@users.noreply.github.com> Date: Sat, 5 Oct 2024 09:48:18 +0700 Subject: [PATCH 1/2] Add exclude proxy properties --- Dockerfile | 1 + cli.js | 4 ++++ docker/entrypoint.sh | 9 ++++++++- docker/scripts/agent.sh | 1 + package-lock.json | 21 +++++++++++++++++++-- package.json | 1 + src/core/api/http.js | 25 ++++++++++++++++++++++++- src/core/api/proxy.js | 25 +++++++++++++++++-------- 8 files changed, 75 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index da06cf82..629ea339 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ ENV SERVER_URL='https://testops.katalon.io' ENV KATALON_API_KEY='' ENV ORGANIZATION_ID='' ENV PROXY='' +ENV PROXY_EXCLUDE_LIST='' ENV LOG_LEVEL='INFO' ENV XVFB_RUN='' ENV X11_DISPLAY='' diff --git a/cli.js b/cli.js index 42120edf..21b4e9e9 100644 --- a/cli.js +++ b/cli.js @@ -30,6 +30,7 @@ program .option('-a, --agent-name ', 'Agent name') .option('-c, --config ', 'Configuration file path') .option('-x, --proxy ', 'HTTTP/HTTPS Proxy') + .option('--proxy-exclude-list ', 'Proxy excluded URLs') .option('--log-level ', 'Log level (ALL < TRACE < DEBUG < INFO < WARN < ERROR < FATAL < MARK < OFF)') .option('--xvfb-run ', 'xvfb-run options') .option('--x11-display ', 'x11 DISPLAY environment variable') @@ -44,6 +45,7 @@ program agentName: command.agentName, configPath: command.config, proxy: command.proxy, + proxyExcludeList: command.proxyExcludeList, logLevel: command.logLevel, xvfbRun: command.xvfbRun, x11Display: command.x11Display, @@ -62,6 +64,7 @@ program .option('-a, --agent-name ', 'Agent name') .option('-c, --config ', 'Configuration file path') .option('-x, --proxy ', 'HTTTP/HTTPS Proxy') + .option('--proxy-exclude-list ', 'Proxy excluded URLs') .option('--ci', 'CI mode') .action((command) => { const options = { @@ -72,6 +75,7 @@ program agentName: command.agentName, configPath: command.config, proxy: command.proxy, + proxyExcludeList: command.proxyExcludeList, }; if (process.platform === 'win32') { readline diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ea2f7da8..784cb491 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -4,9 +4,16 @@ set -xe echo "Entrypoint" -if [ "$AUTO_UPGRADE_ENVIRONMENT" = true ]; then +if [[ "$AUTO_UPGRADE_ENVIRONMENT" = true ]]; then /katalon/scripts/upgrade_environment.sh || true fi + +echo "Setup proxy" +#[ -f /etc/machine-id ] || echo $RANDOM | md5sum | fold -w 32 | head -n 1 > /etc/machine-id +if [[ -n "${PROXY}" ]]; then + git config --global http.proxy ${PROXY} +fi + if [ -z "$KATALON_USER_ID" ]; then exec "$@" else diff --git a/docker/scripts/agent.sh b/docker/scripts/agent.sh index ff33cc13..30282607 100755 --- a/docker/scripts/agent.sh +++ b/docker/scripts/agent.sh @@ -15,6 +15,7 @@ cd $KATALON_AGENT_DIR --organizationid "$ORGANIZATION_ID" \ --agent-name "$AGENT_NAME" \ --proxy "$PROXY" \ + --proxy-exclude-list "$PROXY_EXCLUDE_LIST" \ --log-level "$LOG_LEVEL" \ --xvfb-run "$XVFB_RUN" \ --x11-display "$X11_DISPLAY" \ diff --git a/package-lock.json b/package-lock.json index 94e1114a..fba98565 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "katalon-agent", - "version": "v2.3.0", + "version": "v2.3.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "katalon-agent", - "version": "v2.3.0", + "version": "v2.3.4", "license": "ISC", "dependencies": { "axios": "^1.6.8", @@ -22,6 +22,7 @@ "ip": "^2.0.1", "lodash": "^4.17.21", "log4js": "^6.9.1", + "match-url-wildcard": "0.0.5", "moment": "^2.30.1", "progress": "^2.0.3", "properties": "^1.2.1", @@ -7634,6 +7635,22 @@ "tmpl": "1.0.5" } }, + "node_modules/match-url-wildcard": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/match-url-wildcard/-/match-url-wildcard-0.0.5.tgz", + "integrity": "sha512-2H9OwYxMOjrwE31am07hjtKi9P1HEHfc+FkGbhUOK1XSxICK+pMyhLbmXh3bNyc/5XVsJ7uXUvRmM6lwwRzziQ==", + "dependencies": { + "escape-string-regexp": "^1.0.5" + } + }, + "node_modules/match-url-wildcard/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/meow": { "version": "12.1.1", "resolved": "https://registry.npmjs.org/meow/-/meow-12.1.1.tgz", diff --git a/package.json b/package.json index 66978a3e..ae469980 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "ini": "^4.1.2", "ip": "^2.0.1", "lodash": "^4.17.21", + "match-url-wildcard": "0.0.5", "log4js": "^6.9.1", "moment": "^2.30.1", "progress": "^2.0.3", diff --git a/src/core/api/http.js b/src/core/api/http.js index a676dcb9..56408e2e 100644 --- a/src/core/api/http.js +++ b/src/core/api/http.js @@ -2,9 +2,11 @@ const axios = require('axios').default; const fs = require('fs'); const path = require('path'); const ProgressBar = require('progress'); +// const wildcard = require('wildcard'); const { FILTERED_ERROR_CODE } = require('./constants'); const logger = require('../../config/logger'); const { getProxy, getDefaultHttpsAgent } = require('./proxy'); +// const config = require('../config'); const PROGRESS_RENDER_THROTTLE = 5000; @@ -43,6 +45,27 @@ axios.interceptors.response.use( }, ); +// axios.interceptors.request.use((config1) => { +// const { proxy, proxyExcludedUrls } = config; +// const httpAgent = new HttpProxyAgent(proxy, { rejectUnauthorized: false, keepAlive: true }); +// const httpsAgent = new HttpsProxyAgent(proxy, { rejectUnauthorized: false, keepAlive: true }); +// const { url } = config1; +// if (proxyExcludedUrls) { +// const excludedUrls = proxyExcludedUrls.split(','); +// const isExcluded = excludedUrls.some((excludedUrl) => wildcard(excludedUrl, url)); +// if (isExcluded) { +// return config1; +// } +// } +// // Set the proxy agents for non-excluded URLs +// if (url.startsWith('http://')) { +// config1.httpAgent = httpAgent; +// } else if (url.startsWith('https://')) { +// config1.httpsAgent = httpsAgent; +// } +// return config1; +// }); + module.exports = { get(urlParam, headers) { return this.request('get', urlParam, null, headers); @@ -127,7 +150,7 @@ module.exports = { params: urlParam.params, data, headers, - proxy: getProxy(), + proxy: getProxy(urlParam.url), ...overrideOpts, httpsAgent: getDefaultHttpsAgent(), }); diff --git a/src/core/api/proxy.js b/src/core/api/proxy.js index c2241afc..6f43a73c 100644 --- a/src/core/api/proxy.js +++ b/src/core/api/proxy.js @@ -1,11 +1,24 @@ const https = require('https'); +const matchUrl = require('match-url-wildcard'); const config = require('../config'); const logger = require('../../config/logger'); -function getProxy() { - const { proxy } = config; +const agent = new https.Agent({ + rejectUnauthorized: false, + keepAlive: true, +}); + +function getProxy(url) { + const { proxy, proxyExcludeList } = config; if (!proxy) { - return null; + return false; + } + if (proxyExcludeList) { + const excludedUrls = proxyExcludeList.split(','); + const isExcluded = excludedUrls.some((excludedUrl) => matchUrl(url, excludedUrl)); + if (isExcluded) { + return false; + } } try { const { protocol, hostname: host, port, username, password } = new URL(proxy); @@ -25,7 +38,7 @@ function getProxy() { return proxyInfo; } catch (e) { logger.error('Unable to setup proxy:', e); - return null; + return false; } } @@ -37,10 +50,6 @@ function getIgnoreSsl() { } function getDefaultHttpsAgent() { - const agent = new https.Agent({ - rejectUnauthorized: false, - keepAlive: true, - }); return agent; } From d06f07c83e40f9055dd73c228f6f5ce8913730e1 Mon Sep 17 00:00:00 2001 From: Qui Le <43738052+quidl@users.noreply.github.com> Date: Sat, 5 Oct 2024 10:00:14 +0700 Subject: [PATCH 2/2] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ae469980..1ad12079 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "katalon-agent", - "version": "v2.3.4", + "version": "v2.4.0", "description": "", "main": "cli.js", "scripts": {