Merge pull request #267 from katalon-studio/add-prop

Add exclude proxy properties

Dockerfile

@@ -45,6 +45,7 @@ ENV SERVER_URL='https://testops.katalon.io'
 ENV KATALON_API_KEY=''
 ENV ORGANIZATION_ID=''
 ENV PROXY=''
+ENV PROXY_EXCLUDE_LIST=''
 ENV LOG_LEVEL='INFO'
 ENV XVFB_RUN=''
 ENV X11_DISPLAY=''

cli.js

@@ -30,6 +30,7 @@ program
   .option('-a, --agent-name <value>', 'Agent name')
   .option('-c, --config <value>', 'Configuration file path')
   .option('-x, --proxy <value>', 'HTTTP/HTTPS Proxy')
+  .option('--proxy-exclude-list <value>', 'Proxy excluded URLs')
   .option('--log-level <value>', 'Log level (ALL < TRACE < DEBUG < INFO < WARN < ERROR < FATAL < MARK < OFF)')
   .option('--xvfb-run <value>', 'xvfb-run options')
   .option('--x11-display <value>', 'x11 DISPLAY environment variable')
@@ -44,6 +45,7 @@ program
       agentName: command.agentName,
       configPath: command.config,
       proxy: command.proxy,
+      proxyExcludeList: command.proxyExcludeList,
       logLevel: command.logLevel,
       xvfbRun: command.xvfbRun,
       x11Display: command.x11Display,
@@ -62,6 +64,7 @@ program
   .option('-a, --agent-name <value>', 'Agent name')
   .option('-c, --config <value>', 'Configuration file path')
   .option('-x, --proxy <value>', 'HTTTP/HTTPS Proxy')
+  .option('--proxy-exclude-list <value>', 'Proxy excluded URLs')
   .option('--ci', 'CI mode')
   .action((command) => {
     const options = {
@@ -72,6 +75,7 @@ program
       agentName: command.agentName,
       configPath: command.config,
       proxy: command.proxy,
+      proxyExcludeList: command.proxyExcludeList,
     };
     if (process.platform === 'win32') {
       readline

docker/entrypoint.sh

@@ -4,9 +4,16 @@ set -xe
 
 echo "Entrypoint"
 
-if [ "$AUTO_UPGRADE_ENVIRONMENT" = true ]; then
+if [[ "$AUTO_UPGRADE_ENVIRONMENT" = true ]]; then
     /katalon/scripts/upgrade_environment.sh || true
 fi
+
+echo "Setup proxy"
+#[ -f /etc/machine-id ] || echo $RANDOM | md5sum | fold -w 32 | head -n 1 > /etc/machine-id
+if [[ -n "${PROXY}" ]]; then
+    git config --global http.proxy ${PROXY}
+fi
+
 if [ -z "$KATALON_USER_ID" ]; then
     exec "$@"
 else

docker/scripts/agent.sh

@@ -15,6 +15,7 @@ cd $KATALON_AGENT_DIR
     --organizationid "$ORGANIZATION_ID" \
     --agent-name "$AGENT_NAME" \
     --proxy "$PROXY" \
+    --proxy-exclude-list "$PROXY_EXCLUDE_LIST" \
     --log-level "$LOG_LEVEL" \
     --xvfb-run "$XVFB_RUN" \
     --x11-display "$X11_DISPLAY" \

package.json

@@ -39,6 +39,7 @@
     "ini": "^4.1.2",
     "ip": "^2.0.1",
     "lodash": "^4.17.21",
+    "match-url-wildcard": "0.0.5",
     "log4js": "^6.9.1",
     "moment": "^2.30.1",
     "progress": "^2.0.3",

src/core/api/http.js

@@ -2,9 +2,11 @@ const axios = require('axios').default;
 const fs = require('fs');
 const path = require('path');
 const ProgressBar = require('progress');
+// const wildcard = require('wildcard');
 const { FILTERED_ERROR_CODE } = require('./constants');
 const logger = require('../../config/logger');
 const { getProxy, getDefaultHttpsAgent } = require('./proxy');
+// const config = require('../config');
 
 const PROGRESS_RENDER_THROTTLE = 5000;
 
@@ -43,6 +45,27 @@ axios.interceptors.response.use(
   },
 );
 
+// axios.interceptors.request.use((config1) => {
+//   const { proxy, proxyExcludedUrls } = config;
+//   const httpAgent = new HttpProxyAgent(proxy, { rejectUnauthorized: false, keepAlive: true });
+//   const httpsAgent = new HttpsProxyAgent(proxy, { rejectUnauthorized: false, keepAlive: true });
+//   const { url } = config1;
+//   if (proxyExcludedUrls) {
+//     const excludedUrls = proxyExcludedUrls.split(',');
+//     const isExcluded = excludedUrls.some((excludedUrl) => wildcard(excludedUrl, url));
+//     if (isExcluded) {
+//       return config1;
+//     }
+//   }
+//   // Set the proxy agents for non-excluded URLs
+//   if (url.startsWith('http://')) {
+//     config1.httpAgent = httpAgent;
+//   } else if (url.startsWith('https://')) {
+//     config1.httpsAgent = httpsAgent;
+//   }
+//   return config1;
+// });
+
 module.exports = {
   get(urlParam, headers) {
     return this.request('get', urlParam, null, headers);
@@ -127,7 +150,7 @@ module.exports = {
       params: urlParam.params,
       data,
       headers,
-      proxy: getProxy(),
+      proxy: getProxy(urlParam.url),
       ...overrideOpts,
       httpsAgent: getDefaultHttpsAgent(),
     });

src/core/api/proxy.js

@@ -1,11 +1,24 @@
 const https = require('https');
+const matchUrl = require('match-url-wildcard');
 const config = require('../config');
 const logger = require('../../config/logger');
 
-function getProxy() {
-  const { proxy } = config;
+const agent = new https.Agent({
+  rejectUnauthorized: false,
+  keepAlive: true,
+});
+
+function getProxy(url) {
+  const { proxy, proxyExcludeList } = config;
   if (!proxy) {
-    return null;
+    return false;
+  }
+  if (proxyExcludeList) {
+    const excludedUrls = proxyExcludeList.split(',');
+    const isExcluded = excludedUrls.some((excludedUrl) => matchUrl(url, excludedUrl));
+    if (isExcluded) {
+      return false;
+    }
   }
   try {
     const { protocol, hostname: host, port, username, password } = new URL(proxy);
@@ -25,7 +38,7 @@ function getProxy() {
     return proxyInfo;
   } catch (e) {
     logger.error('Unable to setup proxy:', e);
-    return null;
+    return false;
   }
 }
 
@@ -37,10 +50,6 @@ function getIgnoreSsl() {
 }
 
 function getDefaultHttpsAgent() {
-  const agent = new https.Agent({
-    rejectUnauthorized: false,
-    keepAlive: true,
-  });
   return agent;
 }