From e1f6a4578ea37fa93b65655afbfae77125538b96 Mon Sep 17 00:00:00 2001 From: MiracleUFO Date: Mon, 25 Sep 2023 21:59:47 +0100 Subject: [PATCH] chore: rate limiting fix on development environment --- .env.development | 3 + .gitignore | 117 ++++++------------------------------ README.md | 9 ++- package-lock.json | 24 ++++++++ package.json | 3 +- src/constants/index.ts | 13 ++++ src/utils/chunkRequest.ts | 2 - src/utils/getTranslation.ts | 11 +++- 8 files changed, 77 insertions(+), 105 deletions(-) create mode 100644 .env.development diff --git a/.env.development b/.env.development new file mode 100644 index 0000000..3808cf5 --- /dev/null +++ b/.env.development @@ -0,0 +1,3 @@ +NODE_ENV='development' + +TRANSLATE_API_PROXY= \ No newline at end of file diff --git a/.gitignore b/.gitignore index dd16f67..27812a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,51 +1,22 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. -# Bower dependency directory (https://bower.io/) -bower_components +# dependencies +/node_modules +/.pnp +.pnp.js -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ +# dotenv environment variables file +.env.local +.env.development.local +.env.test.local +.env.production.local -# TypeScript v1 declaration files -typings/ +# testing +/coverage -# TypeScript cache -*.tsbuildinfo +# production +/build +/dist # Optional npm cache directory .npm @@ -53,57 +24,9 @@ typings/ # Optional eslint cache .eslintcache -# Microbundle cache -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variables file -.env -.env.test - -# parcel-bundler cache (https://parceljs.org/) -.cache - -# Next.js build output -.next - -# Nuxt.js build / generate output -.nuxt -dist - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and *not* Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# TernJS port file -.tern-port - -.dccache +# misc .DS_Store -lib -*.tgz + +npm-debug.log* +yarn-debug.log* +yarn-error.log* \ No newline at end of file diff --git a/README.md b/README.md index 20a0335..2bb72c7 100644 --- a/README.md +++ b/README.md @@ -111,11 +111,14 @@ See [Usage](#to-get-translation-of-text-directly) - [`to`](#props) *optional* -## SPECIAL CASES +## Special Cases - [`from`](#props) and [`to`](#props) being the same will return original text (determined by google translation API.) - [`from`](#props) and [`to`](#props) being empty strings will be extrapolated from 'en' and *user's current browser langauge setting* respectively. - `text` is not in `from` language and google translate API cannot detect language automatically will return the original text. -## DEVELOPER TESTING -- Some tests in `src/tests` may fail because google translate API sometimes returns synonyms when a string is translated multiple times. +## Developer Testing +- [Install node-modules](#install) +- `npm run test` or `yarn run test` +- Some tests in `src/tests` may fail because google translate API might return synonyms when a string is translated multiple times. +- If `TooManyRequestsError` or Error Code `429` is encountered, update `env` variable `TRANSLATE_API_PROXY` with a correct [proxy](https://free-proxy-list.net/) (with yes in Google column.) This error is due to Google Translate APIs rate limiting per IP address (this limit seems variable, see [discussion](https://github.com/vitalets/google-translate-api/issues/107#issuecomment-1302220214)) \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 08439bb..35a0ad8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@testing-library/react": "^14.0.0", "@testing-library/user-event": "^14.5.0", "@vitalets/google-translate-api": "^9.0.0", + "env-cmd": "^10.1.0", "https-proxy-agent": "^7.0.2", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -9850,6 +9851,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/env-cmd": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/env-cmd/-/env-cmd-10.1.0.tgz", + "integrity": "sha512-mMdWTT9XKN7yNth/6N6g2GuKuJTsKMDHlQFUDacb/heQRRWOTIZ42t1rMHnQu4jYxU1ajdTeJM+9eEETlqToMA==", + "dependencies": { + "commander": "^4.0.0", + "cross-spawn": "^7.0.0" + }, + "bin": { + "env-cmd": "bin/env-cmd.js" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/env-cmd/node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "engines": { + "node": ">= 6" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", diff --git a/package.json b/package.json index f91be08..26c2c9d 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "build:esm": "tsc -p ./configs/tsconfig.esm.json && mv build/esm/index.js build/esm/index.mjs", "build:cjs": "tsc -p ./configs/tsconfig.cjs.json", "semantic-release": "semantic-release", - "test": "react-scripts test", + "test": "env-cmd -f .env.local react-scripts test", "prepack": "npm run build", "lint": "eslint src --ext .js,.jsx,.tsx,.ts --fix" }, @@ -97,6 +97,7 @@ "@testing-library/react": "^14.0.0", "@testing-library/user-event": "^14.5.0", "@vitalets/google-translate-api": "^9.0.0", + "env-cmd": "^10.1.0", "https-proxy-agent": "^7.0.2", "react": "^18.2.0", "react-dom": "^18.2.0", diff --git a/src/constants/index.ts b/src/constants/index.ts index ef8b1a9..04d7a01 100644 --- a/src/constants/index.ts +++ b/src/constants/index.ts @@ -1,5 +1,14 @@ import language from '../types/language'; +const { NODE_ENV, TRANSLATE_API_PROXY } = process.env; + +// NODE ENVIRONMENT +const NODE_DEVELOPMENT = 'development'; +const NODE_TEST = 'test'; +const IS_DEVELOPMENT_OR_TEST = NODE_ENV && [NODE_DEVELOPMENT, NODE_TEST].includes(NODE_ENV); + +const PROXY = TRANSLATE_API_PROXY; + const DEFAULT_PROPS = { from: 'en', to: 'auto', @@ -22,6 +31,10 @@ const DEFAULT_BROWSER_LANGUAGE : language = window?.navigator?.language.startsWi const TRANSLATION_NOT_FOUND_MESSAGE = 'react-g-translator: Err 404: No translation found. Check `to` & `from` props.'; export { + NODE_DEVELOPMENT, + NODE_TEST, + IS_DEVELOPMENT_OR_TEST, + PROXY, DEFAULT_PROPS, DEFAULT_QUERY_OPTIONS, DEFAULT_LANGUAGE_FROM, diff --git a/src/utils/chunkRequest.ts b/src/utils/chunkRequest.ts index 3546490..f9347b2 100644 --- a/src/utils/chunkRequest.ts +++ b/src/utils/chunkRequest.ts @@ -4,7 +4,6 @@ const chunkRequest = async ( chunkSize: number = 0, ) => { const promises = []; - const chunks = []; for (let i = 0; i < text.length;) { const endIndex = i + chunkSize; @@ -13,7 +12,6 @@ const chunkRequest = async ( // to aid better translation const endIndexFullSentence = (Math.min(endIndex, text.lastIndexOf('.', endIndex - 1) + 1) || endIndex); const chunk = (text.slice(i, endIndexFullSentence)); - chunks.push(chunk); promises.push(translationReqFunc(chunk)); // afterthough: next index is the last translated full sentence endIndex diff --git a/src/utils/getTranslation.ts b/src/utils/getTranslation.ts index 3f40ce0..d73e6a0 100644 --- a/src/utils/getTranslation.ts +++ b/src/utils/getTranslation.ts @@ -1,7 +1,10 @@ import { translate } from '@vitalets/google-translate-api'; +import { HttpsProxyAgent } from 'https-proxy-agent'; import chunkRequest from './chunkRequest'; import enableCors from './enableCorsAndLimitRate'; + +import { IS_DEVELOPMENT_OR_TEST, PROXY } from '../constants'; import language from '../types/language'; const getTranslation = async ( @@ -9,12 +12,16 @@ const getTranslation = async ( from?: language, to?: language, ) : Promise => { + // for development or testing + // in case `TooManyRequestsError` or Error Code `429` + const fetchOptions = IS_DEVELOPMENT_OR_TEST && PROXY && { agent: new HttpsProxyAgent(PROXY) }; + const translateRequest = async (chunk: string | string[]) => { - // CORS policy overriding (if any) + // CORS policy overriding enableCors(1); // translating happens here. ✨ bing! ✨ - const translation = await translate(chunk as string, { from, to }); + const translation = await translate(chunk as string, { from, to, fetchOptions }); return translation.text; };