From bbd3787120b1dd5f4f436be8f3d2fe0d651efba3 Mon Sep 17 00:00:00 2001 From: Lachie Date: Thu, 12 Sep 2024 00:13:43 +1000 Subject: [PATCH] feat(deezer-sdk): simple redis implementation --- .changeset/polite-clocks-pull.md | 5 ++ .changeset/short-walls-appear.md | 5 ++ deemix/src/plugins/spotify.ts | 2 +- deezer-sdk/package.json | 1 + deezer-sdk/src/api.ts | 31 ++++++++++-- docker-compose.yml | 12 +++++ pnpm-lock.yaml | 82 ++++++++++++++++++++++++++++++++ webui/src/server/main.ts | 3 ++ 8 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 .changeset/polite-clocks-pull.md create mode 100644 .changeset/short-walls-appear.md create mode 100644 docker-compose.yml diff --git a/.changeset/polite-clocks-pull.md b/.changeset/polite-clocks-pull.md new file mode 100644 index 00000000..1f062ad3 --- /dev/null +++ b/.changeset/polite-clocks-pull.md @@ -0,0 +1,5 @@ +--- +"deezer-sdk": minor +--- + +Use redis to cache Deezer API calls diff --git a/.changeset/short-walls-appear.md b/.changeset/short-walls-appear.md new file mode 100644 index 00000000..094b7fe1 --- /dev/null +++ b/.changeset/short-walls-appear.md @@ -0,0 +1,5 @@ +--- +"deemix-webui": minor +--- + +Significantly faster Deezer API calls by caching using Redis diff --git a/deemix/src/plugins/spotify.ts b/deemix/src/plugins/spotify.ts index 562958aa..ff60f387 100644 --- a/deemix/src/plugins/spotify.ts +++ b/deemix/src/plugins/spotify.ts @@ -341,7 +341,7 @@ export default class SpotifyPlugin extends BasePlugin { } callback(); - }, settings.queueConcurrency * 20); + }, settings.queueConcurrency); downloadObject.conversion_data.forEach((track, pos) => { q.push({ track, pos }, () => {}); diff --git a/deezer-sdk/package.json b/deezer-sdk/package.json index 4f3cd047..8eec8dcb 100644 --- a/deezer-sdk/package.json +++ b/deezer-sdk/package.json @@ -21,6 +21,7 @@ "license": "GPL-3.0-or-later", "dependencies": { "got": "14.4.2", + "redis": "^4.7.0", "tough-cookie": "^4.0.0", "zod": "^3.23.8" }, diff --git a/deezer-sdk/src/api.ts b/deezer-sdk/src/api.ts index 58ef4ac3..76277748 100644 --- a/deezer-sdk/src/api.ts +++ b/deezer-sdk/src/api.ts @@ -1,4 +1,5 @@ import got from "got"; +import { createClient, type RedisClientType } from "redis"; import { CookieJar } from "tough-cookie"; import { APIError, @@ -11,10 +12,8 @@ import { PermissionException, WrongParameterException, } from "./errors.js"; -import { Deezer, type APIOptions } from "./index.js"; -import { albumSchema, type DeezerAlbum } from "./schema/album-schema.js"; +import { type APIOptions } from "./index.js"; import { trackSchema, type DeezerTrack } from "./schema/track-schema.js"; -import { artistSchema } from "./schema/contributor-schema.js"; // Possible values for order parameter in search export const SearchOrder = { @@ -204,16 +203,39 @@ export class API { http_headers: { "User-Agent": string }; cookie_jar: CookieJar; access_token: string | null; + redisClient?: RedisClientType; constructor(cookie_jar: CookieJar, headers: { "User-Agent": string }) { this.http_headers = headers; this.cookie_jar = cookie_jar; this.access_token = null; + + const url = process.env.REDIS_URL || "redis://localhost:6379"; + if (url && process.env.REDIS_PASSWORD) { + this.redisClient = createClient({ + url: url, + password: process.env.REDIS_PASSWORD, + }); + + this.redisClient.on("error", (err) => { + console.error("Redis error", err); + }); + + this.redisClient.connect(); + } } async call(endpoint: string, args: APIArgs = {}): Promise { if (this.access_token) args["access_token"] = this.access_token; + if (this.redisClient?.isReady) { + const cachedResponse = await this.redisClient.get(endpoint); + + if (cachedResponse) { + return JSON.parse(cachedResponse); + } + } + let response; try { response = await got @@ -291,6 +313,9 @@ export class API { throw new APIError(response.error); } + if (this.redisClient?.isReady) + this.redisClient.set(endpoint, JSON.stringify(response)); + return response; } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..fa5c5693 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,12 @@ +services: + cache: + image: redis:7.4-alpine + restart: always + ports: + - "6379:6379" + command: redis-server --save 20 1 --loglevel warning --requirepass eYVX7EwVmmxKPCDmwMtyKVge8oLd2t81 + volumes: + - cache:/data +volumes: + cache: + driver: local diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7678d835..4ee4d70d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -117,6 +117,9 @@ importers: got: specifier: 14.4.2 version: 14.4.2 + redis: + specifier: ^4.7.0 + version: 4.7.0 tough-cookie: specifier: ^4.0.0 version: 4.1.4 @@ -1265,6 +1268,35 @@ packages: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} + '@redis/bloom@1.2.0': + resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/client@1.6.0': + resolution: {integrity: sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==} + engines: {node: '>=14'} + + '@redis/graph@1.1.1': + resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/json@1.0.7': + resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/search@1.2.0': + resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/time-series@1.1.0': + resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==} + peerDependencies: + '@redis/client': ^1.0.0 + '@rollup/rollup-android-arm-eabi@4.21.2': resolution: {integrity: sha512-fSuPrt0ZO8uXeS+xP3b+yYTCBUd05MoSp2N/MFOgjhhUhMmchXlpTQrTpI8T+YAwAQuK7MafsCOxW7VrPMrJcg==} cpu: [arm] @@ -2091,6 +2123,10 @@ packages: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -2942,6 +2978,10 @@ packages: generate-object-property@1.2.0: resolution: {integrity: sha512-TuOwZWgJ2VAMEGJvAyPWvpqxSANF0LDpmyHauMjFYzaACvn+QTT/AZomvPCzVBV7yDN3OmwHQ5OvHaeLKre3JQ==} + generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -4709,6 +4749,9 @@ packages: resolution: {integrity: sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==} engines: {node: '>= 10.13.0'} + redis@4.7.0: + resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==} + regenerator-runtime@0.14.1: resolution: {integrity: sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==} @@ -7011,6 +7054,32 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true + '@redis/bloom@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/client@1.6.0': + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + + '@redis/graph@1.1.1(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/json@1.0.7(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/search@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/time-series@1.1.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + '@rollup/rollup-android-arm-eabi@4.21.2': optional: true @@ -7986,6 +8055,8 @@ snapshots: clone@1.0.4: {} + cluster-key-slot@1.1.2: {} + co@4.6.0: {} collect-v8-coverage@1.0.2: {} @@ -9003,6 +9074,8 @@ snapshots: is-property: 1.0.2 optional: true + generic-pool@3.9.0: {} + gensync@1.0.0-beta.2: {} genversion@3.2.0: @@ -10830,6 +10903,15 @@ snapshots: dependencies: resolve: 1.22.8 + redis@4.7.0: + dependencies: + '@redis/bloom': 1.2.0(@redis/client@1.6.0) + '@redis/client': 1.6.0 + '@redis/graph': 1.1.1(@redis/client@1.6.0) + '@redis/json': 1.0.7(@redis/client@1.6.0) + '@redis/search': 1.2.0(@redis/client@1.6.0) + '@redis/time-series': 1.1.0(@redis/client@1.6.0) + regenerator-runtime@0.14.1: {} repeat-string@1.6.1: diff --git a/webui/src/server/main.ts b/webui/src/server/main.ts index ee5b23fd..9a07d1a3 100644 --- a/webui/src/server/main.ts +++ b/webui/src/server/main.ts @@ -20,6 +20,9 @@ import indexRouter from "./routes/index.js"; import type { Arguments, Listener } from "./types.js"; import { registerWebsocket } from "./websocket/index.js"; +import dotenv from "dotenv"; +dotenv.config({ path: join(import.meta.dirname, ".env") }); + const MemoryStore = memorystore(session); // TODO: Remove type assertion while keeping correct types