From 7faf94cfab97f5d7be3877f5aba7e837344efcf6 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 3 Dec 2024 15:58:08 -0500 Subject: [PATCH 01/11] feat(ytm): Re-implement auth with cookie and oauth Youtube TV seems no longer have scope for reading history or account details? Cookie may work and custom oauth seems most stable based on reporting from LuanRT/YouTube.js#803 --- package-lock.json | 200 +++++++++++++++++- package.json | 3 +- .../infrastructure/config/source/ytmusic.ts | 19 ++ src/backend/server/auth.ts | 18 +- src/backend/sources/YTMusicSource.ts | 105 +++++++-- 5 files changed, 321 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index 12ed8880..db865546 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,7 @@ "fixed-size-list": "^0.3.0", "formidable": "^3.5", "glob": "^11.0.0", + "google-auth-library": "^9.15.0", "gotify": "^1.1.0", "iso-websocket": "^0.3.0", "iti": "^0.6.0", @@ -81,7 +82,7 @@ "vite-express": "^0.16.0", "vlc-client": "^1.1.1", "xml2js": "0.6.1", - "youtubei.js": "^10.5.0" + "youtubei.js": "^11.0.1" }, "devDependencies": { "@dbus-types/notifications": "^0.0.5", @@ -3241,6 +3242,17 @@ "node": ">= 10.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -3587,6 +3599,14 @@ "pnpm": ">=6" } }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -3714,6 +3734,11 @@ "ieee754": "^1.2.1" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -4599,6 +4624,14 @@ "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -5096,6 +5129,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/eyes": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", @@ -5493,6 +5531,45 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -5671,6 +5748,22 @@ "node": ">=8" } }, + "node_modules/google-auth-library": { + "version": "9.15.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.0.tgz", + "integrity": "sha512-7ccSEJFDFO7exFbO6NRyC+xH8/mZ1GZGG2xxx9iHxZWcjUjJpjWxIMw3cofAKcueZ6DATiukmmprD7yavQHOyQ==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -5738,6 +5831,18 @@ "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/has-bigints": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", @@ -5909,6 +6014,18 @@ "node": ">= 0.8" } }, + "node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -6298,8 +6415,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, - "peer": true, "engines": { "node": ">=8" }, @@ -6695,9 +6810,9 @@ } }, "node_modules/jintr": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/jintr/-/jintr-2.1.1.tgz", - "integrity": "sha512-89cwX4ouogeDGOBsEVsVYsnWWvWjchmwXBB4kiBhmjOKw19FiOKhNhMhpxhTlK2ctl7DS+d/ethfmuBpzoNNgA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/jintr/-/jintr-3.0.2.tgz", + "integrity": "sha512-5g2EBudeJFOopjAX4exAv5OCCW1DgUISfoioCsm1h9Q9HJ41LmnZ6J52PCsqBlQihsmp0VDuxreAVzM7yk5nFA==", "funding": [ "https://github.com/sponsors/LuanRT" ], @@ -6750,6 +6865,14 @@ "node": ">=6" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -6849,6 +6972,25 @@ "node": "*" } }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -7513,6 +7655,25 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==" }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -10444,6 +10605,11 @@ "node": ">= 4.0.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/ts_lru_map": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/ts_lru_map/-/ts_lru_map-1.0.2.tgz", @@ -11348,6 +11514,20 @@ "phin": "^3.6.1" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -11742,15 +11922,15 @@ } }, "node_modules/youtubei.js": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-10.5.0.tgz", - "integrity": "sha512-iyA+VF28c15tCCKH9ExM2RKC3zYiHzA/eixGlJ3vERANkuI+xYKzAZ4vtOhmyqwrAddu88R/DkzEsmpph5NWjg==", + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-11.0.1.tgz", + "integrity": "sha512-ZsbOd+5XF2Ofi3FrLMfYd+f9g9H8xswlouFhjhOqbwT68dMJtX6CRGsHNj5VTFCR/+L/865x1lnUlllB2dDDTA==", "funding": [ "https://github.com/sponsors/LuanRT" ], "dependencies": { "@bufbuild/protobuf": "^2.0.0", - "jintr": "^2.1.1", + "jintr": "^3.0.2", "tslib": "^2.5.0", "undici": "^5.19.1" } diff --git a/package.json b/package.json index 726d52a2..4d3d2549 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,7 @@ "fixed-size-list": "^0.3.0", "formidable": "^3.5", "glob": "^11.0.0", + "google-auth-library": "^9.15.0", "gotify": "^1.1.0", "iso-websocket": "^0.3.0", "iti": "^0.6.0", @@ -111,7 +112,7 @@ "vite-express": "^0.16.0", "vlc-client": "^1.1.1", "xml2js": "0.6.1", - "youtubei.js": "^10.5.0" + "youtubei.js": "^11.0.1" }, "devDependencies": { "@dbus-types/notifications": "^0.0.5", diff --git a/src/backend/common/infrastructure/config/source/ytmusic.ts b/src/backend/common/infrastructure/config/source/ytmusic.ts index 36802dcb..513c4cc1 100644 --- a/src/backend/common/infrastructure/config/source/ytmusic.ts +++ b/src/backend/common/infrastructure/config/source/ytmusic.ts @@ -1,8 +1,27 @@ import { PollingOptions } from "../common.js"; import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js"; +import { Innertube } from 'youtubei.js'; + +//type InnertubeOptions = Omit[0], 'cookie' | 'cache' | 'fetch'>; export interface YTMusicData extends CommonSourceData, PollingOptions { + /** + * The cookie retrieved from the Request Headers of music.youtube.com after logging in. + * + * See https://ytmusicapi.readthedocs.io/en/stable/setup/browser.html#copy-authentication-headers for how to retrieve this value. + * + * @examples ["VISITOR_INFO1_LIVE=jMp2xA1Xz2_PbVc; __Secure-3PAPISID=3AxsXpy0M/AkISpjek; ..."] + * */ + cookie?: string + + clientId?: string + + clientSecret?: string + + redirectUri?: string } +//export type YTMusicData = YTMusicDataCommon & InnertubeOptions; + export interface YTMusicSourceConfig extends CommonSourceConfig { data?: YTMusicData options?: CommonSourceOptions & { diff --git a/src/backend/server/auth.ts b/src/backend/server/auth.ts index dc41fa0c..8f4605bf 100644 --- a/src/backend/server/auth.ts +++ b/src/backend/server/auth.ts @@ -8,6 +8,8 @@ import LastfmSource from "../sources/LastfmSource.js"; import ScrobbleSources from "../sources/ScrobbleSources.js"; import SpotifySource from "../sources/SpotifySource.js"; import YTMusicSource from "../sources/YTMusicSource.js"; +import { sortAndDeduplicateDiagnostics } from "typescript"; +import { source } from "common-tags"; export const setupAuthRoutes = (app: ExpressWithAsync, logger: Logger, sourceMiddle: ExpressHandler, clientMiddle: ExpressHandler, scrobbleSources: ScrobbleSources, scrobbleClients: ScrobbleClients) => { app.use('/api/client/auth', clientMiddle); @@ -65,7 +67,8 @@ export const setupAuthRoutes = (app: ExpressWithAsync, logger: Logger, sourceMid } const { query: { - state + state, + name } = {} } = req; if (req.url.includes('lastfm')) { @@ -86,6 +89,19 @@ export const setupAuthRoutes = (app: ExpressWithAsync, logger: Logger, sourceMid } catch (e) { return res.send(e.message); } + } else if(req.url.includes('ytmusic')) { + const entity: YTMusicSource | undefined = scrobbleSources.getByName(name) as (YTMusicSource | undefined); + if(entity === undefined) { + logger.error(`No YTMUsic source with name ${state} was found`); + } + const result = await entity.handleAuthCodeCallback(req.query); + let responseContent = 'OK'; + if(result === true) { + entity.poll(); + } else { + responseContent = result; + } + return res.send(responseContent); } else { // TODO right now all sources requiring source interaction are covered by logic branches (deezer above and spotify here) // but eventually should update all source callbacks to url specific URLS to avoid ambiguity... diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index e137c42a..6a2c47ba 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -3,9 +3,10 @@ import EventEmitter from "events"; import { PlayObject } from "../../core/Atomic.js"; import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; import { YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js"; -import { Innertube, UniversalCache, Parser, YTNodes, ApiResponse, IBrowseResponse } from 'youtubei.js'; +import { Innertube, UniversalCache, Parser, YTNodes, ApiResponse, IBrowseResponse, Log } from 'youtubei.js'; +import { OAuth2Client } from 'google-auth-library'; import {resolve} from 'path'; -import { sleep } from "../utils.js"; +import { joinedUrl, sleep } from "../utils.js"; import { getPlaysDiff, humanReadableDiff, @@ -14,7 +15,6 @@ import { playsAreSortConsistent } from "../utils/PlayComparisonUtils.js"; import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; -import { ListDiff } from "@donedeal0/superdiff"; export const ytiHistoryResponseToListItems = (res: ApiResponse): YTNodes.MusicResponsiveListItem[] => { const page = Parser.parseResponse(res.data); @@ -61,6 +61,8 @@ export default class YTMusicSource extends AbstractSource { requiresAuth = true; requiresAuthInteraction = true; + cookieBased: boolean = false; + declare config: YTMusicSourceConfig recentlyPlayed: PlayObject[] = []; @@ -68,12 +70,15 @@ export default class YTMusicSource extends AbstractSource { yti: Innertube; userCode?: string; verificationUrl?: string; + oauthClient?: OAuth2Client; workingCredsPath: string; constructor(name: string, config: YTMusicSourceConfig, internal: InternalConfig, emitter: EventEmitter) { super('ytmusic', name, config, internal, emitter); this.canPoll = true; + Log.setLevel(Log.Level.ERROR); + this.cookieBased = this.config.data?.cookie !== undefined; this.supportsUpstreamRecentlyPlayed = true; this.workingCredsPath = resolve(this.configDir, `yti-${this.name}`); } @@ -88,6 +93,7 @@ export default class YTMusicSource extends AbstractSource { protected async doBuildInitData(): Promise { this.yti = await Innertube.create({ + ...this.config.data, cache: new UniversalCache(true, this.workingCredsPath) }); this.yti.session.on('update-credentials', async ({ credentials }) => { @@ -111,10 +117,11 @@ export default class YTMusicSource extends AbstractSource { } else { this.logger.debug('Auth success'); } - await this.yti.session.oauth.cacheCredentials(); this.userCode = undefined; this.verificationUrl = undefined; this.authed = true; + await this.yti.session.oauth.cacheCredentials(); + const f =1; }); return true; } @@ -127,33 +134,107 @@ export default class YTMusicSource extends AbstractSource { } clearCredentials = async () => { - if(this.yti.session.logged_in) { + if(this.yti.session.logged_in && !this.cookieBased) { await this.yti.session.signOut(); } } + async handleAuthCodeCallback(obj: Record): Promise { + if (obj.code === undefined) { + this.logger.error(`Authorization callback did not contain 'code' in URL`); + return false; + } + + const { tokens } = await this.oauthClient.getToken(obj.code as string); + + if (tokens.access_token && tokens.refresh_token && tokens.expiry_date) { + await this.yti.session.signIn({ + access_token: tokens.access_token, + refresh_token: tokens.refresh_token, + expiry_date: new Date(tokens.expiry_date).toISOString(), + client: { + client_id: this.config.data.clientId, + client_secret: this.config.data.clientSecret + } + }); + this.authed = true; + this.verificationUrl = undefined; + this.userCode = undefined; + await this.yti.session.oauth.cacheCredentials(); + Log.setLevel(Log.Level.ERROR); + return true; + } else { + this.logger.error(`Token data did not return all required properties.`); + return tokens; + } + } + doAuthentication = async () => { try { + if (this.cookieBased) { + try { + await this.yti.account.getInfo() + this.authed = true; + } catch (e) { + const info = loggedErrorExtra(e); + if (info !== undefined) { + this.logger.error(info, 'Additional API response details') + } + this.logger.error(new Error('Cookie-based authentication failed. Try recreating cookie or using custom OAuth Client', { cause: e })); + } + } + await Promise.race([ - sleep(300), + sleep(1000), this.yti.session.signIn() ]); - if(this.authed === false && this.userCode !== undefined) { - if(this.userCode !== undefined) { - throw new Error(`Sign in with the code '${this.userCode}' using the authentication link on the dashboard or ${this.verificationUrl}`) + if (this.authed === false) { + + if (this.config.data.clientId !== undefined) { + const redirectUri = this.config.data?.redirectUri ?? joinedUrl(this.localUrl, `ytmusic/callback?name=${this.name}`).toString(); + + this.logger.info(`Using Custom OAuth Client with Redirect URI: ${redirectUri}`); + this.oauthClient = new OAuth2Client({ + clientId: this.config.data.clientId, + clientSecret: this.config.data.clientSecret, + redirectUri + }); + + const authorizationUrl = this.oauthClient.generateAuthUrl({ + access_type: 'offline', + scope: [ + "http://gdata.youtube.com", + "https://www.googleapis.com/auth/youtube", + "https://www.googleapis.com/auth/youtube.force-ssl", + "https://www.googleapis.com/auth/youtube-paid-content", + "https://www.googleapis.com/auth/accounts.reauth", + ], + include_granted_scopes: true, + prompt: 'consent', + }); + + this.verificationUrl = authorizationUrl; + this.userCode = undefined; + throw new Error(`Sign in using ${authorizationUrl}`); } else { - throw new Error('Waited too long for auth response from YTM!'); + if (this.userCode !== undefined) { + this.logger.warn('Logging in with YoutubeTV Oauth will likely NOT provide access to Youtube Music history!! You should try to use either cookies or a custom OAuth Client ID/Secret'); + throw new Error(`Sign in with the code '${this.userCode}' using the authentication link on the dashboard or ${this.verificationUrl}`) + } else { + throw new Error('Waited too long for auth response from YTM!'); + } } } try { await this.yti.account.getInfo() } catch (e) { const info = loggedErrorExtra(e); - if(info !== undefined) { + if (info !== undefined) { this.logger.error(info, 'Additional API response details') } - throw new Error('Credentials exist but API calls are failing. Try re-authenticating?', {cause: e}); + throw new Error('Credentials exist but API calls are failing. Try re-authenticating?', { cause: e }); } + Log.setLevel(Log.Level.ERROR); return true; } catch (e) { throw e; From f74721b36618edb046a6e692c7984a67a61f8482 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 4 Dec 2024 15:04:36 +0000 Subject: [PATCH 02/11] fix(ytmusic): Fix missing api prefix for redirectUri --- src/backend/sources/YTMusicSource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index 6a2c47ba..9d7e13a9 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -191,7 +191,7 @@ export default class YTMusicSource extends AbstractSource { if (this.authed === false) { if (this.config.data.clientId !== undefined) { - const redirectUri = this.config.data?.redirectUri ?? joinedUrl(this.localUrl, `ytmusic/callback?name=${this.name}`).toString(); + const redirectUri = this.config.data?.redirectUri ?? joinedUrl(this.localUrl, `api/ytmusic/callback?name=${this.name}`).toString(); this.logger.info(`Using Custom OAuth Client with Redirect URI: ${redirectUri}`); this.oauthClient = new OAuth2Client({ From 4de28e039010cea8cb2d0d76bed060bfa021c183 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 4 Dec 2024 19:38:54 +0000 Subject: [PATCH 03/11] feat(ytmusic): Add more innertube options to user configuration --- .../infrastructure/config/source/ytmusic.ts | 59 +++++++++++++++++-- src/backend/sources/YTMusicSource.ts | 9 ++- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/backend/common/infrastructure/config/source/ytmusic.ts b/src/backend/common/infrastructure/config/source/ytmusic.ts index 513c4cc1..b0b91cea 100644 --- a/src/backend/common/infrastructure/config/source/ytmusic.ts +++ b/src/backend/common/infrastructure/config/source/ytmusic.ts @@ -1,11 +1,42 @@ import { PollingOptions } from "../common.js"; import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js"; -import { Innertube } from 'youtubei.js'; -//type InnertubeOptions = Omit[0], 'cookie' | 'cache' | 'fetch'>; +export interface InnertubeOptions { + /** + * Proof of Origin token + * + * May be required if YTM starts returning 403 + * + * @see https://github.com/yt-dlp/yt-dlp/wiki/Extractors#po-token-guide + */ + po_token?: string + + /** + * Visitor ID value found in VISITOR_INFO1_LIVE or visitorData cookie + * + * May be required if YTM starts returning 403 + * + * @see https://github.com/yt-dlp/yt-dlp/wiki/Extractors#po-token-guide + */ + visitor_data?: string + + /** + * If account login results in being able to choose multiple account, use a zero-based index to choose which one to monitor + * + * @examples [0,1] + */ + account_index?: number + + location?: string + lang?: string + generate_session_locally?: boolean + device_category?: string + client_type?: string + timezone?: string +} export interface YTMusicData extends CommonSourceData, PollingOptions { - /** + /** * The cookie retrieved from the Request Headers of music.youtube.com after logging in. * * See https://ytmusicapi.readthedocs.io/en/stable/setup/browser.html#copy-authentication-headers for how to retrieve this value. @@ -14,13 +45,33 @@ export interface YTMusicData extends CommonSourceData, PollingOptions { * */ cookie?: string + /** + * Google Cloud Console project OAuth Client ID + * + * Generated from a custom OAuth Client, see docs + */ clientId?: string + /** + * Google Cloud Console project OAuth Client Secret + * + * Generated from a custom OAuth Client, see docs + */ clientSecret?: string + /** + * Google Cloud Console project OAuth Client Authorized redirect URI + * + * Generated from a custom OAuth Client, see docs. multi-scrobbler will generate a default based on BASE_URL. + * Only specify this if the default does not work for you. + */ redirectUri?: string + + /** + * Additional options for authorization and tailoring YTM client + */ + innertubeOptions?: InnertubeOptions } -//export type YTMusicData = YTMusicDataCommon & InnertubeOptions; export interface YTMusicSourceConfig extends CommonSourceConfig { data?: YTMusicData diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index 9d7e13a9..c9de4395 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -3,7 +3,7 @@ import EventEmitter from "events"; import { PlayObject } from "../../core/Atomic.js"; import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; import { YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js"; -import { Innertube, UniversalCache, Parser, YTNodes, ApiResponse, IBrowseResponse, Log } from 'youtubei.js'; +import { Innertube, UniversalCache, Parser, YTNodes, ApiResponse, IBrowseResponse, Log, SessionOptions } from 'youtubei.js'; import { OAuth2Client } from 'google-auth-library'; import {resolve} from 'path'; import { joinedUrl, sleep } from "../utils.js"; @@ -92,8 +92,13 @@ export default class YTMusicSource extends AbstractSource { } protected async doBuildInitData(): Promise { + const { + cookie, + innertubeOptions = {}, + } = this.config.data || {}; this.yti = await Innertube.create({ - ...this.config.data, + ...(innertubeOptions as SessionOptions), + cookie, cache: new UniversalCache(true, this.workingCredsPath) }); this.yti.session.on('update-credentials', async ({ credentials }) => { From b4a95e0cf9fd1e8dcdfe4e0d08f53facacb83135 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 5 Dec 2024 09:18:05 -0500 Subject: [PATCH 04/11] fix(ytmusic): query string on redirect uri incorrectly encoded --- src/backend/sources/YTMusicSource.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index c9de4395..b5c32304 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -196,7 +196,12 @@ export default class YTMusicSource extends AbstractSource { if (this.authed === false) { if (this.config.data.clientId !== undefined) { - const redirectUri = this.config.data?.redirectUri ?? joinedUrl(this.localUrl, `api/ytmusic/callback?name=${this.name}`).toString(); + let redirectUri = this.config.data?.redirectUri; + if(redirectUri === undefined) { + const u = joinedUrl(this.localUrl, 'api/ytmusic/callback'); + u.searchParams.append('name', this.name); + redirectUri = u.toString(); + } this.logger.info(`Using Custom OAuth Client with Redirect URI: ${redirectUri}`); this.oauthClient = new OAuth2Client({ From e2a7fff6f9e1f7f084a9ce08d2b13bca6d7babbc Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 6 Dec 2024 15:17:18 +0000 Subject: [PATCH 05/11] chore: Bump youtubei.js major version * Fixes parsing warning for history response --- package-lock.json | 16 ++++++++-------- package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index db865546..e26d4213 100644 --- a/package-lock.json +++ b/package-lock.json @@ -82,7 +82,7 @@ "vite-express": "^0.16.0", "vlc-client": "^1.1.1", "xml2js": "0.6.1", - "youtubei.js": "^11.0.1" + "youtubei.js": "^12.0.0" }, "devDependencies": { "@dbus-types/notifications": "^0.0.5", @@ -6810,9 +6810,9 @@ } }, "node_modules/jintr": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/jintr/-/jintr-3.0.2.tgz", - "integrity": "sha512-5g2EBudeJFOopjAX4exAv5OCCW1DgUISfoioCsm1h9Q9HJ41LmnZ6J52PCsqBlQihsmp0VDuxreAVzM7yk5nFA==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jintr/-/jintr-3.1.0.tgz", + "integrity": "sha512-azhCHApkRfBH8INpiUCwKBYaNCdB5G+x3NApsI2MxQXSlgFAx7rap3YwE3JAkN08GO8f3ilZsGB0Yvc+412ntQ==", "funding": [ "https://github.com/sponsors/LuanRT" ], @@ -11922,15 +11922,15 @@ } }, "node_modules/youtubei.js": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-11.0.1.tgz", - "integrity": "sha512-ZsbOd+5XF2Ofi3FrLMfYd+f9g9H8xswlouFhjhOqbwT68dMJtX6CRGsHNj5VTFCR/+L/865x1lnUlllB2dDDTA==", + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/youtubei.js/-/youtubei.js-12.0.0.tgz", + "integrity": "sha512-pGmVb1I9b2gseqmuMx+BCajzVUi04+r+8zxj4Fk/iQaGQGvBCbY87Tu9mdvEgIQYTkkb4Fza7GZGrH9AjYNbrw==", "funding": [ "https://github.com/sponsors/LuanRT" ], "dependencies": { "@bufbuild/protobuf": "^2.0.0", - "jintr": "^3.0.2", + "jintr": "^3.1.0", "tslib": "^2.5.0", "undici": "^5.19.1" } diff --git a/package.json b/package.json index 4d3d2549..405c3a80 100644 --- a/package.json +++ b/package.json @@ -112,7 +112,7 @@ "vite-express": "^0.16.0", "vlc-client": "^1.1.1", "xml2js": "0.6.1", - "youtubei.js": "^11.0.1" + "youtubei.js": "^12.0.0" }, "devDependencies": { "@dbus-types/notifications": "^0.0.5", From 97222ed233b79f775e3d6bdc9895324cd29cbb96 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 6 Dec 2024 15:18:07 +0000 Subject: [PATCH 06/11] refactor(ytmusic): Improve auth handling for reauthentication --- src/backend/sources/YTMusicSource.ts | 200 ++++++++++++++------------- 1 file changed, 107 insertions(+), 93 deletions(-) diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index b5c32304..83f0f99a 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -4,7 +4,7 @@ import { PlayObject } from "../../core/Atomic.js"; import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js"; import { YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js"; import { Innertube, UniversalCache, Parser, YTNodes, ApiResponse, IBrowseResponse, Log, SessionOptions } from 'youtubei.js'; -import { OAuth2Client } from 'google-auth-library'; +import { GenerateAuthUrlOpts, OAuth2Client } from 'google-auth-library'; import {resolve} from 'path'; import { joinedUrl, sleep } from "../utils.js"; import { @@ -15,6 +15,7 @@ import { playsAreSortConsistent } from "../utils/PlayComparisonUtils.js"; import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; +import { truncateStringToLength } from "../../core/StringUtils.js"; export const ytiHistoryResponseToListItems = (res: ApiResponse): YTNodes.MusicResponsiveListItem[] => { const page = Parser.parseResponse(res.data); @@ -56,13 +57,24 @@ export const ytiHistoryResponseFromShelfToPlays = (res: ApiResponse): PlayObject return items; } +const GOOGLE_OAUTH_OPTS: GenerateAuthUrlOpts = { + access_type: 'offline', + scope: [ + "http://gdata.youtube.com", + "https://www.googleapis.com/auth/youtube", + "https://www.googleapis.com/auth/youtube.force-ssl", + "https://www.googleapis.com/auth/youtube-paid-content", + "https://www.googleapis.com/auth/accounts.reauth", + ], + include_granted_scopes: true, + prompt: 'consent', +}; + export default class YTMusicSource extends AbstractSource { requiresAuth = true; requiresAuthInteraction = true; - cookieBased: boolean = false; - declare config: YTMusicSourceConfig recentlyPlayed: PlayObject[] = []; @@ -77,8 +89,6 @@ export default class YTMusicSource extends AbstractSource { constructor(name: string, config: YTMusicSourceConfig, internal: InternalConfig, emitter: EventEmitter) { super('ytmusic', name, config, internal, emitter); this.canPoll = true; - Log.setLevel(Log.Level.ERROR); - this.cookieBased = this.config.data?.cookie !== undefined; this.supportsUpstreamRecentlyPlayed = true; this.workingCredsPath = resolve(this.configDir, `yti-${this.name}`); } @@ -91,6 +101,54 @@ export default class YTMusicSource extends AbstractSource { return data; } + protected configureYTIEvents() { + this.yti.session.on('update-credentials', async ({ credentials }) => { + if(this.config.options?.logAuth) { + this.logger.debug(credentials, 'Credentials updated'); + } else { + this.logger.debug('Credentials updated'); + } + await this.yti.session.oauth.cacheCredentials(); + }); + this.yti.session.on('auth-pending', async (data) => { + if(this.oauthClient === undefined) { + this.userCode = data.user_code; + this.verificationUrl = data.verification_url; + } + }); + this.yti.session.on('auth-error', async (data) => { + this.logger.error(new Error('YTM Authentication error', {cause: data})); + }); + this.yti.session.on('auth', async ({ credentials }) => { + if(this.config.options?.logAuth) { + this.logger.debug(credentials, 'Auth success'); + } else { + this.logger.debug('Auth success'); + } + this.userCode = undefined; + this.authed = true; + await this.yti.session.oauth.cacheCredentials(); + }); + } + + protected configureCustomOauth() { + let redirectUri = this.config.data?.redirectUri; + if(redirectUri === undefined) { + const u = joinedUrl(this.localUrl, 'api/ytmusic/callback'); + u.searchParams.append('name', this.name); + redirectUri = u.toString(); + } + + this.oauthClient = new OAuth2Client({ + clientId: this.config.data.clientId, + clientSecret: this.config.data.clientSecret, + redirectUri + }); + + const authorizationUrl = this.oauthClient.generateAuthUrl(GOOGLE_OAUTH_OPTS); + this.verificationUrl = authorizationUrl; + } + protected async doBuildInitData(): Promise { const { cookie, @@ -101,45 +159,35 @@ export default class YTMusicSource extends AbstractSource { cookie, cache: new UniversalCache(true, this.workingCredsPath) }); - this.yti.session.on('update-credentials', async ({ credentials }) => { - if(this.config.options?.logAuth) { - this.logger.debug(credentials, 'Credentials updated'); - } else { - this.logger.debug('Credentials updated'); - } - await this.yti.session.oauth.cacheCredentials(); - }); - this.yti.session.on('auth-pending', async (data) => { - this.userCode = data.user_code; - this.verificationUrl = data.verification_url; - }); - this.yti.session.on('auth-error', async (data) => { - this.logger.error(new Error('YTM Authentication error', {cause: data})); - }); - this.yti.session.on('auth', async ({ credentials }) => { - if(this.config.options?.logAuth) { - this.logger.debug(credentials, 'Auth success'); - } else { - this.logger.debug('Auth success'); - } - this.userCode = undefined; - this.verificationUrl = undefined; - this.authed = true; - await this.yti.session.oauth.cacheCredentials(); - const f =1; - }); + + if (this.config.data.clientId !== undefined && this.config.data.clientSecret !== undefined) { + this.configureCustomOauth(); + this.logger.info(`Will use custom OAuth Client`); + } else if (this.config.data.clientId !== undefined || this.config.data.clientSecret !== undefined) { + const missing = this.config.data.clientId !== undefined ? 'clientSecret' : 'clientId'; + throw new Error(`It looks like you tried to configure a custom OAuth Client but are missing '${missing}'! Cannot build client.`); + } else if (cookie !== undefined) { + this.logger.info(`Will use cookie '${truncateStringToLength(10)(cookie)}' for auth`); + } else { + this.logger.warn('You have not provided a cookie or custom OAuth client for authorization. MS will use the fallback YoutubeTV auth but this will likely NOT provide access to Youtube Music history!! You should use one of the other methods.'); + } + + this.configureYTIEvents(); + return true; } reauthenticate = async () => { await this.tryStopPolling(); - await this.clearCredentials(); - this.authed = false; - await this.testAuth(); + if(this.authed) { + await this.clearCredentials(); + this.authed = false; + await this.testAuth(); + } } clearCredentials = async () => { - if(this.yti.session.logged_in && !this.cookieBased) { + if(this.yti.session.logged_in) { await this.yti.session.signOut(); } } @@ -164,9 +212,7 @@ export default class YTMusicSource extends AbstractSource { }); this.authed = true; this.verificationUrl = undefined; - this.userCode = undefined; await this.yti.session.oauth.cacheCredentials(); - Log.setLevel(Log.Level.ERROR); return true; } else { this.logger.error(`Token data did not return all required properties.`); @@ -176,7 +222,7 @@ export default class YTMusicSource extends AbstractSource { doAuthentication = async () => { try { - if (this.cookieBased) { + if (this.config.data.cookie !== undefined) { try { await this.yti.account.getInfo() this.authed = true; @@ -185,66 +231,34 @@ export default class YTMusicSource extends AbstractSource { if (info !== undefined) { this.logger.error(info, 'Additional API response details') } - this.logger.error(new Error('Cookie-based authentication failed. Try recreating cookie or using custom OAuth Client', { cause: e })); + throw new Error('Cookie-based authentication failed. Try recreating cookie or using custom OAuth Client', { cause: e }); } - } - - await Promise.race([ - sleep(1000), - this.yti.session.signIn() - ]); - if (this.authed === false) { - - if (this.config.data.clientId !== undefined) { - let redirectUri = this.config.data?.redirectUri; - if(redirectUri === undefined) { - const u = joinedUrl(this.localUrl, 'api/ytmusic/callback'); - u.searchParams.append('name', this.name); - redirectUri = u.toString(); - } - - this.logger.info(`Using Custom OAuth Client with Redirect URI: ${redirectUri}`); - this.oauthClient = new OAuth2Client({ - clientId: this.config.data.clientId, - clientSecret: this.config.data.clientSecret, - redirectUri - }); - - const authorizationUrl = this.oauthClient.generateAuthUrl({ - access_type: 'offline', - scope: [ - "http://gdata.youtube.com", - "https://www.googleapis.com/auth/youtube", - "https://www.googleapis.com/auth/youtube.force-ssl", - "https://www.googleapis.com/auth/youtube-paid-content", - "https://www.googleapis.com/auth/accounts.reauth", - ], - include_granted_scopes: true, - prompt: 'consent', - }); - - this.verificationUrl = authorizationUrl; - this.userCode = undefined; - throw new Error(`Sign in using ${authorizationUrl}`); - } else { - if (this.userCode !== undefined) { - this.logger.warn('Logging in with YoutubeTV Oauth will likely NOT provide access to Youtube Music history!! You should try to use either cookies or a custom OAuth Client ID/Secret'); - throw new Error(`Sign in with the code '${this.userCode}' using the authentication link on the dashboard or ${this.verificationUrl}`) + } else { + await Promise.race([ + sleep(1000), + this.yti.session.signIn() + ]); + if (this.authed === false) { + + if(this.oauthClient !== undefined) { + throw new Error(`Sign in using the authentication link on the dashboard or ${this.verificationUrl}`); } else { - throw new Error('Waited too long for auth response from YTM!'); + throw new Error(`Sign in with the code '${this.userCode}' using the authentication link on the dashboard or ${this.verificationUrl}`) } + } - } - try { - await this.yti.account.getInfo() - } catch (e) { - const info = loggedErrorExtra(e); - if (info !== undefined) { - this.logger.error(info, 'Additional API response details') + + try { + await this.yti.account.getInfo() + } catch (e) { + const info = loggedErrorExtra(e); + if (info !== undefined) { + this.logger.error(info, 'Additional API response details') + } + throw new Error('Credentials exist but API calls are failing. Try re-authenticating?', { cause: e }); } - throw new Error('Credentials exist but API calls are failing. Try re-authenticating?', { cause: e }); } - Log.setLevel(Log.Level.ERROR); + return true; } catch (e) { throw e; From 570dee30e32077e12b74f97bb8f7676f39268d79 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 6 Dec 2024 15:33:02 +0000 Subject: [PATCH 07/11] fix(ytmusic): Add missing logging for redirect URI --- src/backend/sources/YTMusicSource.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index 83f0f99a..ab81b697 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -82,6 +82,7 @@ export default class YTMusicSource extends AbstractSource { yti: Innertube; userCode?: string; verificationUrl?: string; + redirectUri?: string; oauthClient?: OAuth2Client; workingCredsPath: string; @@ -132,17 +133,17 @@ export default class YTMusicSource extends AbstractSource { } protected configureCustomOauth() { - let redirectUri = this.config.data?.redirectUri; - if(redirectUri === undefined) { + this.redirectUri = this.config.data?.redirectUri; + if(this.redirectUri === undefined) { const u = joinedUrl(this.localUrl, 'api/ytmusic/callback'); u.searchParams.append('name', this.name); - redirectUri = u.toString(); + this.redirectUri = u.toString(); } this.oauthClient = new OAuth2Client({ clientId: this.config.data.clientId, clientSecret: this.config.data.clientSecret, - redirectUri + redirectUri: this.redirectUri, }); const authorizationUrl = this.oauthClient.generateAuthUrl(GOOGLE_OAUTH_OPTS); @@ -162,7 +163,10 @@ export default class YTMusicSource extends AbstractSource { if (this.config.data.clientId !== undefined && this.config.data.clientSecret !== undefined) { this.configureCustomOauth(); - this.logger.info(`Will use custom OAuth Client`); + this.logger.info(`Will use custom OAuth Client: +Client ID : ${truncateStringToLength(10)(this.config.data.clientId)} +Client Secret : ${truncateStringToLength(10)(this.config.data.clientSecret)} +Redirect URI : ${this.redirectUri}`); } else if (this.config.data.clientId !== undefined || this.config.data.clientSecret !== undefined) { const missing = this.config.data.clientId !== undefined ? 'clientSecret' : 'clientId'; throw new Error(`It looks like you tried to configure a custom OAuth Client but are missing '${missing}'! Cannot build client.`); From e8716a45b2c04514255017269f15b10f236bd95e Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 6 Dec 2024 15:56:32 +0000 Subject: [PATCH 08/11] feat(ytmusic): Custom redirectUri checks --- src/backend/sources/YTMusicSource.ts | 31 +++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index ab81b697..884d4dfd 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -138,6 +138,31 @@ export default class YTMusicSource extends AbstractSource { const u = joinedUrl(this.localUrl, 'api/ytmusic/callback'); u.searchParams.append('name', this.name); this.redirectUri = u.toString(); + } else { + // verify custom URI has required parts + let u: URL; + try { + u = new URL(this.redirectUri); + } catch(e) { + throw new Error(`custom redirectUri '${this.redirectUri}' could not be parsed as a URL`, {cause: e}); + } + + if(!u.protocol.includes('http')) { + throw new Error(`Custom redirectUri '${this.redirectUri}' is missing protocol! Must start with 'http' or 'https'`); + } + if(!u.pathname.includes('api')) { + this.logger.warn(`Custom redirectUri '${this.redirectUri}' does not contain 'api' in path! Unless you know what you are doing with redirects this will likely cause authentication to fail.`); + } + if(null === u.pathname.match(/ytmusic\/callback$/)) { + throw new Error(`Custom redirectUri '${this.redirectUri}' must end in 'ytmusic/callback' before querystring!`); + } + if(!u.searchParams.has('name')) { + throw new Error(`Custom redirectUri '${this.redirectUri}' is missing 'name' in querystring! EX ?name=${this.name}`); + } + const nameVal = u.searchParams.get('name'); + if(nameVal !== this.name) { + throw new Error(`Custom redirectUri '${this.redirectUri}' has wrong value '${nameVal}' for 'name' key in querystring. Must match Source name, case-sensitive -- EX ?name=${this.name}`); + } } this.oauthClient = new OAuth2Client({ @@ -162,7 +187,11 @@ export default class YTMusicSource extends AbstractSource { }); if (this.config.data.clientId !== undefined && this.config.data.clientSecret !== undefined) { - this.configureCustomOauth(); + try { + this.configureCustomOauth(); + } catch (e) { + throw new Error('Unable to build custom OAuth Client', { cause: e }); + } this.logger.info(`Will use custom OAuth Client: Client ID : ${truncateStringToLength(10)(this.config.data.clientId)} Client Secret : ${truncateStringToLength(10)(this.config.data.clientSecret)} From c33d42808a60dc25c1f675e2d3bf32ba761bec43 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 6 Dec 2024 19:09:32 +0000 Subject: [PATCH 09/11] docs(ytmusic): Add new auth docs and update FAQ --- config/ytmusic.json.example | 10 ++ docsite/docs/FAQ.md | 70 +++++++- docsite/docs/configuration/configuration.mdx | 164 +++++++++++++++++-- 3 files changed, 224 insertions(+), 20 deletions(-) diff --git a/config/ytmusic.json.example b/config/ytmusic.json.example index ffb48e99..736c1840 100644 --- a/config/ytmusic.json.example +++ b/config/ytmusic.json.example @@ -4,6 +4,16 @@ "enable": true, "clients": [], "data": { + "cookie": "__Secure-1PSIDTS=sidts-CjEB3EgAEvCd-......", + // either cookie or id/secret needs to be provided + "clientId": "891098404....apps.googleusercontent.com", + "clientSecret": "GOCS..." + + // optional + //"redirectUri": "http://my.custom.tld/api/ytmusic/callback?name=MyYTMusic" + }, + "options": { + "logDiff": true } } ] diff --git a/docsite/docs/FAQ.md b/docsite/docs/FAQ.md index b8c2171a..fea33e67 100644 --- a/docsite/docs/FAQ.md +++ b/docsite/docs/FAQ.md @@ -130,15 +130,25 @@ Deezer has discontinued support for their API and the Deezer Source is now [**de ### Youtube Music fails after some time -The Youtube Music library relies on scraping the YTM site (pretending to be a browser) by using cookies/auth from your actual browser. It does its best to keep these up to date but since this is not an official way to access the service YTM may invalidate your access _to the authenticated session_ at any time. How this is triggered is unknown and not something multi-scrobbler can control. - If you see errors in multi-scrobbler for YTM that contain **401** or **403** like ``` Error: Could not send the specified request to browse. Status code: 401 ``` -then YTM has invalidated your access. [Follow the YTM instructions to retrieve a new set of cookies for multi-scrobbler](configuration/configuration.mdx#youtube-music) and then restart MS to potentially resolve the problem. See [this issue](https://github.com/FoxxMD/multi-scrobbler/issues/158) for further discussion of the problem. +then YTM has invalidated your authentication. + +First, ensure you are NOT using [YoutubeTV authentication.](configuration/configuration.mdx?ytmAuth=ytt#youtube-music) If you completed authentication by entering a "User Code" you are using YoutubeTV which has stopped working. You should reauthenticate using **Cookies** or **Custom OAuth.** + +#### When using OAuth Client Authentication + +Refresh your authentication by using the **(Re)authenticate** link from MS's web dashboard. + +#### When using Cookies Authentication + +The library MS uses relies on scraping the YTM site by using cookies from your actual browser to pretend it is a browser. It does its best to keep these up to date but since this is not an official way to access the service YTM may invalidate your access _to the authenticated session_ at any time. How this is triggered is unknown and not something multi-scrobbler can control. You can help limit the chance of your session being invalidated by [getting the cookie from an Incognito/Private Session](https://github.com/LuanRT/YouTube.js/issues/803#issuecomment-2504032666) and then immediately closing the browser afterwards. + +To re-authenticate MS [follow the YTM instructions to retrieve a new set of cookies for multi-scrobbler](configuration/configuration.mdx?ytmAuth=cookie#youtube-music) and then restart MS to potentially resolve the problem. ## Configuration Issues @@ -194,10 +204,58 @@ Refer to [Force Media Tracking](configuration/configuration.mdx#forcing-media-tr Before reporting an issue turn on metadata logging in the MS VLC configuration, [see the VLC documentation.](configuration/configuration.mdx#vlc-information-reporting) -### Youtube Music misses scrobbles +### Youtube Music misses or duplicates scrobbles In order for multi-scrobbler to accurately determine if a song has been scrobbled it needs **a source of truth.** For YTM this is a "history" list scraped from the YTM website. Unfortunately, the data in this list can be (often) inconsistent which makes it hard for multi-scrobbler to "trust" that it is correct and determine when/if new scrobbles occur. This inconsistency is not something multi-scrobbler can control -- it is a side-effect of having to use an unofficial method to access YTM (scraping). -In order to compensate for this multi-scrobbler resets when it considers this list the "source of truth" based on if the list changes in an inconsistent way between consecutive checks. New scrobbles can only be detected when this list is "OK" as a source of truth for N+1 checks. Therefore, any new tracks that appear when the list is inconsistent will be ignored. +To compensate for this multi-scrobbler resets when it considers this list the "source of truth" based on if the list changes in an inconsistent way between consecutive checks. New scrobbles can only be detected when this list is "OK" as a source of truth for N+1 checks. Therefore, any new tracks that appear when the list is inconsistent will be ignored. + +Duplicate scrobbles can also occur if the change between two checks is technically consistent. For instance, if you listen to a track twice in some period, separated by other music, YTM will sometimes "remove" the track from the earlier time (further down in your history) and "re-add" it at the top of the history. + +#### Reporting YTM scrobble issues + +If you experience these behaviors you can help improve MS's YTM heureistic by providing thorough feedback as [an issue.](https://github.com/FoxxMD/multi-scrobbler/issues/new?assignees=&labels=bug&projects=&template=01-bug-report.yml&title=bug%3A+) **Please do the following to provide the most useful report:** + +##### Turn on Change Detection + +In your YTM configuration (`ytmusic.json`) add `logDiff` under `options` like this: + + +```json +{ + "type": "ytmusic", + "name": "MyYTM", + "data": { ... }, + "options": { + "logDiff": true + } +} +``` + +This will cause MS to log YTM history changes similar to this: + +``` +[Ytmusic - MyYTM] Changes from last seen list: +1. (tuhe1CpHRxY) KNOWER - I’m The President --- undefined => Moved - Originally at 6 +2. (Mtg8V6Xa2nc) Vulfpeck - Romanian Drinking Song --- Schvitz => Moved - Originally at 1 +3. (rxbCaiyYSXM) Nightmares On Wax - You Wish --- In A Space Outta Sound => Moved - Originally at 2 +4. (tMt_YXr90AM) Gorillaz - O Green World --- undefined => Moved - Originally at 3 +... +``` + +Which are essential to troubleshooting this behavior. + +##### Provide Detail and Context + +Provide a detailed account of how you were using YTM when the issue occurred, including things like: + +* the platform listening on (desktop, mobile, 3rd party client, etc...) +* any changes in platform + * > I switched from desktop to listening on my phone... +* how you were listening to music + * > I was playing an album start to finish + * > I listened to two songs in a row, then browsed for a new song in library by artist, then went back to a song in the queue... + +Explain the expected behavior (it should have scrobbled songs x, y, then z) and what actually happened (it scrobbled songs x, then y, then x again, then z) -See [this issue](https://github.com/FoxxMD/multi-scrobbler/issues/156#issuecomment-2312533486) for further discussion and a more detailed explanation of why this is happening and how multi-scrobbler compensates for it. +Provide ALL logs from the time when the issue occurred including logs from BEFORE (ideally 2-3 minutes of logs) and AFTER the issue. \ No newline at end of file diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index dd684dac..bfb0b8ce 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -721,30 +721,166 @@ After starting multi-scrobbler with credentials in-place open the dashboard (`ht ### [Youtube Music](https://music.youtube.com) -
+:::warning - Migrating from YT Music cookie-based Source +* Communication with YT Music is **unofficial** and not supported or endorsed by Google. This means that **this integration may stop working at any time** if Google decides to change how YT Music works in the browser. + * Due to this scrobble history from YTM is often inconsistent and can cause missed scrobbles. [See the FAQ](../FAQ.md#youtube-music-misses-scrobbles) for a more detailed explanation. - In multi-scrobbler **below v0.9.0** YT Music credentials were extracted from browser cookies. Due to authentication inconsistency and YT service changes this was approach was dropped in favor of [oauth authentication which is more stable.](https://ytjs.dev/guide/authentication.html#youtube-tv-oauth2) +::: - Your existing credentials cannot be migrated. However, the oauth approach is quite easy. Continue following the directions below to setup new authentication for your YT Music Source. +#### Authentication -
+Only one of these methods needs to be used. **Cookies** are easier but **OAuth Client** may be more stable. -:::note + + + :::info -* Communication to YT Music is **unofficial** and not supported or endorsed by Google. This means that **this integration may stop working at any time** if Google decides to change how YT Music works in the browser. - * Due to this scrobble history from YTM is often inconsistent and can cause missed scrobbles. [See the FAQ](../FAQ.md#youtube-music-misses-scrobbles) for a more detailed explanation. + If cookies stop working for you or are being invalidated often try switching to **OAuth Client** authentication. -::: + ::: -To authenticate simply start multi-scrobbler with an empty YT Music configuration. An authentication URL/code will be logged in additon to being available from the dashboard. + Use instructions from + + * https://github.com/patrickkfkan/Volumio-YouTube.js/wiki/How-to-obtain-Cookie or + * https://ytmusicapi.readthedocs.io/en/stable/setup/browser.html#copy-authentication-headers + + to get the **Cookie** value from a browser. + + It is highly recommended to [get the cookie from an Incognito/Private Session](https://github.com/LuanRT/YouTube.js/issues/803#issuecomment-2504032666) to limit the chance the session is invalidated from normal browsing. + + Add the cookie to your `ytmusic.json` config in `data`: + + ```json + { + "type": "ytmusic", + "enable": true, + "name": "MyYTM", + "data": { + "cookie": "__Secure-1PSIDTS=sidts-CjEB3EgAEvCd-......" + }, + "options": { + "logAuthUpdateChanges": true, + "logDiff": true + } + } + ``` -``` -[2024-10-09 15:24:17.358 -0400] INFO : [App] [Sources] [Ytmusic - MyYTM] ERROR: Sign in with the code 'CLV-KFA-BVKY' using the authentication link on the dashboard or https://www.google.com/device -``` + If MS gives you authentication errors (session invalidated) at some point in the future follow the same instructions to get new cookies. + + + + :::note + + This is likely to be the most stable method and least likely to be blocked or have authentication invalidated after an extended period. It requires more setup but is worth the effort. + + ::: + + [Based on the instructions from here...](https://github.com/LuanRT/YouTube.js/issues/803#issuecomment-2479689924) + + * Login to [Google Cloud console](https://console.cloud.google.com/) (create an account, if necessary) + * [Create a new project](https://console.cloud.google.com/projectcreate) + * Go to APIs and services. + * Configure the OAuth consent screen + * Use the old experience if possible + * If new is unavoidable then do not fill out any branding and under Authorized Domains you can delete the empty one (in order to save) + * Add yourself as an authorized user + * Navigate to Credentials + * Create Credentials -> choose "OAuth client ID" + * Application Type is **Web Application** + * **Name** is whatever you want, leave Authorization Javascript origins blank + * Authorized redirect URIs + * This must be **exactly** the same as what is displayed in MS! For now leave it blank so we can generate it from MS first + * Create + * In the newly created client popup save the **Client ID** and **Client Secret**, then copy them into `ytmusic.json` + + ```json + { + "type": "ytmusic", + "enable": true, + "name": "MyYTM", + "data": { + "clientId": "8910....6jqupl.apps.googleusercontent.com", + "clientSecret": "GOCSPX-WGXL6BSuQ343..." + }, + "options": { + "logAuthUpdateChanges": true, + "logDiff": true + } + } + ``` + + Now, start MS and during the YTMusic startup it will log something like this: -Visit the authentication URL and enter the code that was provided (also available on the dashboard). After completing the setup flow MS will log `Auth success` and the YT Music dashboard card will display as **Idle** after refreshing. Click the **Start** link to begin monitoring. + ``` + Using Custom OAuth Client: + Client ID: ... + Client Secret: ... + Redirect URI: http://localhost:9078/api/ytmusic/callback?name=MyYTM + ``` + + If the beginning of the URL (before `api`) is EXACTLY how you would reach the MS dashboard from your browser (EX `http://localhost:9078`) then edit your google oauth client section for `Authorized redirect URIs` and add the URL MS has displayed. + + If it is NOT EXACTLY the same you either need to set MS's [base url](https://foxxmd.github.io/multi-scrobbler/docs/configuration/#base-url) or you can provide your own (Custom) Redirect URI for MS to use by setting it in `ytmusic.json`. + +
+ + Using a Custom Redirect URI + + The three parts of the URL that must be the same: + + * it must start with `api` (after domain or subdirectory IE `my.domain.tld/api...` or `whatever.tld/subDir/api...` + * it must end in `ytmusic/callback` + * It must include `name=[NameOfSource]` in the query string + + Remember to add your custom URL to the `Authorized redirect URIs` section in the google oauth client! + + ```json + { + "type": "ytmusic", + "enable": true, + "name": "MyYTM", + "data": { + "clientId": "8910....6jqupl.apps.googleusercontent.com", + "clientSecret": "GOCSPX-WGXL6BSuQ343...", + "redirectUri": "http://my.custom.domain/api/ytmusic/callback?name=MyYTM" + }, + "options": { + "logAuthUpdateChanges": true, + "logDiff": true + } + } + ``` + +
+ + AFTER changing the Authorized redirect URIs on Google Cloud console you may need to wait a few minutes for it to take affect. Then restart MS. From the dashboard click `(Re)authenticate` on the YTmusic source card and follow the auth flow: + + * On the screen about "testing" make sure you hit **Continue** (not Back To Safety) + * Make sure to select ALL scopes/permissions/grants it asks you about + * `Select what [YourAppName] can access` -> Select all + + Once the flow is finished MS will get the credentials and start polling automatically. You should not need to re-authenticate again after restarting MS as it saves the credentials to the `/config` folder. + + + + + :::warning + + Using the built-in YoutubeTV authentication is unlikely to work due to [Google restricting what permissions TV clients can have](https://github.com/yt-dlp/yt-dlp/issues/11462#issuecomment-2471703090). This authentication method should not be used. + + ::: + + To authenticate start multi-scrobbler with an empty YT Music configuration. An authentication URL/code will be logged in additon to being available from the dashboard. + + ``` + ERROR: Sign in with the code 'CLV-KFA-BVKY' using the authentication link on the dashboard or https://www.google.com/device + ``` + + Visit the authentication URL and enter the code that was provided (also available on the dashboard). After completing the setup flow MS will log `Auth success` and the YT Music dashboard card will display as **Idle** after refreshing. Click the **Start** link to begin monitoring. + + + #### Configuration From 5c0f419e7ed240f3901959a1115f0cc67feb5a00 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 6 Dec 2024 19:28:32 +0000 Subject: [PATCH 10/11] feat(ytm): Implement ENV config --- docsite/docs/configuration/configuration.mdx | 13 ++++++++++++- src/backend/sources/ScrobbleSources.ts | 20 +++++++++++++++++++- src/backend/sources/YTMusicSource.ts | 12 +++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index bfb0b8ce..af3f800a 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -886,7 +886,18 @@ Only one of these methods needs to be used. **Cookies** are easier but **OAuth C - No ENV support + + + +| Environmental Variable | Required? | Default | Description | +|------------------------|-----------|---------|-----------------------------------------------| +| YTM_COOKIE | No | | Value for Cookie Authentication | +| YTM_CLIENT_ID | No | | Client ID for OAuth Athentication | +| YTM_CLIENT_SECRET | No | | Client Secret for OAuth Athentication | +| YTM_REDIRECT_URI | No | | A custom redirect URI for OAuth Athentication | +| YTM_LOG_DIFF | No | false | Log YTM history changes | + +
diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index 097a9f84..6dfc1d45 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -26,7 +26,7 @@ import { SubsonicData, SubSonicSourceConfig } from "../common/infrastructure/con import { TautulliSourceConfig } from "../common/infrastructure/config/source/tautulli.js"; import { VLCData, VLCSourceConfig } from "../common/infrastructure/config/source/vlc.js"; import { WebScrobblerSourceConfig } from "../common/infrastructure/config/source/webscrobbler.js"; -import { YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js"; +import { YTMusicData, YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js"; import { WildcardEmitter } from "../common/WildcardEmitter.js"; import { parseBool, readJson } from "../utils.js"; import { validateJson } from "../utils/ValidationUtils.js"; @@ -484,6 +484,24 @@ export default class ScrobbleSources { }); } break; + case 'ytmusic': + const ytm = { + redirectUri: process.env.YTM_REDIRECT_URI, + clientId: process.env.YTM_CLIENT_ID, + clientSecret: process.env.YTM_CLIENT_SECRET, + cookie: process.env.YTM_COOKIE + } + if (!Object.values(ytm).every(x => x === undefined)) { + configs.push({ + type: 'ytmusic', + name: 'unnamed', + source: 'ENV', + mode: 'single', + configureAs: defaultConfigureAs, + data: ytm as YTMusicData + }); + } + break; default: break; } diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index 884d4dfd..71696809 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -6,7 +6,7 @@ import { YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmu import { Innertube, UniversalCache, Parser, YTNodes, ApiResponse, IBrowseResponse, Log, SessionOptions } from 'youtubei.js'; import { GenerateAuthUrlOpts, OAuth2Client } from 'google-auth-library'; import {resolve} from 'path'; -import { joinedUrl, sleep } from "../utils.js"; +import { joinedUrl, parseBool, sleep } from "../utils.js"; import { getPlaysDiff, humanReadableDiff, @@ -92,6 +92,16 @@ export default class YTMusicSource extends AbstractSource { this.canPoll = true; this.supportsUpstreamRecentlyPlayed = true; this.workingCredsPath = resolve(this.configDir, `yti-${this.name}`); + + const diffEnv = process.env.YTM_LOG_DIFF; + if(diffEnv !== undefined && this.config.options?.logDiff === undefined) { + const logDiff = parseBool(diffEnv); + const opts = this.config.options ?? {}; + this.config.options = { + ...opts, + logDiff + } + } } public additionalApiData(): Record { From 9f11d799ac7040395d945da01ebb6b57e92c8cc2 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 9 Dec 2024 18:44:43 +0000 Subject: [PATCH 11/11] feat(source): Add experimental file logging --- src/backend/common/AbstractComponent.ts | 13 ++++++ .../infrastructure/config/source/index.ts | 8 ++++ src/backend/common/logging.ts | 44 ++++++++++++++++++- src/backend/index.ts | 2 +- src/backend/ioc.ts | 11 ++++- src/backend/sources/AbstractSource.ts | 29 ++++++++++-- 6 files changed, 99 insertions(+), 8 deletions(-) diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 3d42b7ee..1fb7d9ae 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -41,6 +41,7 @@ export default abstract class AbstractComponent { regexCache!: ReturnType; logger: Logger; + componentLogger?: Logger; protected constructor(config: CommonClientConfig | CommonSourceConfig) { this.config = config; @@ -50,6 +51,9 @@ export default abstract class AbstractComponent { this.logger.debug('Attempting to initialize...'); try { this.initializing = true; + if(this.componentLogger === undefined) { + await this.buildComponentLogger(); + } await this.buildInitData(); this.buildTransformRules(); await this.checkConnection(); @@ -69,6 +73,15 @@ export default abstract class AbstractComponent { } } + private async buildComponentLogger() { + await this.doBuildComponentLogger(); + return; + } + + protected async doBuildComponentLogger() { + return; + } + public async buildInitData() { if(this.buildOK) { return; diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts index c482099c..b7d7bd13 100644 --- a/src/backend/common/infrastructure/config/source/index.ts +++ b/src/backend/common/infrastructure/config/source/index.ts @@ -1,3 +1,4 @@ +import { FileLogOptions, LogLevel } from "@foxxmd/logging"; import { PlayTransformConfig, PlayTransformOptions } from "../../Atomic.js"; import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; @@ -72,6 +73,13 @@ export interface CommonSourceOptions extends SourceRetryOptions { * */ logPlayerState?: boolean + /** + * **Exprimental:** Log to a separate file for this Source. + * + * Useful for debugging long-running Sources + */ + logToFile?: true | LogLevel | FileLogOptions + /** * If this source * diff --git a/src/backend/common/logging.ts b/src/backend/common/logging.ts index c9cd5d05..9daa88a7 100644 --- a/src/backend/common/logging.ts +++ b/src/backend/common/logging.ts @@ -1,5 +1,5 @@ -import { childLogger, Logger, loggerAppRolling, LogOptions, parseLogOptions, } from '@foxxmd/logging'; -import { buildDestinationJsonPrettyStream, buildDestinationStdout, buildLogger } from "@foxxmd/logging/factory"; +import { childLogger, FileLogOptions, Logger, loggerAppRolling, LogLevel, LogLevelStreamEntry, LogOptions, parseLogOptions, } from '@foxxmd/logging'; +import { buildDestinationJsonPrettyStream, buildDestinationRollingFile, buildDestinationStdout, buildLogger } from "@foxxmd/logging/factory"; import { PassThrough, Transform } from "node:stream"; import path from "path"; import process from "process"; @@ -32,6 +32,46 @@ export const appLogger = async (config: LogOptions = {}): Promise<[Logger, PassT }); return [logger, stream]; } + +export const componentFileLogger = async (type: string, name: string, fileConfig: true | LogLevel | FileLogOptions, config: LogOptions = {}): Promise => { + const opts = parseLogOptions(config, { + logBaseDir: typeof process.env.CONFIG_DIR === 'string' ? process.env.CONFIG_DIR : undefined, + logDefaultPath: './logs/scrobble.log' + }); + + const base = path.dirname(typeof opts.file.path === 'function' ? opts.file.path() : opts.file.path); + const componentLogPath = path.join(base, `${type}-${name}.log`); + + const componentConfig: LogOptions = { + level: opts.level ?? 'debug' + }; + if (fileConfig === true) { + componentConfig.file = { + path: componentLogPath, + } + } else if (typeof fileConfig === 'string') { + componentConfig.file = { + level: fileConfig as LogLevel, + path: componentLogPath + } + } else { + componentConfig.file = fileConfig; + } + + const strongOpts = parseLogOptions(componentConfig); + + const streams: LogLevelStreamEntry[] = []; + + if(strongOpts.file.level !== false) { + const file = await buildDestinationRollingFile(componentConfig.file.level ?? componentConfig.level, {...strongOpts.file}) + streams.push(file); + + return buildLogger('debug' as LogLevel, streams); + } else { + throw new Error('File must be set'); + } +} + export class MaybeLogger { logger?: Logger diff --git a/src/backend/index.ts b/src/backend/index.ts index 94e033be..f37c19d6 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -82,7 +82,7 @@ const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`) const [aLogger, appLoggerStream] = await appLogger(logging) logger = childLogger(aLogger, 'App'); - const root = getRoot({...config, logger}); + const root = getRoot({...config, logger, loggingConfig: logging, loggerStream: appLoggerStream}); initLogger.info(`Version: ${root.get('version')}`); initLogger.info('Generating schema definitions...'); diff --git a/src/backend/ioc.ts b/src/backend/ioc.ts index de08740d..f4b80070 100644 --- a/src/backend/ioc.ts +++ b/src/backend/ioc.ts @@ -1,5 +1,5 @@ import { getVersion } from "@foxxmd/get-version"; -import { Logger } from "@foxxmd/logging"; +import { Logger, LogOptions } from "@foxxmd/logging"; import { EventEmitter } from "events"; import { createContainer } from "iti"; import path from "path"; @@ -9,6 +9,7 @@ import { Notifiers } from "./notifier/Notifiers.js"; import ScrobbleClients from "./scrobblers/ScrobbleClients.js"; import ScrobbleSources from "./sources/ScrobbleSources.js"; import { generateBaseURL } from "./utils.js"; +import { PassThrough } from "stream"; let version: string = 'unknown'; @@ -23,13 +24,17 @@ export interface RootOptions { port?: string | number logger: Logger disableWeb?: boolean + loggerStream?: PassThrough + loggingConfig?: LogOptions } const createRoot = (options?: RootOptions) => { const { port = 9078, baseUrl = process.env.BASE_URL, - disableWeb: dw + disableWeb: dw, + loggerStream, + loggingConfig } = options || {}; const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`); let disableWeb = dw; @@ -54,6 +59,8 @@ const createRoot = (options?: RootOptions) => { clientEmitter: () => cEmitter, sourceEmitter: () => sEmitter, notifierEmitter: () => new EventEmitter(), + loggerStream, + loggingConfig, }).add((items) => { const localUrl = generateBaseURL(baseUrl, items.port) return { diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index d2f6123b..709d34e4 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -1,4 +1,4 @@ -import { childLogger } from '@foxxmd/logging'; +import { childLogger, LogDataPretty } from '@foxxmd/logging'; import dayjs, { Dayjs } from "dayjs"; import { EventEmitter } from "events"; import { FixedSizeList } from "fixed-size-list"; @@ -23,6 +23,7 @@ import { import { SourceConfig } from "../common/infrastructure/config/source/sources.js"; import TupleMap from "../common/TupleMap.js"; import { + difference, formatNumber, genGroupId, playObjDataMatch, @@ -32,6 +33,8 @@ import { sortByOldestPlayDate, } from "../utils.js"; import { comparePlayTemporally, temporalAccuracyIsAtLeast } from "../utils/TimeUtils.js"; +import { getRoot } from '../ioc.js'; +import { componentFileLogger } from '../common/logging.js'; export interface RecentlyPlayedOptions { limit?: number @@ -76,13 +79,16 @@ export default abstract class AbstractSource extends AbstractComponent implement protected recentDiscoveredPlays: GroupedFixedPlays = new TupleMap>(); + protected loggerLabel: string; + constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) { super(config); const {clients = [] } = config; this.type = type; this.name = name; - this.identifier = `Source - ${capitalize(this.type)} - ${name}`; - this.logger = childLogger(internal.logger, `${capitalize(this.type)} - ${name}`); + this.loggerLabel = `${capitalize(this.type)} - ${name}`; + this.identifier = `Source - ${this.loggerLabel}`; + this.logger = childLogger(internal.logger, `${this.loggerLabel}`); this.config = config; this.clients = clients; this.instantiatedAt = dayjs(); @@ -502,4 +508,21 @@ export default abstract class AbstractSource extends AbstractComponent implement public async destroy() { this.emitter.removeAllListeners(); } + + protected async doBuildComponentLogger(): Promise { + if(this.config.options.logToFile) { + this.logger.debug('Enabling component logger...'); + const root = getRoot(); + const stream = root.get('loggerStream'); + const logConfig = root.get('loggingConfig'); + const cLogger = await componentFileLogger(this.type, this.name, true, logConfig); + this.componentLogger = childLogger(cLogger, this.logger.labels); + stream.on('data', (d: LogDataPretty) => { + const {level, msg, line, labels, ...rest} = d; + if(d.labels.includes(this.loggerLabel)) { + this.componentLogger[this.componentLogger.levels.labels[d.level]]({...rest, labels: difference(labels, this.logger.labels)}, msg); + } + }); + } + } }