diff --git a/migrations/0050_mediaRef_isChapterToc.sql b/migrations/0050_mediaRef_isChapterToc.sql new file mode 100644 index 00000000..2186c3b3 --- /dev/null +++ b/migrations/0050_mediaRef_isChapterToc.sql @@ -0,0 +1,2 @@ +ALTER TABLE ONLY public."mediaRefs" + ADD COLUMN "isChapterToc" boolean DEFAULT NULL; diff --git a/migrations/0051_mediaRef_chaptersIndex.sql b/migrations/0051_mediaRef_chaptersIndex.sql new file mode 100644 index 00000000..478fdcda --- /dev/null +++ b/migrations/0051_mediaRef_chaptersIndex.sql @@ -0,0 +1,12 @@ +ALTER TABLE ONLY public."mediaRefs" + ADD COLUMN "chapterHash" uuid DEFAULT NULL; + +-- Create a unique index that allows NULL for the chapterHash column +CREATE UNIQUE INDEX chapterHash_3col_unique_idx + ON public."mediaRefs" ("episodeId", "isOfficialChapter", "chapterHash") + WHERE "mediaRefs"."isOfficialChapter" IS TRUE + AND "mediaRefs"."chapterHash" IS NOT NULL; + +-- Drop the deprecated index +ALTER TABLE public."mediaRefs" + DROP CONSTRAINT "mediaRef_index_episode_isOfficialChapter_startTime"; diff --git a/package.json b/package.json index c0254f2b..647dda36 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "podverse-api", - "version": "4.14.11", + "version": "4.14.12", "description": "Data API, database migration scripts, and backend services for all Podverse models.", "contributors": [ "Mitch Downey" @@ -147,6 +147,7 @@ "@koa/cors": "3.0.0", "@types/crypto-js": "^3.1.47", "@types/jest": "26.0.3", + "@types/lodash": "4.14.200", "@typescript-eslint/eslint-plugin": "2.x", "@typescript-eslint/parser": "2.x", "archiver": "^5.3.1", @@ -159,7 +160,7 @@ "class-validator": "0.14.0", "clean-webpack-plugin": "3.0.0", "cookie": "0.4.0", - "crypto-js": "~3.2.1", + "crypto-js": "~4.2.0", "csvtojson": "^2.0.10", "date-fns": "2.8.1", "docker-cli-js": "2.9.0", @@ -187,6 +188,7 @@ "koa-static": "5.0.0", "koa2-ratelimit": "0.9.1", "lint-staged": "9.5.0", + "lodash": "4.17.21", "manticoresearch": "^2.0.3", "moment": "^2.29.4", "node-fetch": "2.6.7", @@ -198,7 +200,7 @@ "paypal-rest-sdk": "2.0.0-rc.2", "pg": "8.7.3", "podcast-partytime": "4.6.2", - "podverse-shared": "^4.13.14", + "podverse-shared": "^4.14.9", "reflect-metadata": "0.1.13", "request": "^2.88.2", "request-promise-native": "1.0.8", @@ -211,7 +213,7 @@ "tsconfig-paths": "3.9.0", "tsconfig-paths-webpack-plugin": "3.2.0", "typeorm": "0.2.45", - "uuid": "3.3.3", + "uuid": "^9.0.1", "valid-url": "1.0.9", "validator": "13.7.0", "web-push": "^3.6.3", diff --git a/src/controllers/episode.ts b/src/controllers/episode.ts index 7f3dcb32..c8fb925f 100644 --- a/src/controllers/episode.ts +++ b/src/controllers/episode.ts @@ -1,15 +1,18 @@ import { getConnection, getRepository } from 'typeorm' import { config } from '~/config' -import { Episode, EpisodeMostRecent, MediaRef } from '~/entities' +import { Episode, EpisodeMostRecent, MediaRef, Podcast } from '~/entities' import { request } from '~/lib/request' import { addOrderByToQuery, getManticoreOrderByColumnName, removeAllSpaces } from '~/lib/utility' import { validateSearchQueryString } from '~/lib/utility/validation' import { manticoreWildcardSpecialCharacters, searchApi } from '~/services/manticore' import { liveItemStatuses } from './liveItem' import { createMediaRef, updateMediaRef } from './mediaRef' -import { getPodcast } from './podcast' +import { getPodcast, getPodcastByPodcastGuid } from './podcast' +import { getLightningKeysendValueItem } from 'podverse-shared' const createError = require('http-errors') const SqlString = require('sqlstring') +import { v5 as uuidv5, NIL as uuidNIL } from 'uuid' + const { superUserId } = config const relations = [ @@ -519,7 +522,127 @@ const removeEpisodes = async (episodes: any[]) => { ]) } -const retrieveLatestChapters = async (id) => { +const checkIfValidVTSRemoteItem = (valueTimeSplit: any) => { + return ( + valueTimeSplit && + valueTimeSplit.type === 'remoteItem' && + valueTimeSplit.startTime !== null && + valueTimeSplit.startTime >= 0 && + valueTimeSplit.duration !== null && + valueTimeSplit.duration >= 0 + ) +} + +const getLightningKeysendVTSAsChapters = async (episodeId: string) => { + try { + const episodeRepository = getRepository(Episode) + const episode = (await episodeRepository + .createQueryBuilder('episode') + .select('episode.id', 'id') + .select('episode.value', 'value') + .where('episode.id = :id', { id: episodeId }) + .getRawOne()) as Episode + + let valueTags = episode?.value || [] + + if (typeof valueTags === 'string') { + valueTags = JSON.parse(valueTags) + } + + const lightningKeysendValueTags = getLightningKeysendValueItem(valueTags) + const valueTimeSplits = lightningKeysendValueTags?.valueTimeSplits as any[] + + const vtsChapters: any[] = [] + if (valueTimeSplits && valueTimeSplits.length > 0) { + for (let i = 0; i <= valueTimeSplits.length; i++) { + try { + const valueTimeSplit = valueTimeSplits[i] + if (checkIfValidVTSRemoteItem(valueTimeSplit)) { + const startTime = Math.floor(valueTimeSplit.startTime) + const duration = Math.floor(valueTimeSplit.duration) + const endTime = startTime + duration + let title = '' + let imageUrl = '' + let linkUrl = '' + let remoteEpisodeId = '' + let remotePodcastId = '' + + let episode: Episode | null = null + let podcast: Podcast | null = null + + // TODO: is remoteItem missing from the partytime type? + const remoteItemGuid = valueTimeSplit.remoteItem?.itemGuid + const remoteFeedGuid = valueTimeSplit.remoteItem?.feedGuid + + const includeRelations = true + podcast = await getPodcastByPodcastGuid(remoteFeedGuid, includeRelations) + remotePodcastId = podcast.id + if (podcast?.id && remoteItemGuid && remoteFeedGuid) { + try { + episode = await getEpisodeByPodcastIdAndGuid(podcast.id, remoteItemGuid) + remoteEpisodeId = episode?.id || '' + } catch (error) { + // probably a 404 + // console.log('ep error', error) + } + } + const podcastAuthors = podcast?.authors + const episodeAuthors = episode?.authors + + let authorsTitle = '' + if (episodeAuthors && episodeAuthors.length > 0) { + for (const episodeAuthor of episodeAuthors) { + authorsTitle += `${episodeAuthor.name || ''},` + } + } else if (podcastAuthors && podcastAuthors.length > 0) { + for (const podcastAuthor of podcastAuthors) { + authorsTitle += `${podcastAuthor.name || ''},` + } + } + authorsTitle = authorsTitle.slice(0, -1) + + if (authorsTitle && episode?.title) { + title = `${authorsTitle} – ${episode.title}` + } else if (episode?.title) { + title = `${episode.title}` + } + + imageUrl = episode?.imageUrl || podcast?.imageUrl || podcast?.shrunkImageUrl || '' + + linkUrl = episode?.linkUrl || podcast?.linkUrl || '' + + const vtsChapter = { + id: `vts-${i}`, + endTime, + imageUrl, + isChapterToc: false, + isChapterVts: true, + linkUrl, + startTime, + title, + remoteEpisodeId, + remotePodcastId + } + + vtsChapters.push(vtsChapter) + } + } catch (error) { + // probably a 404 + // console.log('podcast error', error) + } + } + } + + return vtsChapters + } catch (error) { + console.log('getVTSAsChapters error', error) + } + + return [] +} + +// TOC = Table of Contents chapters +const retrieveLatestChapters = async (id, includeNonToc = true) => { const episodeRepository = getRepository(Episode) const mediaRefRepository = getRepository(MediaRef) @@ -538,23 +661,23 @@ const retrieveLatestChapters = async (id) => { ;(async function () { // Update the latest chapters only once every 1 hour for an episode. // If less than 1 hours, then just return the latest chapters from the database. - const halfDay = new Date().getTime() - 1 * 1 * 60 * 60 * 1000 // days hours minutes seconds milliseconds + const oneHour = new Date().getTime() - 1 * 1 * 60 * 60 * 1000 // days hours minutes seconds milliseconds const chaptersUrlLastParsedDate = new Date(chaptersUrlLastParsed).getTime() - if (chaptersUrl && (!chaptersUrlLastParsed || halfDay > chaptersUrlLastParsedDate)) { + if (chaptersUrl && (!chaptersUrlLastParsed || oneHour > chaptersUrlLastParsedDate)) { try { await episodeRepository.update(episode.id, { chaptersUrlLastParsed: new Date() }) const response = await request(chaptersUrl) const trimmedResponse = (response && response.trim()) || {} const parsedResponse = JSON.parse(trimmedResponse) - let { chapters: newChapters } = parsedResponse + const { chapters: newChapters } = parsedResponse if (newChapters) { const qb = mediaRefRepository .createQueryBuilder('mediaRef') .select('mediaRef.id', 'id') .addSelect('mediaRef.isOfficialChapter', 'isOfficialChapter') - .addSelect('mediaRef.startTime', 'startTime') + .addSelect('mediaRef.chapterHash', 'chapterHash') .where({ isOfficialChapter: true, episode: episode.id, @@ -564,6 +687,7 @@ const retrieveLatestChapters = async (id) => { // Temporarily hide all existing chapters, // then display the new valid ones at the end. + // TODO: can we remove / improve this? for (const existingChapter of existingChapters) { await updateMediaRef( { @@ -574,44 +698,46 @@ const retrieveLatestChapters = async (id) => { ) } - // NOTE: we are temporarily removing `toc: false` chapters until we add - // proper UX support in our client-side applications. - // chapters with `toc: false` are not considered part of the "table of contents", - // but our client-side apps currently expect chapters to behave as a table of contents. - newChapters = newChapters.filter((chapter) => { - return chapter.toc !== false - }) - - for (const newChapter of newChapters) { + for (let i = 0; i < newChapters.length; i++) { try { - const startTime = Math.round(newChapter.startTime) - // If a chapter with that startTime already exists, then update it. + const newChapter = newChapters[i] + const roundedStartTime = Math.floor(newChapter.startTime) + const isChapterToc = newChapter.toc === false ? false : newChapter.toc + const chapterHash = uuidv5(JSON.stringify(newChapter), uuidNIL) + + // If a chapter with that chapterHash already exists, then update it. // If it does not exist, then create a new mediaRef with isOfficialChapter = true. - const existingChapter = existingChapters.find((x) => x.startTime === startTime) + const existingChapter = existingChapters.find((x) => x.chapterHash === chapterHash) if (existingChapter && existingChapter.id) { await updateMediaRef( { + endTime: newChapter.endTime || null, id: existingChapter.id, imageUrl: newChapter.img || null, isOfficialChapter: true, isPublic: true, linkUrl: newChapter.url || null, - startTime, + startTime: roundedStartTime, title: newChapter.title, - episodeId: id + episodeId: id, + isChapterToc, + chapterHash }, superUserId ) } else { await createMediaRef({ + endTime: newChapter.endTime || null, imageUrl: newChapter.img || null, isOfficialChapter: true, isPublic: true, linkUrl: newChapter.url || null, - startTime, + startTime: roundedStartTime, title: newChapter.title, owner: superUserId, - episodeId: id + episodeId: id, + isChapterToc, + chapterHash }) } } catch (error) { @@ -625,7 +751,7 @@ const retrieveLatestChapters = async (id) => { } })() - const officialChaptersForEpisode = await mediaRefRepository + const qbOfficialChaptersForEpisode = mediaRefRepository .createQueryBuilder('mediaRef') .select('mediaRef.id') .addSelect('mediaRef.endTime') @@ -634,11 +760,19 @@ const retrieveLatestChapters = async (id) => { .addSelect('mediaRef.linkUrl') .addSelect('mediaRef.startTime') .addSelect('mediaRef.title') + .addSelect('mediaRef.isChapterToc') + .addSelect('mediaRef.chapterHash') .where({ isOfficialChapter: true, episode: id, isPublic: true }) + + if (!includeNonToc) { + qbOfficialChaptersForEpisode.andWhere('mediaRef.isChapterToc IS NOT FALSE') + } + + const officialChaptersForEpisode = await qbOfficialChaptersForEpisode .orderBy('mediaRef.startTime', 'ASC') .getManyAndCount() @@ -789,6 +923,7 @@ export { getEpisodesFromSearchEngine, getEpisodesWithLiveItemsWithMatchingGuids, getEpisodesWithLiveItemsWithoutMatchingGuids, + getLightningKeysendVTSAsChapters, refreshEpisodesMostRecentMaterializedView, removeDeadEpisodes, retrieveLatestChapters, diff --git a/src/entities/mediaRef.ts b/src/entities/mediaRef.ts index 51330388..e5c606a2 100644 --- a/src/entities/mediaRef.ts +++ b/src/entities/mediaRef.ts @@ -21,7 +21,10 @@ import { Author, Category, Episode, User, UserHistoryItem, UserNowPlayingItem, U import { generateShortId } from '~/lib/utility' @Entity('mediaRefs') -@Unique('mediaRef_index_episode_isOfficialChapter_startTime', ['episode', 'isOfficialChapter', 'startTime']) +// Deprecated: we no longer ensure uniqueness with startTime as it was too buggy +// @Unique('mediaRef_index_episode_isOfficialChapter_startTime', ['episode', 'isOfficialChapter', 'startTime']) +// Instead, we ensure uniqueness with a compound index on chapterHash column +@Unique('chapterHash_3col_unique_idx', ['episode', 'isOfficialChapter', 'chapterHash']) export class MediaRef { @PrimaryColumn('varchar', { default: generateShortId(), @@ -34,6 +37,14 @@ export class MediaRef { @Generated('increment') int_id: number + // If chapters should update, we overwrite the existing chapter + // at that chapterHash. This is just to prevent us from creating an + // endless amount of chapter rows in our database whenever we + // reparse a chapters file. + @Index() + @Column({ type: 'uuid', nullable: true }) + chapterHash: string + @ValidateIf((a) => a.endTime != null) @IsInt() @Min(1) @@ -45,6 +56,13 @@ export class MediaRef { @Column({ nullable: true }) imageUrl?: string + // If a chapter has true or null set for toc, then it should be handled as part of + // the "table of contents" in the app UX. If it is set to false, then it should not. + // podcasting 2.0 spec: https://github.com/Podcastindex-org/podcast-namespace/blob/main/chapters/jsonChapters.md#optional-attributes-1 + @Index() + @Column({ default: null, nullable: true }) + isChapterToc: boolean + @Index() @Column({ default: null, nullable: true }) isOfficialChapter: boolean diff --git a/src/middleware/auth/resetPassword.ts b/src/middleware/auth/resetPassword.ts index 240afd11..8807859a 100644 --- a/src/middleware/auth/resetPassword.ts +++ b/src/middleware/auth/resetPassword.ts @@ -1,4 +1,5 @@ import { addSeconds } from 'date-fns' +import { v4 as uuidv4 } from 'uuid' import { config } from '~/config' import { getUserByEmail, @@ -8,7 +9,6 @@ import { } from '~/controllers/user' import { emitRouterError } from '~/lib/errors' import { sendResetPasswordEmail } from '~/services/auth/sendResetPasswordEmail' -const uuidv4 = require('uuid/v4') export const resetPassword = async (ctx) => { const { password, resetPasswordToken } = ctx.request.body diff --git a/src/middleware/auth/signUpUser.ts b/src/middleware/auth/signUpUser.ts index f2714379..f28d90cb 100644 --- a/src/middleware/auth/signUpUser.ts +++ b/src/middleware/auth/signUpUser.ts @@ -1,12 +1,12 @@ import { addSeconds } from 'date-fns' import { Connection } from 'typeorm' +import { v4 as uuidv4 } from 'uuid' import isEmail from 'validator/lib/isEmail' import { config } from '~/config' import { User } from '~/entities' import { CustomStatusError, emitRouterError } from '~/lib/errors' import { createUser } from '~/controllers/user' import { sendVerificationEmail } from '~/services/auth/sendVerificationEmail' -const uuidv4 = require('uuid/v4') const emailExists = async (conn: Connection, email) => { const user = await conn.getRepository(User).findOne({ email }) diff --git a/src/middleware/auth/verification.ts b/src/middleware/auth/verification.ts index ee0e0ee3..228aec98 100644 --- a/src/middleware/auth/verification.ts +++ b/src/middleware/auth/verification.ts @@ -1,9 +1,9 @@ import { addSeconds } from 'date-fns' +import { v4 as uuidv4 } from 'uuid' import { config } from '~/config' import { getUserByEmail, getUserByVerificationToken, updateUserEmailVerificationToken } from '~/controllers/user' import { emitRouterError } from '~/lib/errors' import { sendVerificationEmail } from '~/services/auth/sendVerificationEmail' -const uuidv4 = require('uuid/v4') export const sendVerification = async (ctx, email) => { try { diff --git a/src/routes/episode.ts b/src/routes/episode.ts index 99dbe222..dadc3036 100644 --- a/src/routes/episode.ts +++ b/src/routes/episode.ts @@ -11,6 +11,7 @@ import { getEpisodesByCategoryIds, getEpisodesByPodcastIds, getEpisodesFromSearchEngine, + getLightningKeysendVTSAsChapters, retrieveLatestChapters } from '~/controllers/episode' import { parseQueryPageOptions } from '~/middleware/parseQueryPageOptions' @@ -18,7 +19,8 @@ import { validateEpisodeSearch } from '~/middleware/queryValidation/search' import { parseNSFWHeader } from '~/middleware/parseNSFWHeader' import { getThreadcap } from '~/services/socialInteraction/threadcap' import { request } from '~/lib/request' - +// import { MediaRef } from '~/entities' +const lodash = require('lodash') const router = new Router({ prefix: `${config.apiPrefix}${config.apiVersion}/episode` }) const delimitKeys = ['authors', 'mediaRefs'] @@ -69,7 +71,16 @@ router.get('/:id', parseNSFWHeader, async (ctx) => { router.get('/:id/retrieve-latest-chapters', async (ctx) => { try { if (!ctx.params.id) throw new Error('An episodeId is required.') - const latestChapters = await retrieveLatestChapters(ctx.params.id) + const includeNonToc = ctx.query?.includeNonToc === 'true' + const latestChapters = await retrieveLatestChapters(ctx.params.id, includeNonToc) + + if (!!ctx.query.includeLightningKeysendVTS) { + const vtsChapters = await getLightningKeysendVTSAsChapters(ctx.params.id) + latestChapters[0] = latestChapters[0]?.concat(vtsChapters) + latestChapters[1] = latestChapters[0]?.length + } + + latestChapters[0] = lodash.orderBy(latestChapters[0], ['startTime'], 'asc') ctx.body = latestChapters } catch (error) { emitRouterError(error, ctx) diff --git a/src/routes/podcastIndex.ts b/src/routes/podcastIndex.ts index 041d6c9e..f0883171 100644 --- a/src/routes/podcastIndex.ts +++ b/src/routes/podcastIndex.ts @@ -29,6 +29,8 @@ router.get('/podcast/by-id/:id', podcastByIdLimiter, async (ctx) => { }) // Get value tags from Podcast Index by feed and item guids. +// NOTE: this is more accurately "get remote item data" at this point, +// as value tags are not strictly required for "remote items" to be useful. router.get('/value/by-guids', async (ctx) => { const podcastGuid = ctx.query.podcastGuid as string const episodeGuid = ctx.query.episodeGuid as string @@ -38,7 +40,6 @@ router.get('/value/by-guids', async (ctx) => { const data = await getValueTagForItemFromPodcastIndexByGuids(podcastGuid, episodeGuid) ctx.body = data } catch (error) { - console.log('error', error) emitRouterError(error, ctx) } } else if (podcastGuid) { diff --git a/src/seeds/qa/podcastIndexIds.ts b/src/seeds/qa/podcastIndexIds.ts index d2c907da..eb4464cf 100644 --- a/src/seeds/qa/podcastIndexIds.ts +++ b/src/seeds/qa/podcastIndexIds.ts @@ -5,14 +5,14 @@ export const podcastIndexIds = [ 5718023, 387129, 3662287, 160817, 150842, 878147, 487548, 167137, 465231, 767934, 577105, 54545, 650774, 955598, 3758236, 203827, 879740, 393504, 575694, 921030, 41504, 5341434, 757675, 174725, 920666, 1333070, 227573, 5465405, - 5498327, 5495489, 556715, 5485175, 202764, 830124, 66844, 4169501 + 5498327, 5495489, 556715, 5485175, 202764, 830124, 66844, 4169501, 6524027 ] export const podcastIndexIdsQuick = [5718023, 387129, 3662287, 160817] export const podcastIndexIdsWithLiveItems = [ 4935828, 5495489, 162612, 5461087, 486332, 480983, 3727160, 5496786, 901876, 5498327, 4207213, 5710520, 5465405, 5485175, 574891, 920666, 540927, 4432692, 5718023, 41504, 3756449, 150842, 937170, 946122, 5373053, 624721, 5700613, - 288180, 955598 + 288180, 955598, 6524027 ] export const podcastIndexIdsWithLiveItemsQuick = [4935828, 5495489, 162612, 5461087] diff --git a/src/seeds/qa/users.ts b/src/seeds/qa/users.ts index 8de64f56..64135a08 100644 --- a/src/seeds/qa/users.ts +++ b/src/seeds/qa/users.ts @@ -78,7 +78,10 @@ export const generateQAUsers = async () => { export const getQAUserByEmail = async (email: string) => { const userRepository = getRepository(User) - return await userRepository.findOne({ email }) + return await userRepository.findOne({ + where: { email }, + select: ['id', 'email'] + }) } const enrichRowExportData = (userLite: RowExportData) => { diff --git a/src/services/bitpay.ts b/src/services/bitpay.ts index 55c98c2f..a1097ebc 100644 --- a/src/services/bitpay.ts +++ b/src/services/bitpay.ts @@ -2,9 +2,9 @@ const fs = require('fs') const bitpay = require('bitpay-rest') const bitauth = require('bitauth') import { config } from '~/config' +import { v4 as uuidv4 } from 'uuid' const { bitpayConfig } = config const { apiKeyPassword, apiKeyPath, currency, notificationURL, price, redirectURL } = bitpayConfig -const uuidv4 = require('uuid/v4') // NOTE: It's necessary to decrypt your key even if you didn't enter a password // when you generated it. If you did specify a password, pass it as the diff --git a/src/services/parser.ts b/src/services/parser.ts index 1c71369f..18650d49 100644 --- a/src/services/parser.ts +++ b/src/services/parser.ts @@ -35,7 +35,8 @@ import { import { getEpisodeByPodcastIdAndGuid, getEpisodesWithLiveItemsWithMatchingGuids, - getEpisodesWithLiveItemsWithoutMatchingGuids + getEpisodesWithLiveItemsWithoutMatchingGuids, + retrieveLatestChapters } from '~/controllers/episode' import { getLiveItemByGuid } from '~/controllers/liveItem' import { PhasePendingChat } from 'podcast-partytime/dist/parser/phase/phase-pending' @@ -522,15 +523,15 @@ export const parseFeedUrl = async (feedUrl, forceReparsing = false, cacheBust = await feedUrlRepo.update(feedUrl.id, cleanedFeedUrl) logPerformance('feedUrlRepo.update', _logEnd) + // Retrieve the episode to make sure we have the episode.id + const latestEpisodeWithId = await getEpisodeByPodcastIdAndGuid(podcast.id, latestEpisodeGuid) + if (shouldSendNewEpisodeNotification) { logPerformance('sendNewEpisodeDetectedNotification', _logStart) const podcastShrunkImageUrl = podcast.shrunkImageUrl const podcastFullImageUrl = podcast.imageUrl const episodeFullImageUrl = latestEpisodeImageUrl - // Retrieve the episode to make sure we have the episode.id - const latestEpisodeWithId = await getEpisodeByPodcastIdAndGuid(podcast.id, latestEpisodeGuid) - if (latestEpisodeWithId?.id) { await sendNewEpisodeDetectedNotification({ podcastId: podcast.id, @@ -596,6 +597,15 @@ export const parseFeedUrl = async (feedUrl, forceReparsing = false, cacheBust = } } logPerformance('newEpisodes updateSoundBites', _logEnd) + + logPerformance('retrieveLatestChapters', _logStart) + // Run retrieveLatestChapters only for the latest episode to make sure + // chapters are pre-populated in our database. Otherwise the first person + // who plays the episode will not see chapters. + if (latestEpisodeWithId?.id) { + await retrieveLatestChapters(latestEpisodeWithId.id) + } + logPerformance('retrieveLatestChapters', _logEnd) } catch (error) { throw error } finally { @@ -925,6 +935,8 @@ const getMostRecentEpisodeDataFromFeed = (meta, episodes) => { } } + // TODO: call retrieve chapters once + return { mostRecentEpisodeTitle, mostRecentEpisodePubDate, mostRecentUpdateDateFromFeed } } diff --git a/src/services/podcastIndex.ts b/src/services/podcastIndex.ts index 09cf5858..53677c64 100644 --- a/src/services/podcastIndex.ts +++ b/src/services/podcastIndex.ts @@ -195,6 +195,13 @@ export const getPodcastValueTagForPodcastIndexId = async (id: string) => { return pvValueTagArray } +// These getValueTagFor* services were intended for getting "value tag" info from Podcast Index, +// but at this point they more broadly is for retrieving the "remote item" data +// our client side apps need. The most common use case involves needing value tags +// for value time splits (VTS), but we also return additional data as the +// second item in the response data array, which gets handled as a "chapter" +// in the client side apps, to display to the listener which value time split track +// (usually a song) is playing right now. export const getValueTagForChannelFromPodcastIndexByGuids = async (podcastGuid: string) => { const url = `${podcastIndexConfig.baseUrl}/podcasts/byguid?guid=${podcastGuid}` let podcastValueTag: ValueTag[] | null = null @@ -216,6 +223,7 @@ export const getValueTagForChannelFromPodcastIndexByGuids = async (podcastGuid: return podcastValueTag } +// see note above export const getValueTagForItemFromPodcastIndexByGuids = async (podcastGuid: string, episodeGuid: string) => { const url = `${podcastIndexConfig.baseUrl}/episodes/byguid?podcastguid=${podcastGuid}&guid=${episodeGuid}` let episodeValueTag: ValueTag[] | null = null diff --git a/yarn.lock b/yarn.lock index 9d39246c..40a46ba8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -973,6 +973,16 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.119.tgz#be847e5f4bc3e35e46d041c394ead8b603ad8b39" integrity sha512-Z3TNyBL8Vd/M9D9Ms2S3LmFq2sSMzahodD6rCS9V2N44HUMINb75jNkSuwAx7eo2ufqTdfOdtGQpNbieUjPQmw== +"@types/lodash@4.14.172": + version "4.14.172" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.172.tgz#aad774c28e7bfd7a67de25408e03ee5a8c3d028a" + integrity sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw== + +"@types/lodash@4.14.200": + version "4.14.200" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.200.tgz#435b6035c7eba9cdf1e039af8212c9e9281e7149" + integrity sha512-YI/M/4HRImtNf3pJgbF+W6FrXovqj+T+/HpENLTooK9PnkacBsDpeP3IpHab40CClUfhNmdM2WTNP2sa2dni5Q== + "@types/mime@*": version "3.0.1" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-3.0.1.tgz#5f8f2bca0a5863cb69bc0b0acd88c96cb1d4ae10" @@ -2980,10 +2990,10 @@ crypto-browserify@^3.11.0: randombytes "^2.0.0" randomfill "^1.0.3" -crypto-js@~3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-3.2.1.tgz#9408ed6695905ae97e05e8a6ca11937819b216a7" - integrity sha512-fIEXOyiXnmPbPk2+q8t97VYDSo8naqvI+2v0AJeLraQzhuL/GZ2qgcRpEadVQ7r8pXwBOHVjwOdyAXYYb3DWiQ== +crypto-js@~4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.2.0.tgz#4d931639ecdfd12ff80e8186dba6af2c2e856631" + integrity sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q== crypto-random-string@^1.0.0: version "1.0.0" @@ -6712,7 +6722,7 @@ lodash.union@^4.6.0: resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88" integrity sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw== -lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.3, lodash@^4.17.5, lodash@^4.7.0: +lodash@4.17.21, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.3, lodash@^4.17.5, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -8079,13 +8089,15 @@ podcast-partytime@4.6.2: ramda "^0.27.1" tiny-invariant "^1.2.0" -podverse-shared@^4.13.14: - version "4.13.14" - resolved "https://registry.yarnpkg.com/podverse-shared/-/podverse-shared-4.13.14.tgz#7ef2ce8597d4e42374f10d3432ba2aa1ef7e8688" - integrity sha512-1ZW0ooZIz98tMSwtavgU7Kig5z9soaMkDbni/nPilfgJ5rcqIkyi1O9OIwcb1xRExBMO2U2vrQqANpAjbt0l0Q== +podverse-shared@^4.14.9: + version "4.14.9" + resolved "https://registry.yarnpkg.com/podverse-shared/-/podverse-shared-4.14.9.tgz#19e3823cbe9f18fcb43daa93e56ca92b9be2797d" + integrity sha512-AZplb5KhkM41Y4+fSHeLvKu24T2TMcUgA0eFT3p7cVfPODGYiAkUgJPo4UXydMKJ5oOI/WOzyrYvWdw+0iHDnQ== dependencies: + "@types/lodash" "4.14.172" he "^1.2.0" html-entities "^2.3.2" + lodash "4.17.21" striptags "^3.2.0" typescript "3.9.7" @@ -10231,11 +10243,6 @@ uuid@3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== -uuid@3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866" - integrity sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ== - uuid@^3.1.0, uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -10251,6 +10258,11 @@ uuid@^8.3.0, uuid@^8.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +uuid@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + v8-compile-cache@2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.3.tgz#00f7494d2ae2b688cfe2899df6ed2c54bef91dbe"