Merge pull request #739 from podverse/develop

Release v4.16.9

package.json

@@ -1,6 +1,6 @@
 {
   "name": "podverse-api",
-  "version": "4.16.8",
+  "version": "4.16.9",
   "description": "Data API, database migration scripts, and backend services for all Podverse models.",
   "contributors": [
     "Mitch Downey"
@@ -28,6 +28,7 @@
     "dev:scripts:addNonPodcastIndexFeedUrlsToPriorityQueue": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/queue/addNonPodcastIndexFeedUrlsToPriorityQueue.ts",
     "dev:scripts:addRecentlyUpdatedFeedUrlsToPriorityQueue": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/addRecentlyUpdatedFeedUrlsToPriorityQueue.ts",
     "dev:scripts:backupDatabase": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/backupDatabase/backupDatabase.ts",
+    "dev:scripts:devtools:generateDeadRowsCSV": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/devtools/db/generateDeadRowsCSV.ts",
     "dev:scripts:dropAndRecreateEpisodesMostRecentMaterializedView": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/episodes/dropAndRecreateEpisodesMostRecentMaterializedView.ts",
     "dev:scripts:hideDeadPodcasts": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/hideDeadPodcasts.ts",
     "dev:scripts:parseFeedUrlsByPodcastIds": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/parser/parseFeedUrlsByPodcastIds.ts",
@@ -43,6 +44,7 @@
     "dev:scripts:refreshMediaRefsVideosMaterializedView": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/mediaRefs_videos/refreshMediaRefsVideosMaterializedView.ts",
     "dev:scripts:removeDeadChapters": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/mediaRefs/removeDeadChapters.ts",
     "dev:scripts:removeDeadEpisodes": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/episodes/removeDeadEpisodes.ts",
+    "dev:scripts:removePodcastsByIds": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcasts/removePodcastsByIds.ts",
     "dev:scripts:syncWithFeedUrlsCSVDump": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/syncWithFeedUrlsCSVDump.ts",
     "dev:scripts:updateRecentEpisodesTables": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/recentEpisodes/updateRecentEpisodesTables.ts",
     "dev:seeds:categories": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/seeds/categories.ts",
@@ -62,6 +64,7 @@
     "scripts:addNonPodcastIndexFeedUrlsToPriorityQueue": "ts-node -r tsconfig-paths/register ./src/scripts/queue/addNonPodcastIndexFeedUrlsToPriorityQueue.ts",
     "scripts:addRecentlyUpdatedFeedUrlsToPriorityQueue": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/addRecentlyUpdatedFeedUrlsToPriorityQueue.ts",
     "scripts:backupDatabase": "ts-node -r tsconfig-paths/register ./src/scripts/backupDatabase/backupDatabase.ts",
+    "scripts:devtools:generateDeadRowsCSV": "ts-node -r tsconfig-paths/register ./src/scripts/devtools/db/generateDeadRowsCSV.ts",
     "scripts:dropAndRecreateEpisodesMostRecentMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/dropAndRecreateEpisodesMostRecentMaterializedView.ts",
     "scripts:hideDeadPodcasts": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/hideDeadPodcasts.ts",
     "scripts:parseFeedUrlsByPodcastIds": "ts-node -r tsconfig-paths/register ./src/scripts/parser/parseFeedUrlsByPodcastIds.ts",
@@ -76,6 +79,7 @@
     "scripts:refreshEpisodesMostRecentMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/refreshEpisodesMostRecentMaterializedView.ts",
     "scripts:refreshMediaRefsVideosMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/mediaRefs_videos/refreshMediaRefsVideosMaterializedView.ts",
     "scripts:removeDeadEpisodes": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/removeDeadEpisodes.ts",
+    "scripts:removePodcastsByIds": "ts-node -r tsconfig-paths/register ./src/scripts/podcasts/removePodcastsByIds.ts",
     "scripts:syncWithFeedUrlsCSVDump": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/syncWithFeedUrlsCSVDump.ts",
     "scripts:updateRecentEpisodesTables": "ts-node -r tsconfig-paths/register ./src/scripts/recentEpisodes/updateRecentEpisodesTables.ts",
     "seeds:categories": "ts-node -r tsconfig-paths/register ./src/seeds/categories.ts",
@@ -177,6 +181,7 @@
     "isomorphic-fetch": "^3.0.0",
     "jest": "26.1.0",
     "joi": "14.3.1",
+    "json-2-csv": "^5.5.1",
     "koa": "2.11.0",
     "koa-bodyparser": "4.2.1",
     "koa-helmet": "5.2.0",

src/controllers/devtools/db.ts

@@ -0,0 +1,20 @@
+import { getConnection } from 'typeorm'
+import { _logEnd, _logStart, logPerformance } from '~/lib/utility'
+const json2csv = require('json-2-csv')
+
+const generateDeadRowsCSV = async () => {
+  logPerformance('generateDeadRowCSV', _logStart)
+  const em = await getConnection().createEntityManager()
+  const results = await em.query(`
+    SELECT n_live_tup, n_dead_tup, relname
+    FROM pg_stat_all_tables
+    ORDER BY n_dead_tup DESC;
+  `)
+
+  const csv = json2csv.json2csv(results)
+  console.log('csv', csv)
+
+  logPerformance('generateDeadRowCSV', _logEnd)
+}
+
+export { generateDeadRowsCSV }

src/controllers/podcast.ts

@@ -2,7 +2,14 @@ import { getRepository, In } from 'typeorm'
 import { config } from '~/config'
 import { getUserSubscribedPodcastIds } from '~/controllers/user'
 import { FeedUrl, Podcast, User } from '~/entities'
-import { addOrderByToQuery, getManticoreOrderByColumnName, removeAllSpaces } from '~/lib/utility'
+import {
+  _logEnd,
+  _logStart,
+  addOrderByToQuery,
+  getManticoreOrderByColumnName,
+  logPerformance,
+  removeAllSpaces
+} from '~/lib/utility'
 import { validateSearchQueryString } from '~/lib/utility/validation'
 import { manticoreWildcardSpecialCharacters, searchApi } from '~/services/manticore'
 import { deleteNotification } from './notification'
@@ -506,6 +513,21 @@ const getPodcastWebUrl = async ({ podcastGuid, podcastIndexId }: GetPodcastWebUr
   }
 }
 
+const removePodcasts = async (ids: string[]) => {
+  logPerformance('removePodcasts', _logStart)
+  const repository = getRepository(Podcast)
+  for (const id of ids) {
+    try {
+      logPerformance(`removePodcast ${id}`, _logStart)
+      await repository.delete(id)
+      logPerformance(`removePodcast ${id}`, _logEnd)
+    } catch (error) {
+      console.log(`error removing podcast ${id}`, error)
+    }
+  }
+  logPerformance('removePodcasts', _logEnd)
+}
+
 export {
   findPodcastsByFeedUrls,
   getPodcast,
@@ -519,5 +541,6 @@ export {
   subscribeToPodcast,
   toggleSubscribeToPodcast,
   updateHasPodcastIndexValueTags,
-  getPodcastWebUrl
+  getPodcastWebUrl,
+  removePodcasts
 }

src/routes/tools.ts

@@ -25,7 +25,7 @@ router.post('/findHTTPS', async (ctx) => {
 
 const extractHTTPSFromURL = async (url, tries) => {
   try {
-    const res = await request(url, { followRedirect: false, method: 'head' })
+    const res = await request(url, { followRedirect: true, method: 'head' })
     if (tries > 5) {
       throw new createError.NotFound('Secure URL for ' + url + ' was not found. Too many redirects')
     } else if (!res.location) {
@@ -35,7 +35,7 @@ const extractHTTPSFromURL = async (url, tries) => {
         try {
           const attemptHttpsUrl = url.replace('http://', 'https://')
           // If no error is thrown,then assume it is a valid url.
-          await request(attemptHttpsUrl, { followRedirect: false, method: 'head' })
+          await request(attemptHttpsUrl, { followRedirect: true, method: 'head' })
           return attemptHttpsUrl
         } catch (error) {
           throw new createError.NotFound('Secure URL for ' + url + ' was not found.')

src/scripts/devtools/db/generateDeadRowsCSV.ts

@@ -0,0 +1,17 @@
+import { generateDeadRowsCSV as generateDeadRowsCSVCont } from '~/controllers/devtools/db'
+import { connectToDb } from '~/lib/db'
+;(async function () {
+  const generateDeadRowCSV = async () => {
+    try {
+      const connection = await connectToDb()
+
+      if (connection && connection.isConnected) {
+        await generateDeadRowsCSVCont()
+      }
+    } catch (error) {
+      console.log(error)
+    }
+  }
+
+  generateDeadRowCSV()
+})()

src/scripts/podcasts/removePodcastsByIds.ts

@@ -0,0 +1,25 @@
+import { removePodcasts } from '~/controllers/podcast'
+import { connectToDb } from '~/lib/db'
+;(async function () {
+  const removePodcastsByIds = async () => {
+    try {
+      const podcastIdsString = process.argv.length > 2 ? process.argv[2] : ''
+
+      if (!podcastIdsString) {
+        console.log('Provide a comma delimited string of podcastIds as a parameter.')
+        return
+      }
+
+      const podcastIds = podcastIdsString.split(',')
+      const connection = await connectToDb()
+
+      if (connection && connection.isConnected) {
+        await removePodcasts(podcastIds)
+      }
+    } catch (error) {
+      console.log(error)
+    }
+  }
+
+  removePodcastsByIds()
+})()

yarn.lock

@@ -3227,6 +3227,11 @@ dedent@^0.7.0:
   resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c"
   integrity sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==
 
+[email protected]:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/deeks/-/deeks-3.1.0.tgz#ecc47c600bcc6dbda656c6d86bbc96cbe126fd41"
+  integrity sha512-e7oWH1LzIdv/prMQ7pmlDlaVoL64glqzvNgkgQNgyec9ORPHrT2jaOqMtRyqJuwWjtfb6v+2rk9pmaHj+F137A==
+
 deep-eql@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df"
@@ -3404,6 +3409,11 @@ dir-glob@^3.0.1:
   dependencies:
     path-type "^4.0.0"
 
+[email protected]:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/doc-path/-/doc-path-4.1.1.tgz#5be8c1671877f6b719af5a6077c41904be8913ac"
+  integrity sha512-h1ErTglQAVv2gCnOpD3sFS6uolDbOKHDU1BZq+Kl3npPqroU3dYL42lUgMfd5UimlwtRgp7C9dLGwqQ5D2HYgQ==
+
 [email protected]:
   version "2.9.0"
   resolved "https://registry.yarnpkg.com/docker-cli-js/-/docker-cli-js-2.9.0.tgz#1584461134cbb6beb548fed2c33de8f5593280bf"
@@ -6276,6 +6286,14 @@ jsesc@^2.5.1:
   resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4"
   integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==
 
+json-2-csv@^5.5.1:
+  version "5.5.1"
+  resolved "https://registry.yarnpkg.com/json-2-csv/-/json-2-csv-5.5.1.tgz#2ad4e69d4b2bb4590203948a36f11347a4d7aed9"
+  integrity sha512-KgAtAXTQopRwe90gh8SgjRSxgt9bUWbGAPMo9W0TZLA8SqiQH7khtagFfeEUjG3NBPwJu/+9uX5pMvunKaPvrQ==
+  dependencies:
+    deeks "3.1.0"
+    doc-path "4.1.1"
+
 json-bigint@^0.3.0:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-0.3.1.tgz#0c1729d679f580d550899d6a2226c228564afe60"