From 12b7f55f7061451a56b1e5c56faa013530ade7c7 Mon Sep 17 00:00:00 2001 From: Zhoukun Cheng <chengzhoukun@gmail.com> Date: Thu, 2 May 2024 19:49:40 +0800 Subject: [PATCH 01/10] fix(route/linkedin): handle missing elements in jobs parsing (#15438) --- lib/routes/linkedin/utils.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/routes/linkedin/utils.ts b/lib/routes/linkedin/utils.ts index d069ddb2280ae7..40ef3db80e55f1 100644 --- a/lib/routes/linkedin/utils.ts +++ b/lib/routes/linkedin/utils.ts @@ -87,11 +87,11 @@ function parseJobSearch(data) { const jobs = $('li') .map((i, elem) => { const elemHtml = $(elem); - const link = elemHtml.find('a.base-card__full-link').attr('href').split('?')[0]; - const title = elemHtml.find('h3.base-search-card__title').text().trim(); - const company = elemHtml.find('h4.base-search-card__subtitle').text().trim(); - const location = elemHtml.find('span.job-search-card__location').text().trim(); - const pubDate = elemHtml.find('time').attr('datetime'); + const link = elemHtml.find('a.base-card__full-link, a.base-card--link')?.attr('href')?.split('?')[0]; + const title = elemHtml.find('h3.base-search-card__title')?.text()?.trim(); + const company = elemHtml.find('h4.base-search-card__subtitle')?.text()?.trim(); + const location = elemHtml.find('span.job-search-card__location')?.text()?.trim(); + const pubDate = elemHtml.find('time')?.attr('datetime'); return new Job(title, link, company, location, pubDate); }) From 1f72365451acd8394c8d0374a4e380b8197b7f9f Mon Sep 17 00:00:00 2001 From: sddzhyc <41501986+sddzhyc@users.noreply.github.com> Date: Thu, 2 May 2024 20:42:28 +0800 Subject: [PATCH 02/10] =?UTF-8?q?feat(route):=20Add=20=E4=B8=AD=E5=9B=BD?= =?UTF-8?q?=E7=9F=B3=E6=B2=B9=E5=A4=A7=E5=AD=A6=EF=BC=88=E5=8D=8E=E4=B8=9C?= =?UTF-8?q?=EF=BC=89=E6=95=99=E5=8A=A1=E5=A4=84=E9=80=9A=E7=9F=A5=E5=85=AC?= =?UTF-8?q?=E5=91=8A=20(#15427)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加upc/jwc路由 * 规范代码风格 * Apply suggestions from code review Co-authored-by: Tony <TonyRL@users.noreply.github.com> * Fixing the issues from reviews * fix link and timezone conversion --------- --- lib/routes/upc/jwc.ts | 121 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 lib/routes/upc/jwc.ts diff --git a/lib/routes/upc/jwc.ts b/lib/routes/upc/jwc.ts new file mode 100644 index 00000000000000..e96772d2551a90 --- /dev/null +++ b/lib/routes/upc/jwc.ts @@ -0,0 +1,121 @@ +// 导入必要的模组 +import { Route } from '@/types'; +import got from '@/utils/got'; +import { load } from 'cheerio'; +import { parseDate } from '@/utils/parse-date'; +import cache from '@/utils/cache'; +import timezone from '@/utils/timezone'; + +const typeDict = { + tzgg: '', // 默认为所有通知 + '18519': '教学·运行-', // 教学·运行 + '18520': '学业·学籍-', // 学业·学籍 + '18521': '教学·研究-', // 教学·研究 + '18522': '课程·教材-', // 课程·教材 + '18523': '实践·教学-', // 实践·教学 + '18524': '创新·创业-', // 创新·创业 + yywwz: '语言·文字-', // 语言·文字 + jxwjy: '继续·教育-', // 继续·教育 + bkwzs: '本科·招生-', // 本科·招生 +}; + +// module.exports = async (ctx) => { +const handler = async (ctx) => { + // 从 URL 参数中获取通知分类 + const { type = 'tzgg' } = ctx.req.param(); + // console.log(type); + const baseUrl = 'https://jwc.upc.edu.cn'; + const { data: response } = await got(`${baseUrl}/${type}/list.htm`); + // console.log(`${baseUrl}/${typeDict[type]}/list.htm`); + const $ = load(response); + // const listItems = $('ul.news_list').find('li'); + // console.log(`List item count: ${listItems.length}`); + // const list = $('ul.news_list') 只会得到第一个li + const list = $('ul.news_list') + .find('li') + // 使用“toArray()”方法将选择的所有 DOM 元素以数组的形式返回。 + .toArray() + // 使用“map()”方法遍历数组,并从每个元素中解析需要的数据。 + .map((item) => { + // console.log(item); + item = $(item); + const a = item.find('a').first(); + let linkStr = a.attr('href'); + // 若链接不是以http开头,则加上前缀 + if (a.attr('href').startsWith('http://')) { + // 改为https访问 + linkStr.replace('http://', 'https://'); + } else { + linkStr = `${baseUrl}${a.attr('href')}`; + } + return { + title: a.text(), + link: linkStr, + pubDate: timezone(parseDate(item.find('.news_meta').text()), +8), // 添加发布日期查询 + }; + }); + + const items = await Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link); + const $ = load(response); + // 选择类名为“comment-body”的第一个元素 + item.description = $('.read').first().html(); + // item.pubDate = $('.arti_update').html() === null ? '' : $('.arti_update').html().slice(5, 15); + // item.publisher = $('.arti_publisher').html(); + item.author = $('.arti_publisher').html(); + // console.log($('.arti_update').html().slice(5, 15)); + // 上面每个列表项的每个属性都在此重用, + // 并增加了一个新属性“description” + return item; + }) + ) + ); + + /* ctx.state.data = { + // 源标题 + title: `${typeDict[type]}教务处通知-中国石油大学(华东)`, + // 源链接 + link: `https://jwc.upc.edu.cn/tzgg/list.htm`, + // 源文章 + item: items, + }; */ + + return { + // 源标题 + title: `${typeDict[type]}教务处通知-中国石油大学(华东)`, + // 源链接 + link: `${baseUrl}/${type}/list.htm`, + // 源文章 + item: items, + }; +}; + +export const route: Route = { + path: '/jwc/:type?', + categories: ['university'], + example: '/upc/jwc/tzgg', + parameters: { type: '分类,见下表,其值与对应网页url路径参数一致,默认为所有通知' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['jwc.upc.edu.cn', 'jwc.upc.edu.cn/:type/list.htm'], + target: '/jwc/:type?', + }, + ], + name: '教务处通知公告', + maintainers: ['sddzhyc'], + description: `| 所有通知 | 教学·运行 | 学业·学籍 | 教学·研究 | 课程·教材 | 实践·教学 | 创新·创业 | 语言·文字 | 继续·教育 | 本科·招生 | + | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | + | tzgg | 18519 | 18520 | 18521 | 18522 | 18523 | 18524 | yywwz | jxwjy | bkwzs |`, + url: 'jwc.upc.edu.cn/tzgg/list.htm', + handler, +}; From 4a8c56122e4d5542913091bee3705b62ae395c71 Mon Sep 17 00:00:00 2001 From: Ethan <yif.wang@icloud.com> Date: Thu, 2 May 2024 06:08:24 -0700 Subject: [PATCH 03/10] fix(route): techcrunch cleanup useless nav element (#15441) --- lib/routes/techcrunch/news.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/routes/techcrunch/news.ts b/lib/routes/techcrunch/news.ts index 5780e78f6b99eb..09ff9ab9031b85 100644 --- a/lib/routes/techcrunch/news.ts +++ b/lib/routes/techcrunch/news.ts @@ -44,6 +44,8 @@ async function handler() { const description = $('#root'); description.find('.article__title').remove(); description.find('.article__byline__meta').remove(); + description.find('.mobile-header-nav').remove(); + description.find('.desktop-nav').remove(); return { title: item.title, pubDate: item.pubDate, From b251ae16ff245ca8f3c6e7b8c2f35e43a3e4d9c7 Mon Sep 17 00:00:00 2001 From: Ethan <yif.wang@icloud.com> Date: Thu, 2 May 2024 06:50:19 -0700 Subject: [PATCH 04/10] feat(route): magnumphotos (#15442) --- lib/routes/magnumphotos/magazine.ts | 63 ++++++++++++++++++++++++++++ lib/routes/magnumphotos/namespace.ts | 6 +++ 2 files changed, 69 insertions(+) create mode 100644 lib/routes/magnumphotos/magazine.ts create mode 100644 lib/routes/magnumphotos/namespace.ts diff --git a/lib/routes/magnumphotos/magazine.ts b/lib/routes/magnumphotos/magazine.ts new file mode 100644 index 00000000000000..6ca90ef7c8d6fe --- /dev/null +++ b/lib/routes/magnumphotos/magazine.ts @@ -0,0 +1,63 @@ +import { Route } from '@/types'; +import cache from '@/utils/cache'; +import parser from '@/utils/rss-parser'; +import ofetch from '@/utils/ofetch'; +import { load } from 'cheerio'; +const host = 'https://www.magnumphotos.com'; +export const route: Route = { + path: '/magazine', + categories: ['picture'], + example: '/magnumphotos/magazine', + parameters: {}, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['magnumphotos.com/'], + }, + ], + name: 'Magazine', + maintainers: ['EthanWng97'], + handler, + url: 'magnumphotos.com/', +}; + +async function handler() { + const rssUrl = `${host}/feed/`; + const feed = await parser.parseURL(rssUrl); + const items = await Promise.all( + feed.items.map((item) => + cache.tryGet(item.link, async () => { + if (!item.link) { + return; + } + const data = await ofetch(item.link); + const $ = load(data); + const description = $('#content'); + description.find('ul.share').remove(); + description.find('h1').remove(); + + return { + title: item.title, + pubDate: item.pubDate, + link: item.link, + category: item.categories, + description: description.html(), + }; + }) + ) + ); + + return { + title: 'Magnum Photos', + link: host, + description: 'Magnum is a community of thought, a shared human quality, a curiosity about what is going on in the world, a respect for what is going on and a desire to transcribe it visually', + item: items, + }; +} diff --git a/lib/routes/magnumphotos/namespace.ts b/lib/routes/magnumphotos/namespace.ts new file mode 100644 index 00000000000000..36e2f12075e8f9 --- /dev/null +++ b/lib/routes/magnumphotos/namespace.ts @@ -0,0 +1,6 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Magnum Photos', + url: 'magnumphotos.com', +}; From 5c687c718988d02fe62dad67b0f60e268b405d90 Mon Sep 17 00:00:00 2001 From: Tony <TonyRL@users.noreply.github.com> Date: Thu, 2 May 2024 23:28:12 +0800 Subject: [PATCH 05/10] style(eslint): enable no-array-callback-reference --- .eslintrc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.eslintrc.json b/.eslintrc.json index 6b758a500b3a95..4bfca3e1dcb3ad 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -59,7 +59,7 @@ "unicorn/explicit-length-check": 0, "unicorn/filename-case": ["error", { "case": "kebabCase", "ignore": [".*\\.(yaml|yml)$", "RequestInProgress\\.js$"] }], "unicorn/new-for-builtins": 0, - "unicorn/no-array-callback-reference": 0, + "unicorn/no-array-callback-reference": 1, "unicorn/no-array-reduce": 1, "unicorn/no-await-expression-member": 0, "unicorn/no-empty-file": 1, From aecdcd98222be865ba741f87a417451de23f04e5 Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Fri, 3 May 2024 01:44:39 +0800 Subject: [PATCH 06/10] =?UTF-8?q?fix(route):=20IT=E4=B9=8B=E5=AE=B6?= =?UTF-8?q?=E4=B8=93=E9=A2=98=20(#15446)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(route): IT之家专题 * docs: add link * Update lib/routes/ithome/zt.ts --------- --- lib/routes/ithome/templates/description.art | 13 ++ lib/routes/ithome/zt.ts | 174 +++++++++++++------- 2 files changed, 131 insertions(+), 56 deletions(-) create mode 100644 lib/routes/ithome/templates/description.art diff --git a/lib/routes/ithome/templates/description.art b/lib/routes/ithome/templates/description.art new file mode 100644 index 00000000000000..0a7f83a6f60fb1 --- /dev/null +++ b/lib/routes/ithome/templates/description.art @@ -0,0 +1,13 @@ +{{ if images }} + {{ each images image }} + {{ if image?.src }} + <figure> + <img + {{ if image.alt }} + alt="{{ image.alt }}" + {{ /if }} + src="{{ image.src }}"> + </figure> + {{ /if }} + {{ /each }} +{{ /if }} \ No newline at end of file diff --git a/lib/routes/ithome/zt.ts b/lib/routes/ithome/zt.ts index a33e5270c88780..4e7a251dd26cb6 100644 --- a/lib/routes/ithome/zt.ts +++ b/lib/routes/ithome/zt.ts @@ -1,88 +1,150 @@ import { Route } from '@/types'; +import { getCurrentPath } from '@/utils/helpers'; +const __dirname = getCurrentPath(import.meta.url); + import cache from '@/utils/cache'; import got from '@/utils/got'; import { load } from 'cheerio'; import timezone from '@/utils/timezone'; import { parseDate } from '@/utils/parse-date'; +import { art } from '@/utils/render'; +import path from 'node:path'; -export const route: Route = { - path: '/zt/:id', - categories: ['new-media'], - example: '/ithome/zt/xijiayi', - parameters: { id: '专题 id' }, - features: { - requireConfig: false, - requirePuppeteer: false, - antiCrawler: false, - supportBT: false, - supportPodcast: false, - supportScihub: false, - }, - radar: [ - { - source: ['ithome.com/zt/:id'], - }, - ], - name: '专题', - maintainers: ['nczitzk'], - handler, - description: `所有专题请见[此处](https://www.ithome.com/zt)`, -}; - -async function handler(ctx) { - const id = ctx.req.param('id'); +export const handler = async (ctx) => { + const { id = 'xijiayi' } = ctx.req.param(); + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50; const rootUrl = 'https://www.ithome.com'; - const currentUrl = `${rootUrl}/zt/${id}`; + const currentUrl = new URL(`zt/${id}`, rootUrl).href; + + const { data: response } = await got(currentUrl); - const response = await got({ - method: 'get', - url: currentUrl, - }); + const $ = load(response); - const $ = load(response.data); + const author = 'IT之家'; + const language = 'zh'; - const list = $('.newsbody a') - .map((_, item) => { + let items = $('div.newsbody') + .slice(0, limit) + .toArray() + .map((item) => { item = $(item); + const title = item.find('h2').text(); + const image = item.find('img').prop('data-original') ?? item.find('img').prop('src'); + return { - title: item.text(), - link: item.attr('href'), + title, + pubDate: timezone( + parseDate( + item + .find('span.time script') + .text() + .match(/'(.*?)'/) + ), + +8 + ), + link: item.find('a').first().prop('href'), + author: item.find('div.editor').contents().first().text(), + image, + banner: image, + language, }; - }) - .get(); + }); - const items = await Promise.all( - list.map((item) => + items = await Promise.all( + items.map((item) => cache.tryGet(item.link, async () => { - const detailResponse = await got({ - method: 'get', - url: item.link, - }); + const { data: detailResponse } = await got(item.link); + + const $$ = load(detailResponse); + + $$('p.ad-tips, a.topic-bar').remove(); - const content = load(detailResponse.data); - const post = content('.post_content'); + $$('div#paragraph p img').each((_, el) => { + el = $$(el); - post.find('img[data-original]').each((_, ele) => { - ele = $(ele); - ele.attr('src', ele.attr('data-original')); - ele.removeAttr('class'); - ele.removeAttr('data-original'); + const src = el.prop('data-original'); + + if (src) { + el.replaceWith( + art(path.join(__dirname, 'templates/description.art'), { + images: [ + { + src, + alt: el.prop('alt'), + }, + ], + }) + ); + } }); - item.description = post.html(); - item.author = content('#author_baidu').text().replace('作者:', ''); - item.pubDate = timezone(parseDate(content('#pubtime_baidu').text()), +8); + const title = $$('h1').text(); + const description = $$('div#paragraph').html(); + const image = $$('div#paragraph img').first().prop('src'); + + item.title = title; + item.description = description; + item.pubDate = timezone(parseDate($$('span#pubtime_baidu').text()), +8); + item.category = $$('div.cv a') + .toArray() + .map((c) => $$(c).text()) + .slice(1); + item.author = $$('span#author_baidu').contents().last().text() || $$('span#source_baidu').contents().last().text() || $$('span#editor_baidu').contents().last().text(); + item.content = { + html: description, + text: $$('div#paragraph').text(), + }; + item.image = image; + item.banner = image; + item.language = language; return item; }) ) ); + const image = new URL($('meta[property="og:image"]').prop('content'), rootUrl).href; + return { - title: `${$('title').text()} - IT之家`, + title: `${author} - ${$('title').text()}`, + description: $('meta[name="description"]').prop('content'), link: currentUrl, item: items, + allowEmpty: true, + image, + author, + language, }; -} +}; + +export const route: Route = { + path: '/zt/:id?', + name: '专题', + url: 'ithome.com', + maintainers: ['nczitzk'], + handler, + example: '/ithome/zt/xijiayi', + parameters: { category: '专题 id,默认为 xijiayi,即 [喜加一](https://www.ithome.com/zt/xijiayi),可在对应专题页 URL 中找到' }, + description: `:::tip + 更多专题请见 [IT之家专题](https://www.ithome.com/zt) + :::`, + categories: ['new-media'], + + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportRadar: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['ithome.com/zt/:id'], + target: '/zt/:id', + }, + ], +}; From c2510e0e2e1d332e6bc08c5b72bcef0a8b0acd81 Mon Sep 17 00:00:00 2001 From: Tony <TonyRL@users.noreply.github.com> Date: Fri, 3 May 2024 04:12:36 +0800 Subject: [PATCH 07/10] fix(route): set preload to metadata as suggested by the [spec](https://html.spec.whatwg.org/multipage/media.html#attr-media-preload) (#15448) --- lib/routes/caixin/templates/article.art | 2 +- lib/routes/douyin/templates/embed.art | 2 +- lib/routes/fansly/templates/media.art | 2 +- lib/routes/ifeng/templates/video.art | 2 +- lib/routes/instagram/templates/video.art | 2 +- lib/routes/kcna/utils.ts | 2 +- lib/routes/mingpao/templates/fancybox.art | 2 +- lib/routes/missav/templates/preview.art | 2 +- lib/routes/pikabu/templates/video.art | 2 +- lib/routes/pornhub/templates/description.art | 2 +- lib/routes/sina/templates/video.art | 2 +- lib/routes/tiktok/templates/user.art | 2 +- lib/routes/zhihu/topic.ts | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/routes/caixin/templates/article.art b/lib/routes/caixin/templates/article.art index 6b33c31ead9f40..fd30a943eb5570 100644 --- a/lib/routes/caixin/templates/article.art +++ b/lib/routes/caixin/templates/article.art @@ -15,7 +15,7 @@ <% const video = $('script').text().match(/initPlayer\('(.*?)','(.*?)'\)/); %> <% const videoUrl = video[1]; %> <% const poster = video[2]; %> - <video controls preload="none" poster="{{ poster }}" src="{{ videoUrl }}"></video> + <video controls preload="metadata" poster="{{ poster }}" src="{{ videoUrl }}"></video> <br> {{ /if }} diff --git a/lib/routes/douyin/templates/embed.art b/lib/routes/douyin/templates/embed.art index 530226b263a806..e8f32e69091d14 100644 --- a/lib/routes/douyin/templates/embed.art +++ b/lib/routes/douyin/templates/embed.art @@ -1,4 +1,4 @@ -<video controls preload="none" referrerpolicy="no-referrer" +<video controls preload="metadata" referrerpolicy="no-referrer" style="width:50%" {{ if img }} poster="{{ img }}" diff --git a/lib/routes/fansly/templates/media.art b/lib/routes/fansly/templates/media.art index 46bfcd0bf935db..791f06adcadeee 100644 --- a/lib/routes/fansly/templates/media.art +++ b/lib/routes/fansly/templates/media.art @@ -1,5 +1,5 @@ {{ if poster && src }} -<video controls preload="none" poster="{{ poster.location }}"> +<video controls preload="metadata" poster="{{ poster.location }}"> <source src="{{ src.location }}" type="video/mp4"> </video> {{ else if src }} diff --git a/lib/routes/ifeng/templates/video.art b/lib/routes/ifeng/templates/video.art index 7851288f07787e..510f5797b83137 100644 --- a/lib/routes/ifeng/templates/video.art +++ b/lib/routes/ifeng/templates/video.art @@ -1,5 +1,5 @@ {{ if videoInfo.mobileUrl }} -<video controls poster="{{ videoInfo.bigPosterUrl }}" preload="none"> +<video controls poster="{{ videoInfo.bigPosterUrl }}" preload="metadata"> <source src="{{ videoInfo.mobileUrl }}" type="video/mp4"> </video> {{ /if }} diff --git a/lib/routes/instagram/templates/video.art b/lib/routes/instagram/templates/video.art index f89b9b5c02b36e..9a6d0481f8898f 100644 --- a/lib/routes/instagram/templates/video.art +++ b/lib/routes/instagram/templates/video.art @@ -3,6 +3,6 @@ <br> {{ /if }} -<video controls preload="none" poster="{{ image }}" width="{{ video.width }}"> +<video controls preload="metadata" poster="{{ image }}" width="{{ video.width }}"> <source src="{{ video.url }}" type="video/mp4"> </video> diff --git a/lib/routes/kcna/utils.ts b/lib/routes/kcna/utils.ts index d88cc88c817ee0..fa7d3224af3cb7 100644 --- a/lib/routes/kcna/utils.ts +++ b/lib/routes/kcna/utils.ts @@ -54,7 +54,7 @@ const fetchVideo = (ctx, url) => const js = $('script[type="text/javascript"]:not([src])').html(); let sources = js.match(/<[^>]*source[^>]+src[^>]+>/g); sources = sources && sources.map((item) => item.replaceAll("'", '"').replaceAll(/src="([^"]+)"/g, `src="${rootUrl}$1"`)); - return `<video controls preload="none">${sources.join('\n')}</video>`; + return `<video controls preload="metadata">${sources.join('\n')}</video>`; }); export { parseJucheDate, fixDesc, fetchPhoto, fetchVideo }; diff --git a/lib/routes/mingpao/templates/fancybox.art b/lib/routes/mingpao/templates/fancybox.art index 430d1c40f1de7a..07e8ce97b2c2b6 100644 --- a/lib/routes/mingpao/templates/fancybox.art +++ b/lib/routes/mingpao/templates/fancybox.art @@ -1,6 +1,6 @@ {{ each media }} {{ if $value.video }} - <video controls preload="none" poster="{{ $value.video.replace('.mp4', '.jpg') }}"><source src="{{ $value.video }}" type="video/mp4"></video> + <video controls preload="metadata" poster="{{ $value.video.replace('.mp4', '.jpg') }}"><source src="{{ $value.video }}" type="video/mp4"></video> {{ else }} <figure><img src="{{ $value.href }}" alt="{{ $value.title }}"><figcaption>{{ $value.title }}</figcaption></figure> {{ /if }} diff --git a/lib/routes/missav/templates/preview.art b/lib/routes/missav/templates/preview.art index 5f3f875a994c66..5cad91c8d54388 100644 --- a/lib/routes/missav/templates/preview.art +++ b/lib/routes/missav/templates/preview.art @@ -1,3 +1,3 @@ -<video controls preload="none" poster="{{ poster }}"> +<video controls preload="metadata" poster="{{ poster }}"> <source src="{{ video }}" type="{{ type }}"> </video> diff --git a/lib/routes/pikabu/templates/video.art b/lib/routes/pikabu/templates/video.art index f76b0437ee0c2b..09b5faa18781aa 100644 --- a/lib/routes/pikabu/templates/video.art +++ b/lib/routes/pikabu/templates/video.art @@ -1,7 +1,7 @@ {{ if videoId }} <iframe id="ytplayer" type="text/html" width="640" height="360" src="https://www.youtube-nocookie.com/embed/{{ videoId }}" frameborder="0" allowfullscreen></iframe> {{ else if webm || mp4 }} - <video controls preload="none" poster="{{ preview }}" width="{{ width }}"> + <video controls preload="metadata" poster="{{ preview }}" width="{{ width }}"> {{ if webm }}<source src="{{ webm }}" type="video/webm">{{ /if }} {{ if mp4 }}<source src="{{ mp4 }}" type="video/mp4">{{ /if }} </video> diff --git a/lib/routes/pornhub/templates/description.art b/lib/routes/pornhub/templates/description.art index 33aea6d4e73fc0..94b9466bd5de52 100644 --- a/lib/routes/pornhub/templates/description.art +++ b/lib/routes/pornhub/templates/description.art @@ -1,5 +1,5 @@ {{ if previewVideo }} -<video controls preload="none" poster="{{ poster }}"> +<video controls preload="metadata" poster="{{ poster }}"> <source src="{{ previewVideo }}" type="video/webm"> </video> {{ /if }} diff --git a/lib/routes/sina/templates/video.art b/lib/routes/sina/templates/video.art index 25d894f1437a68..c86a810b5bba7b 100644 --- a/lib/routes/sina/templates/video.art +++ b/lib/routes/sina/templates/video.art @@ -1,5 +1,5 @@ {{ if videoUrl }} -<video controls poster="{{ poster }}" preload="none"> +<video controls poster="{{ poster }}" preload="metadata"> <source src="{{ videoUrl }}"> </video> {{ /if }} diff --git a/lib/routes/tiktok/templates/user.art b/lib/routes/tiktok/templates/user.art index 7725b6487dfb33..0c582bfaed3574 100644 --- a/lib/routes/tiktok/templates/user.art +++ b/lib/routes/tiktok/templates/user.art @@ -1,7 +1,7 @@ {{ if useIframe }} <iframe src="https://www.tiktok.com/embed/{{ id }}" height="757" frameborder="0" referrerpolicy="no-referrer"></iframe> {{ else }} -<video controls loop poster="{{ poster }}" preload="none"> +<video controls loop poster="{{ poster }}" preload="metadata"> <source src="{{ source }}"> </video> {{ /if }} diff --git a/lib/routes/zhihu/topic.ts b/lib/routes/zhihu/topic.ts index 0a4436f8d9fcd0..e181dd33a70177 100644 --- a/lib/routes/zhihu/topic.ts +++ b/lib/routes/zhihu/topic.ts @@ -107,7 +107,7 @@ async function handler(ctx) { case 'zvideo': title = item.title; description = `${item.description}<br> - <video controls poster="${item.video.thumbnail}" preload="none"> + <video controls poster="${item.video.thumbnail}" preload="metadata"> <source src="${item.video.playlist.fhd?.url ?? item.video.playlist.hd?.url ?? item.video.playlist.ld?.url ?? item.video.playlist.sd?.url}" type="video/mp4"> </video>`; link = item.url; From cf4f2d21a8914202b6d3e7a6081ddbc818b31597 Mon Sep 17 00:00:00 2001 From: Peng Guanwen <pg999w@outlook.com> Date: Fri, 3 May 2024 05:23:38 +0800 Subject: [PATCH 08/10] fix(route/theinitium): Fix metadata & token expiration (#15444) * fix(route/theinitium): Fix metadata & token expiration * Update full.ts Co-authored-by: Tony <TonyRL@users.noreply.github.com> * fix: split theinitium --------- --- lib/routes/theinitium/author.ts | 23 ++++++++++ lib/routes/theinitium/channel.ts | 28 ++++++++++++ lib/routes/theinitium/follow.ts | 48 +++++++++++++++++++++ lib/routes/theinitium/tags.ts | 23 ++++++++++ lib/routes/theinitium/{full.ts => utils.ts} | 33 ++++++++------ 5 files changed, 141 insertions(+), 14 deletions(-) create mode 100644 lib/routes/theinitium/author.ts create mode 100644 lib/routes/theinitium/channel.ts create mode 100644 lib/routes/theinitium/follow.ts create mode 100644 lib/routes/theinitium/tags.ts rename lib/routes/theinitium/{full.ts => utils.ts} (92%) diff --git a/lib/routes/theinitium/author.ts b/lib/routes/theinitium/author.ts new file mode 100644 index 00000000000000..a1d5dbc60055b7 --- /dev/null +++ b/lib/routes/theinitium/author.ts @@ -0,0 +1,23 @@ +import { Route } from '@/types'; +import { processFeed } from './utils'; + +const handler = (ctx) => processFeed('author', ctx); + +export const route: Route = { + path: '/author/:type/:language?', + name: '作者', + maintainers: ['AgFlore'], + parameters: { + type: '作者 ID,可从作者主页 URL 中获取,如 `https://theinitium.com/author/ninghuilulu`', + language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为简体', + }, + radar: [ + { + source: ['theinitium.com/author/:type'], + target: '/author/:type', + }, + ], + handler, + example: '/theinitium/author/ninghuilulu/zh-hans', + categories: ['new-media'], +}; diff --git a/lib/routes/theinitium/channel.ts b/lib/routes/theinitium/channel.ts new file mode 100644 index 00000000000000..b418b09216906b --- /dev/null +++ b/lib/routes/theinitium/channel.ts @@ -0,0 +1,28 @@ +import { Route } from '@/types'; +import { processFeed } from './utils'; + +const handler = (ctx) => processFeed('channel', ctx); + +export const route: Route = { + path: '/channel/:type?/:language?', + name: '专题・栏目', + maintainers: ['prnake'], + parameters: { + type: '栏目,缺省为最新', + language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为简体', + }, + radar: [ + { + source: ['theinitium.com/channel/:type'], + target: '/channel/:type', + }, + ], + handler, + example: '/theinitium/channel/latest/zh-hans', + categories: ['new-media'], + description: `Type 栏目: + + | 最新 | 深度 | What’s New | 广场 | 科技 | 风物 | 特约 | ... | + | ------ | ------- | ---------- | ----------------- | ---------- | ------- | -------- | --- | + | latest | feature | news-brief | notes-and-letters | technology | culture | pick_up | ... |`, +}; diff --git a/lib/routes/theinitium/follow.ts b/lib/routes/theinitium/follow.ts new file mode 100644 index 00000000000000..84fd599f3ab193 --- /dev/null +++ b/lib/routes/theinitium/follow.ts @@ -0,0 +1,48 @@ +import { Route } from '@/types'; +import { processFeed } from './utils'; + +const handler = (ctx) => processFeed('follow', ctx); + +export const route: Route = { + path: '/follow/articles/:language?', + name: '个人订阅追踪动态', + maintainers: ['AgFlore'], + parameters: { + language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为简体', + }, + radar: [ + { + title: '作者', + source: ['theinitium.com/author/:type'], + target: '/author/:type', + }, + ], + handler, + example: '/theinitium/author/ninghuilulu/zh-hans', + categories: ['new-media'], + description: 'Web 版认证 token 和 iOS 内购回执认证 token 只需选择其一填入即可。你也可选择直接在环境设置中填写明文的用户名和密码', + features: { + requireConfig: [ + { + name: 'INITIUM_BEARER_TOKEN', + optional: true, + description: `端传媒 Web 版认证 token。获取方式:登陆后打开端传媒站内任意页面,打开浏览器开发者工具中 “网络”(Network) 选项卡,筛选 URL 找到任一个地址为 \`api.initium.com\` 开头的请求,点击检查其 “消息头”,在 “请求头” 中找到Authorization字段,将其值复制填入配置即可。你的配置应该形如 \`INITIUM_BEARER_TOKEN: 'Bearer eyJxxxx......xx_U8'\`。使用 token 部署的好处是避免占据登陆设备数的额度,但这个 token 一般有效期为两周,因此只可作临时测试使用。`, + }, + { + name: 'INITIUM_IAP_RECEIPT', + optional: true, + description: `端传媒 iOS 版内购回执认证 token。获取方式:登陆后打开端传媒 iOS app 内任意页面,打开抓包工具,筛选 URL 找到任一个地址为 \`api.initium.com\` 开头的请求,点击检查其 “消息头”,在 “请求头” 中找到 \`X-IAP-Receipt\` 字段,将其值复制填入配置即可。你的配置应该形如 \`INITIUM_IAP_RECEIPT: ef81dee9e4e2fe084a0af1ea82da2f7b16e75f756db321618a119fa62b52550e\`。`, + }, + { + name: 'INITIUM_USERNAME', + optional: true, + description: `端传媒用户名 (邮箱)`, + }, + { + name: 'INITIUM_PASSWORD', + optional: true, + description: `端传媒密码`, + }, + ], + }, +}; diff --git a/lib/routes/theinitium/tags.ts b/lib/routes/theinitium/tags.ts new file mode 100644 index 00000000000000..fd0beb1a284fe3 --- /dev/null +++ b/lib/routes/theinitium/tags.ts @@ -0,0 +1,23 @@ +import { Route } from '@/types'; +import { processFeed } from './utils'; + +const handler = (ctx) => processFeed('tags', ctx); + +export const route: Route = { + path: '/tags/:type/:language?', + name: '话题・标签', + maintainers: ['AgFlore'], + parameters: { + type: '话题 ID,可从话题页 URL 中获取,如 `https://theinitium.com/tags/2019_10/`', + language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为简体', + }, + radar: [ + { + source: ['theinitium.com/tags/:type'], + target: '/tags/:type', + }, + ], + handler, + example: '/theinitium/tags/2019_10/zh-hans', + categories: ['new-media'], +}; diff --git a/lib/routes/theinitium/full.ts b/lib/routes/theinitium/utils.ts similarity index 92% rename from lib/routes/theinitium/full.ts rename to lib/routes/theinitium/utils.ts index 4a680f0d688b96..3a818fcf316064 100644 --- a/lib/routes/theinitium/full.ts +++ b/lib/routes/theinitium/utils.ts @@ -1,22 +1,16 @@ -import { Route } from '@/types'; +import { Context } from 'hono'; import cache from '@/utils/cache'; import got from '@/utils/got'; import { config } from '@/config'; import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; +import InvalidParameterError from '@/errors/types/invalid-parameter'; +import { FetchError } from 'ofetch'; const TOKEN = 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY='; -export const route: Route = { - path: '/:model?/:type?/:language?', - name: 'Unknown', - maintainers: [], - handler, -}; - -async function handler(ctx) { +export const processFeed = async (model: string, ctx: Context) => { // model是channel/tag/etc.,而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话,那就是model是follow,type是articles。 - const model = ctx.req.param('model') ?? 'channel'; const type = ctx.req.param('type') ?? 'latest'; const language = ctx.req.param('language') ?? 'zh-hans'; let listUrl; @@ -38,6 +32,8 @@ async function handler(ctx) { listUrl = `https://api.theinitium.com/api/v2/tag/articles/?language=${language}&slug=${type}`; listLink = `https://theinitium.com/tags/${type}/`; break; + default: + throw new InvalidParameterError('wrong model'); } const key = { @@ -92,9 +88,18 @@ async function handler(ctx) { 'X-IAP-Receipt': key.iapReceipt || '', }; - const response = await got(listUrl, { - headers, - }); + let response; + try { + response = await got(listUrl, { + headers, + }); + } catch (error) { + if (error instanceof FetchError && error.statusCode === 401) { + // 401 说明 token 过期了,将它删掉 + await cache.set('initium:token', ''); + } + throw error; + } const name = response.data.name || (response.data[model] && response.data[model].name) || '追踪'; // 从v1直升的channel和tags里面是digests,v2新增的author和follow出来都是results @@ -175,4 +180,4 @@ async function handler(ctx) { item: items, image, }; -} +}; From 7c4c09c323ac265641fb5a83d6e8cd23da2254cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 05:44:48 +0800 Subject: [PATCH 09/10] chore(deps): bump googleapis from 135.1.0 to 136.0.0 (#15449) * chore(deps): bump googleapis from 135.1.0 to 136.0.0 Bumps [googleapis](https://github.com/googleapis/google-api-nodejs-client) from 135.1.0 to 136.0.0. - [Release notes](https://github.com/googleapis/google-api-nodejs-client/releases) - [Changelog](https://github.com/googleapis/google-api-nodejs-client/blob/main/release-please-config.json) - [Commits](https://github.com/googleapis/google-api-nodejs-client/compare/googleapis-v135.1.0...googleapis-v136.0.0) --- updated-dependencies: - dependency-name: googleapis dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> * chore: fix pnpm install --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- pnpm-lock.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e6aed3b8f4f07e..35bbd29f009411 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ "etag": "1.8.1", "fanfou-sdk": "5.0.0", "form-data": "4.0.0", - "googleapis": "135.1.0", + "googleapis": "136.0.0", "hono": "4.2.9", "html-to-text": "9.0.5", "https-proxy-agent": "7.0.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43400e1f65fc51..9596fb60c1d18e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,8 +78,8 @@ dependencies: specifier: 4.0.0 version: 4.0.0 googleapis: - specifier: 135.1.0 - version: 135.1.0 + specifier: 136.0.0 + version: 136.0.0 hono: specifier: 4.2.9 version: 4.2.9 @@ -5446,8 +5446,8 @@ packages: - supports-color dev: false - /googleapis@135.1.0: - resolution: {integrity: sha512-cQ4P0ok2YJZGOW6nBFI/ZC3rjX5CAzp+onIc1O8ovIp1N7wiVW6vGJhLgB3lfUls9EqNf5yNFP4mqTT3MAYInA==} + /googleapis@136.0.0: + resolution: {integrity: sha512-W2Z1NtFS8ie5SzN74+lnTzLS9d0tS01dybibqXjWKSfdvTkQr9DYsKYNMRpzMAcBva1ZWCAgTiX4fgGz2Cwbaw==} engines: {node: '>=14.0.0'} dependencies: google-auth-library: 9.6.3 From b1ca4375bdb074029cb8a8777fb955da801e0afa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 05:46:03 +0800 Subject: [PATCH 10/10] chore(deps): bump @sentry/node from 7.112.2 to 7.113.0 (#15450) * chore(deps): bump @sentry/node from 7.112.2 to 7.113.0 Bumps [@sentry/node](https://github.com/getsentry/sentry-javascript) from 7.112.2 to 7.113.0. - [Release notes](https://github.com/getsentry/sentry-javascript/releases) - [Changelog](https://github.com/getsentry/sentry-javascript/blob/7.113.0/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-javascript/compare/7.112.2...7.113.0) --- updated-dependencies: - dependency-name: "@sentry/node" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * chore: fix pnpm install --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- pnpm-lock.yaml | 56 +++++++++++++++++++++++++------------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index 35bbd29f009411..14e00da6001c05 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "@hono/zod-openapi": "0.11.0", "@notionhq/client": "2.2.15", "@postlight/parser": "2.2.3", - "@sentry/node": "7.112.2", + "@sentry/node": "7.113.0", "@tonyrl/rand-user-agent": "2.0.61", "aes-js": "3.1.2", "art-template": "4.13.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9596fb60c1d18e..22f52388eef2d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ dependencies: specifier: 2.2.3 version: 2.2.3 '@sentry/node': - specifier: 7.112.2 - version: 7.112.2 + specifier: 7.113.0 + version: 7.113.0 '@tonyrl/rand-user-agent': specifier: 2.0.61 version: 2.0.61 @@ -2463,54 +2463,54 @@ packages: selderee: 0.11.0 dev: false - /@sentry-internal/tracing@7.112.2: - resolution: {integrity: sha512-fT1Y46J4lfXZkgFkb03YMNeIEs2xS6jdKMoukMFQfRfVvL9fSWEbTgZpHPd/YTT8r2i082XzjtAoQNgklm/0Hw==} + /@sentry-internal/tracing@7.113.0: + resolution: {integrity: sha512-8MDnYENRMnEfQjvN4gkFYFaaBSiMFSU/6SQZfY9pLI3V105z6JQ4D0PGMAUVowXilwNZVpKNYohE7XByuhEC7Q==} engines: {node: '>=8'} dependencies: - '@sentry/core': 7.112.2 - '@sentry/types': 7.112.2 - '@sentry/utils': 7.112.2 + '@sentry/core': 7.113.0 + '@sentry/types': 7.113.0 + '@sentry/utils': 7.113.0 dev: false - /@sentry/core@7.112.2: - resolution: {integrity: sha512-gHPCcJobbMkk0VR18J65WYQTt3ED4qC6X9lHKp27Ddt63E+MDGkG6lvYBU1LS8cV7CdyBGC1XXDCfor61GvLsA==} + /@sentry/core@7.113.0: + resolution: {integrity: sha512-pg75y3C5PG2+ur27A0Re37YTCEnX0liiEU7EOxWDGutH17x3ySwlYqLQmZsFZTSnvzv7t3MGsNZ8nT5O0746YA==} engines: {node: '>=8'} dependencies: - '@sentry/types': 7.112.2 - '@sentry/utils': 7.112.2 + '@sentry/types': 7.113.0 + '@sentry/utils': 7.113.0 dev: false - /@sentry/integrations@7.112.2: - resolution: {integrity: sha512-ioC2yyU6DqtLkdmWnm87oNvdn2+9oKctJeA4t+jkS6JaJ10DcezjCwiLscX4rhB9aWJV3IWF7Op0O6K3w0t2Hg==} + /@sentry/integrations@7.113.0: + resolution: {integrity: sha512-w0sspGBQ+6+V/9bgCkpuM3CGwTYoQEVeTW6iNebFKbtN7MrM3XsGAM9I2cW1jVxFZROqCBPFtd2cs5n0j14aAg==} engines: {node: '>=8'} dependencies: - '@sentry/core': 7.112.2 - '@sentry/types': 7.112.2 - '@sentry/utils': 7.112.2 + '@sentry/core': 7.113.0 + '@sentry/types': 7.113.0 + '@sentry/utils': 7.113.0 localforage: 1.10.0 dev: false - /@sentry/node@7.112.2: - resolution: {integrity: sha512-MNzkqER8jc2xOS3ArkCLH5hakzu15tcjeC7qjU7rQ1Ms4WuV+MG0docSRESux0/p23Qjzf9tZOc8C5Eq+Sxduw==} + /@sentry/node@7.113.0: + resolution: {integrity: sha512-Vam4Ia0I9fhVw8GJOzcLP7MiiHJSKl8L9LzLMMLG3+2/dFnDQOyS7sOfk3GqgpwzqPiusP9vFu7CFSX7EMQbTg==} engines: {node: '>=8'} dependencies: - '@sentry-internal/tracing': 7.112.2 - '@sentry/core': 7.112.2 - '@sentry/integrations': 7.112.2 - '@sentry/types': 7.112.2 - '@sentry/utils': 7.112.2 + '@sentry-internal/tracing': 7.113.0 + '@sentry/core': 7.113.0 + '@sentry/integrations': 7.113.0 + '@sentry/types': 7.113.0 + '@sentry/utils': 7.113.0 dev: false - /@sentry/types@7.112.2: - resolution: {integrity: sha512-kCMLt7yhY5OkWE9MeowlTNmox9pqDxcpvqguMo4BDNZM5+v9SEb1AauAdR78E1a1V8TyCzjBD7JDfXWhvpYBcQ==} + /@sentry/types@7.113.0: + resolution: {integrity: sha512-PJbTbvkcPu/LuRwwXB1He8m+GjDDLKBtu3lWg5xOZaF5IRdXQU2xwtdXXsjge4PZR00tF7MO7X8ZynTgWbYaew==} engines: {node: '>=8'} dev: false - /@sentry/utils@7.112.2: - resolution: {integrity: sha512-OjLh0hx0t1EcL4ZIjf+4svlmmP+tHUDGcr5qpFWH78tjmkPW4+cqPuZCZfHSuWcDdeiaXi8TnYoVRqDcJKK/eQ==} + /@sentry/utils@7.113.0: + resolution: {integrity: sha512-nzKsErwmze1mmEsbW2AwL2oB+I5v6cDEJY4sdfLekA4qZbYZ8pV5iWza6IRl4XfzGTE1qpkZmEjPU9eyo0yvYw==} engines: {node: '>=8'} dependencies: - '@sentry/types': 7.112.2 + '@sentry/types': 7.113.0 dev: false /@sinclair/typebox@0.27.8: