From 46cc98daefde4a61c98ce2d877f3d32bac1e0d3d Mon Sep 17 00:00:00 2001 From: MicroBlock <66859419+MicroCBer@users.noreply.github.com> Date: Mon, 15 Apr 2024 05:49:47 +0800 Subject: [PATCH] feat(telegram): support media group, improve message parsing (#261) --- adapters/telegram/src/bot.ts | 4 +- adapters/telegram/src/message.ts | 202 ++++++++++++++++++++++--------- adapters/telegram/src/utils.ts | 119 +++++++++++++----- packages/element/src/index.ts | 12 +- 4 files changed, 242 insertions(+), 95 deletions(-) diff --git a/adapters/telegram/src/bot.ts b/adapters/telegram/src/bot.ts index 45768837..25ee1b7d 100644 --- a/adapters/telegram/src/bot.ts +++ b/adapters/telegram/src/bot.ts @@ -34,6 +34,8 @@ export class TelegramBot { const { data, mime } = await this.$getFile(ctx.params.file) ctx.set('content-type', mime) - ctx.body = data + ctx.body = Buffer.from(data) }) } } diff --git a/adapters/telegram/src/message.ts b/adapters/telegram/src/message.ts index 87dc6da4..ceb13fde 100644 --- a/adapters/telegram/src/message.ts +++ b/adapters/telegram/src/message.ts @@ -1,32 +1,13 @@ -import { Context, Dict, h, MessageEncoder } from '@satorijs/satori' +import { Context, Dict, Element, file, h, MessageEncoder } from '@satorijs/satori' import { TelegramBot } from './bot' import * as Telegram from './utils' type RenderMode = 'default' | 'figure' -type AssetMethod = 'sendPhoto' | 'sendAudio' | 'sendDocument' | 'sendVideo' | 'sendAnimation' | 'sendVoice' - -async function appendAsset(bot: TelegramBot, form: FormData, element: h): Promise { - let method: AssetMethod - const { filename, data, mime } = await bot.ctx.http.file(element.attrs.src || element.attrs.url, element.attrs) - if (element.type === 'img' || element.type === 'image') { - method = mime === 'image/gif' ? 'sendAnimation' : 'sendPhoto' - } else if (element.type === 'file') { - method = 'sendDocument' - } else if (element.type === 'video') { - method = 'sendVideo' - } else if (element.type === 'audio') { - method = element.attrs.type === 'voice' ? 'sendVoice' : 'sendAudio' - } - const value = new Blob([data], { type: mime }) - form.append(method.slice(4).toLowerCase(), value, filename) - return method -} - const supportedElements = ['b', 'strong', 'i', 'em', 'u', 'ins', 's', 'del', 'a'] export class TelegramMessageEncoder extends MessageEncoder> { - private asset: h = null + private asset: h[] = [] private payload: Dict private mode: RenderMode = 'default' private rows: Telegram.InlineKeyboardButton[][] = [] @@ -42,47 +23,151 @@ export class TelegramMessageEncoder extends Message async addResult(result: Telegram.Message) { const session = this.bot.session() await Telegram.decodeMessage(this.bot, result, session.event.message = {}, session.event) + session.event._data ??= {} + session.event._data.message = result this.results.push(session.event.message) session.app.emit(session, 'send', session) } - async sendAsset() { - const form = new FormData() - for (const key in this.payload) { - form.append(key, this.payload[key].toString()) - } - form.append('reply_markup', JSON.stringify({ - inline_keyboard: this.rows, - })) - const method = await appendAsset(this.bot, form, this.asset) - const result = await this.bot.internal[method](form as any) - await this.addResult(result) - delete this.payload.reply_to_message_id - this.asset = null - this.payload.caption = '' - } - async flush() { - if (this.asset) { - // send previous asset if there is any - await this.sendAsset() - } else if (this.payload.caption) { + if (this.payload.caption || this.asset.length > 0) { this.trimButtons() - const result = await this.bot.internal.sendMessage({ - chat_id: this.payload.chat_id, - text: this.payload.caption, - parse_mode: this.payload.parse_mode, - reply_to_message_id: this.payload.reply_to_message_id, - message_thread_id: this.payload.message_thread_id, - disable_web_page_preview: !this.options.linkPreview, - reply_markup: { - inline_keyboard: this.rows, - }, - }) - await this.addResult(result) - delete this.payload.reply_to_message_id - this.payload.caption = '' - this.rows = [] + + if (this.asset.length > 0) { + const files: { + filename: string + data: ArrayBuffer + mime: string + type: string + element: Element + }[] = [] + + const typeMap = { + img: 'photo', + image: 'photo', + audio: 'audio', + video: 'video', + file: 'document', + } + + let i = 0; + for (const element of this.asset) { + const { filename, data, mime } = await this.bot.ctx.http.file(element.attrs.src || element.attrs.url, element.attrs) + files.push({ + filename: (i++) + filename, + data, + mime, + type: filename.endsWith('gif') ? 'animation' : typeMap[element.type] ?? element.type, + element + }) + } + + // Array of InputMediaAudio, InputMediaDocument, InputMediaPhoto and InputMediaVideo + const inputFiles: Telegram.InputFile[] = [] + + for (const { filename, data, mime, type, element } of files) { + const media = 'attach://' + filename + inputFiles.push({ + media, type, + has_spoiler: element.attrs.spoiler + }) + } + + if (files.length > 1) { + inputFiles[0].caption = this.payload.caption + inputFiles[0].parse_mode = this.payload.parse_mode + + const form = new FormData() + + const data = { + chat_id: this.payload.chat_id, + reply_to_message_id: this.payload.reply_to_message_id, + message_thread_id: this.payload.message_thread_id, + media: JSON.stringify(inputFiles) + } + for (const key in data) { + form.append(key, data[key]) + } + + for (const { filename, data, mime } of files) { + form.append(filename, new Blob([data], { type: mime }), filename) + } + + // @ts-ignore + const result = await this.bot.internal.sendMediaGroup(form) + + for (const x of result) + await this.addResult(x) + + if (this.rows.length > 0 && this.rows[0].length > 0) { + const result2 = await this.bot.internal.sendMessage({ + chat_id: this.payload.chat_id, + text: this.payload.caption, + parse_mode: this.payload.parse_mode, + reply_to_message_id: result[0].message_id, + message_thread_id: this.payload.message_thread_id, + disable_web_page_preview: !this.options.linkPreview, + reply_markup: { + inline_keyboard: this.rows, + }, + }) + + await this.addResult(result2) + delete this.payload.reply_to_message_id + this.payload.caption = '' + this.rows = [] + } + + delete this.payload.reply_to_message_id + this.payload.caption = '' + this.rows = [] + } else { + const sendMap = [ + ['audio', ['sendAudio', 'audio']], + ['voice', ['sendAudio', 'audio']], + ['video', ['sendVideo', 'video']], + ['animation', ['sendAnimation', 'animation']], + ['image', ['sendPhoto', 'photo']], + ['photo', ['sendPhoto', 'photo']], + ['document', ['sendDocument', 'document']], + ['', ['sendDocument', 'document']], + ] as const + const [_, [method, dataKey]] = sendMap.find(([key]) => files[0].type.startsWith(key)) || [] + + const formData = new FormData() + formData.append('chat_id', this.payload.chat_id) + formData.append('caption', this.payload.caption) + formData.append('parse_mode', this.payload.parse_mode) + formData.append('reply_to_message_id', this.payload.reply_to_message_id) + formData.append('message_thread_id', this.payload.message_thread_id) + formData.append('has_spoiler', files[0].element.attrs.spoiler ? 'true' : 'false') + formData.append(dataKey, 'attach://' + files[0].filename) + formData.append(files[0].filename, new Blob([files[0].data], { type: files[0].mime }), files[0].filename) + + // @ts-ignore + const result = await this.bot.internal[method](formData) + await this.addResult(result) + this.payload.caption = '' + this.rows = [] + delete this.payload.reply_to_message_id + } + } else { + const result = await this.bot.internal.sendMessage({ + chat_id: this.payload.chat_id, + text: this.payload.caption, + parse_mode: this.payload.parse_mode, + reply_to_message_id: this.payload.reply_to_message_id, + message_thread_id: this.payload.message_thread_id, + disable_web_page_preview: !this.options.linkPreview, + reply_markup: { + inline_keyboard: this.rows, + }, + }) + await this.addResult(result) + delete this.payload.reply_to_message_id + this.payload.caption = '' + this.rows = [] + } } } @@ -143,10 +228,7 @@ export class TelegramMessageEncoder extends Message this.payload.caption += `@${attrs.name || attrs.id}` } } else if (['img', 'image', 'audio', 'video', 'file'].includes(type)) { - if (this.mode === 'default') { - await this.flush() - } - this.asset = element + this.asset.push(element) } else if (type === 'figure') { await this.flush() this.mode = 'figure' diff --git a/adapters/telegram/src/utils.ts b/adapters/telegram/src/utils.ts index 4c1d26ec..9bca6d76 100644 --- a/adapters/telegram/src/utils.ts +++ b/adapters/telegram/src/utils.ts @@ -16,6 +16,11 @@ export const decodeGuildMember = (data: Telegram.ChatMember): Universal.GuildMem title: data['custom_title'], }) +const mediaGroupMap = new Map() + export async function handleUpdate(update: Telegram.Update, bot: TelegramBot) { bot.logger.debug('receive %s', JSON.stringify(update)) // Internal event: get update type from field name. @@ -49,8 +54,36 @@ export async function handleUpdate(update: Telegram.Update, bot: TelegramBot) { await decodeMessage(bot, message, session.event.message = {}, session.event) session.content = session.content.slice(1) } else if (message) { - session.type = update.message || update.channel_post ? 'message' : 'message-updated' - await decodeMessage(bot, message, session.event.message = {}, session.event) + if (update.message?.media_group_id) { + if (!mediaGroupMap.has(update.message.media_group_id)) + mediaGroupMap.set(update.message.media_group_id, [new Date(), []]) + + const [date, updates] = mediaGroupMap.get(update.message.media_group_id) + session.type = update.message || update.channel_post ? 'message' : 'message-updated' + await decodeMessage(bot, message, session.event.message = {}, session.event) + updates.push({ + id: update.message.message_id, + elements: session.event.message.elements + }) + + let thisUpdateTime = new Date() + mediaGroupMap.set(update.message.media_group_id, [thisUpdateTime, updates]) + await new Promise(r => setTimeout(r, 800)) + if (mediaGroupMap.get(update.message.media_group_id)[0] === thisUpdateTime) { + mediaGroupMap.delete(update.message.media_group_id) + // merge all messages + session.event.message.elements = updates.reduce((acc, cur) => acc.concat(cur.elements), []) + session.event.message.content = session.event.message.elements.join('') + session.event.message.id = Math.min(...updates.map(e => e.id)).toString() + session.event._data.mediaGroup = updates.map(e => e.id) + } else { + // the media group is still updating + return + } + } else { + session.type = update.message || update.channel_post ? 'message' : 'message-updated' + await decodeMessage(bot, message, session.event.message = {}, session.event) + } } else if (update.chat_join_request) { session.timestamp = update.chat_join_request.date * 1000 session.type = 'guild-member-request' @@ -111,35 +144,63 @@ export async function decodeMessage( message: Universal.Message, payload: Universal.MessageLike = message, ) { - const parseText = (text: string, entities: Telegram.MessageEntity[]): h[] => { - let curr = 0 - const segs: h[] = [] + const parseText = (text: string | undefined, entities: Telegram.MessageEntity[]): h[] => { + if (!text) return [] + const breakpoints = new Set() for (const e of entities) { - const eText = text.substr(e.offset, e.length) - if (e.type === 'mention') { - if (eText[0] !== '@') throw new Error('Telegram mention does not start with @: ' + eText) - const atName = eText.slice(1) - if (eText === '@' + bot.user.name) { - segs.push(h('at', { id: bot.user.id, name: atName })) - } else { - // TODO handle @others - segs.push(h('text', { content: eText })) - } - } else if (e.type === 'text_mention') { - segs.push(h('at', { id: e.user.id })) - } else { - // TODO: bold, italic, underline, strikethrough, spoiler, code, pre, - // text_link, custom_emoji - segs.push(h('text', { content: eText })) + breakpoints.add(e.offset) + breakpoints.add(e.offset + e.length) + } + + breakpoints.add(text.length) + + for (let i = 0; i < text.length; i++) { + if (text[i] === '\n') { + breakpoints.add(i) + breakpoints.add(i + 1) } - if (e.offset > curr) { - segs.splice(-1, 0, h('text', { content: text.slice(curr, e.offset) })) + } + + const obtainAttributeAtBP = (bp: number) => { + const attr = [] + let url = null, user = null + for (const e of entities) { + if (e.offset <= bp && e.offset + e.length > bp) { + attr.push(e.type) + + if (e.type === 'text_link') { + url = e.url + } else if (e.type === 'mention') { + user = e.user + } + } } - curr = e.offset + e.length + + return { attr, url, user } } - if (curr < text?.length || 0) { - segs.push(h('text', { content: text.slice(curr) })) + + const segs: h[] = [] + let start = 0 + for (const bp of Array.from(breakpoints).sort((a, b) => a - b)) { + if (start < bp) { + const { attr, url, user } = obtainAttributeAtBP(start) + const content = text.slice(start, bp) + let ele = h('text', { content }) + if (attr.includes('bold')) ele = h('b', {}, ele) + if (attr.includes('italic')) ele = h('i', {}, ele) + if (attr.includes('underline')) ele = h('u', {}, ele) + if (attr.includes('strikethrough')) ele = h('s', {}, ele) + if (attr.includes('code')) ele = h('code', {}, ele) + if (attr.includes('pre')) ele = h('pre', {}, ele) + if (attr.includes('spoiler')) ele = h('spl', {}, ele) + if (url) ele = h('a', { href: url }, ele) + if (user) ele = h('at', { id: user.id }, ele) + if (content === '\n') ele = h('br') + segs.push(ele) + } + start = bp } + return segs } @@ -152,7 +213,7 @@ export async function decodeMessage( // make sure text comes first so that commands can be triggered const msgText = data.text || data.caption - segments.push(...parseText(msgText, data.entities || [])) + segments.push(...parseText(msgText, [...(data.entities ?? []), ...(data.caption_entities ?? [])])) if (data.caption) { // add a space to separate caption from media @@ -194,6 +255,8 @@ export async function decodeMessage( await addResource('video', data.video) } else if (data.document) { await addResource('file', data.document) + } else if (data.audio) { + await addResource('audio', data.audio) } message.elements = segments @@ -202,7 +265,7 @@ export async function decodeMessage( if (!payload) return payload.timestamp = data.date * 1000 - payload.user = decodeUser(data.from) + payload.user = data.from ? decodeUser(data.from) : {} as any if (data.chat.type === 'private') { payload.channel = { id: data.chat.id.toString(), diff --git a/packages/element/src/index.ts b/packages/element/src/index.ts index d9d5f0e1..fde76ca3 100644 --- a/packages/element/src/index.ts +++ b/packages/element/src/index.ts @@ -58,7 +58,7 @@ interface Element { toString(strip?: boolean): string } -interface ElementConstructor extends Element {} +interface ElementConstructor extends Element { } class ElementConstructor { get data() { @@ -180,10 +180,10 @@ namespace Element { } export function escape(source: string, inline = false) { - const result = source - .replace(/&/g, '&') - .replace(//g, '>') + const result + = (source ?? '').replace(/&/g, '&') + .replace(//g, '>') return inline ? result.replace(/"/g, '"') : result @@ -511,7 +511,7 @@ namespace Element { } // eslint-disable-next-line prefer-const - export let warn: (message: string) => void = () => {} + export let warn: (message: string) => void = () => { } function createAssetFactory(type: string): Factory<[data: string] | [data: Buffer | ArrayBuffer | ArrayBufferView, type: string]> { return (src, ...args) => {