From 31ad6ffd9ac17aacc9e81f8f97ad6b71204d580b Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Mon, 16 Sep 2024 18:43:26 +0300 Subject: [PATCH 01/13] migration --- ...0916154134-remove-plaintext-and-backups.js | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 packages/commonwealth/server/migrations/20240916154134-remove-plaintext-and-backups.js diff --git a/packages/commonwealth/server/migrations/20240916154134-remove-plaintext-and-backups.js b/packages/commonwealth/server/migrations/20240916154134-remove-plaintext-and-backups.js new file mode 100644 index 00000000000..4268c211daf --- /dev/null +++ b/packages/commonwealth/server/migrations/20240916154134-remove-plaintext-and-backups.js @@ -0,0 +1,24 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.sequelize.transaction(async () => { + await queryInterface.removeColumn('Threads', 'plaintext', { + transaction, + }); + await queryInterface.removeColumn('Threads', 'body_backup', { + transaction, + }); + await queryInterface.removeColumn('Comments', 'plaintext', { + transaction, + }); + await queryInterface.removeColumn('Comments', 'text_backup', { + transaction, + }); + }); + }, + + // data cannot be recovered + async down(queryInterface, Sequelize) {}, +}; From e7f57ece01f11f290ee4f100dc5a46e7cea89942 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Mon, 16 Sep 2024 18:46:32 +0300 Subject: [PATCH 02/13] remove Threads.plaintext from backend --- libs/model/src/models/thread.ts | 1 - libs/model/src/thread/CreateThread.command.ts | 3 --- libs/model/src/thread/UpdateThread.command.ts | 3 --- libs/schemas/src/entities/thread.schemas.ts | 1 - 4 files changed, 8 deletions(-) diff --git a/libs/model/src/models/thread.ts b/libs/model/src/models/thread.ts index 7565a347e2e..284fa83e906 100644 --- a/libs/model/src/models/thread.ts +++ b/libs/model/src/models/thread.ts @@ -28,7 +28,6 @@ export default ( created_by: { type: Sequelize.STRING, allowNull: true }, title: { type: Sequelize.TEXT, allowNull: false }, body: { type: Sequelize.TEXT, allowNull: true }, - plaintext: { type: Sequelize.TEXT, allowNull: true }, kind: { type: Sequelize.STRING, allowNull: false }, stage: { type: Sequelize.TEXT, diff --git a/libs/model/src/thread/CreateThread.command.ts b/libs/model/src/thread/CreateThread.command.ts index ad829ad1727..be205402053 100644 --- a/libs/model/src/thread/CreateThread.command.ts +++ b/libs/model/src/thread/CreateThread.command.ts @@ -20,7 +20,6 @@ import { decodeContent, emitMentions, parseUserMentions, - quillToPlain, uniqueMentions, } from '../utils'; @@ -113,7 +112,6 @@ export function CreateThread(): Command< } const body = decodeContent(payload.body); - const plaintext = kind === 'discussion' ? quillToPlain(body) : body; const mentions = uniqueMentions(parseUserMentions(body)); // == mutation transaction boundary == @@ -127,7 +125,6 @@ export function CreateThread(): Command< topic_id, kind, body, - plaintext, view_count: 0, comment_count: 0, reaction_count: 0, diff --git a/libs/model/src/thread/UpdateThread.command.ts b/libs/model/src/thread/UpdateThread.command.ts index 197e3334706..95fae1e601b 100644 --- a/libs/model/src/thread/UpdateThread.command.ts +++ b/libs/model/src/thread/UpdateThread.command.ts @@ -15,7 +15,6 @@ import { emitMentions, findMentionDiff, parseUserMentions, - quillToPlain, sanitizeQuillText, } from '../utils'; @@ -44,7 +43,6 @@ function getContentPatch( if (typeof body !== 'undefined' && thread.kind === 'discussion') { patch.body = sanitizeQuillText(body); - patch.plaintext = quillToPlain(patch.body); } typeof url !== 'undefined' && thread.kind === 'link' && (patch.url = url); @@ -340,7 +338,6 @@ export function UpdateThread(): Command< 'id', 'address_id', 'text', - ['plaintext', 'plainText'], 'created_at', 'updated_at', 'deleted_at', diff --git a/libs/schemas/src/entities/thread.schemas.ts b/libs/schemas/src/entities/thread.schemas.ts index 67260a8633b..bbb2c62c6b7 100644 --- a/libs/schemas/src/entities/thread.schemas.ts +++ b/libs/schemas/src/entities/thread.schemas.ts @@ -11,7 +11,6 @@ export const Thread = z.object({ kind: z.string(), stage: z.string(), body: z.string().nullish(), - plaintext: z.string().nullish(), url: z.string().nullish(), topic_id: PG_INT.nullish(), pinned: z.boolean().nullish(), From 619033acb11eab4e8b97b4d34b192156573d3987 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Mon, 16 Sep 2024 18:48:10 +0300 Subject: [PATCH 03/13] remove Threads.plaintext from client --- packages/commonwealth/client/scripts/models/Comment.ts | 3 --- packages/commonwealth/client/scripts/models/Thread.ts | 7 ------- 2 files changed, 10 deletions(-) diff --git a/packages/commonwealth/client/scripts/models/Comment.ts b/packages/commonwealth/client/scripts/models/Comment.ts index ff056f26717..3020a23c21e 100644 --- a/packages/commonwealth/client/scripts/models/Comment.ts +++ b/packages/commonwealth/client/scripts/models/Comment.ts @@ -20,7 +20,6 @@ export class Comment { public readonly author: string; public readonly Address: AddressInfo; public readonly text: string; - public readonly plaintext: string; public reactions: Reaction[]; public reactionWeightsSum: number; public readonly id: number; @@ -49,7 +48,6 @@ export class Comment { Address, thread_id, parent_id, - plaintext, reactions, reaction_weights_sum, created_at, @@ -66,7 +64,6 @@ export class Comment { this.communityId = community_id; this.author = Address?.address || author; this.text = deleted_at?.length > 0 ? '[deleted]' : getDecodedString(text); - this.plaintext = deleted_at?.length > 0 ? '[deleted]' : plaintext; this.versionHistory = versionHistory; this.threadId = thread_id; this.id = id; diff --git a/packages/commonwealth/client/scripts/models/Thread.ts b/packages/commonwealth/client/scripts/models/Thread.ts index 74a1837c5e5..f71aae713b4 100644 --- a/packages/commonwealth/client/scripts/models/Thread.ts +++ b/packages/commonwealth/client/scripts/models/Thread.ts @@ -195,7 +195,6 @@ type RecentComment = { id: number; address: string; text: string; - plainText: string; created_at: string; updated_at: string; marked_as_spam_at?: string; @@ -233,7 +232,6 @@ export class Thread implements IUniqueId { public readonly authorCommunity: string; public readonly title: string; public readonly body: string; - public readonly plaintext: string; public pinned: boolean; public readonly kind: ThreadKind; public stage: ThreadStage; @@ -289,7 +287,6 @@ export class Thread implements IUniqueId { community_id, read_only, body, - plaintext, url, pinned, collaborators, @@ -336,7 +333,6 @@ export class Thread implements IUniqueId { links?: Link[]; canvas_signed_data?: string; canvas_msg_id?: string; - plaintext?: string; collaborators?: any[]; last_edited: string; locked_at: string; @@ -375,8 +371,6 @@ export class Thread implements IUniqueId { this.title = getDecodedString(title); // @ts-expect-error StrictNullChecks this.body = getDecodedString(body); - // @ts-expect-error StrictNullChecks - this.plaintext = plaintext; this.id = id; this.identifier = `${id}`; this.createdAt = moment(created_at); @@ -444,7 +438,6 @@ export class Thread implements IUniqueId { author: rc?.address, last_edited: rc?.updated_at ? moment(rc.updated_at) : null, created_at: rc?.created_at ? moment(rc?.created_at) : null, - plaintext: rc?.plainText, text: rc?.text, Address: { user_id: rc?.user_id, From 81b8f488029bb150e574c54674f13d67f24434fc Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Mon, 16 Sep 2024 20:36:28 +0300 Subject: [PATCH 04/13] remove Comments.plaintext --- libs/model/src/comment/CreateComment.command.ts | 3 --- libs/model/src/comment/UpdateComment.command.ts | 4 +--- libs/model/src/models/comment.ts | 1 - libs/schemas/src/entities/comment.schemas.ts | 1 - 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/libs/model/src/comment/CreateComment.command.ts b/libs/model/src/comment/CreateComment.command.ts index 1bb77fe1f1a..2947e31754c 100644 --- a/libs/model/src/comment/CreateComment.command.ts +++ b/libs/model/src/comment/CreateComment.command.ts @@ -9,7 +9,6 @@ import { emitEvent, emitMentions, parseUserMentions, - quillToPlain, uniqueMentions, } from '../utils'; import { getCommentDepth } from '../utils/getCommentDepth'; @@ -53,7 +52,6 @@ export function CreateComment(): Command< } const text = decodeContent(payload.text); - const plaintext = quillToPlain(text); const mentions = uniqueMentions(parseUserMentions(text)); // == mutation transaction boundary == @@ -65,7 +63,6 @@ export function CreateComment(): Command< thread_id, parent_id: parent_id ? parent_id.toString() : null, // TODO: change parent_id from string to number text, - plaintext, address_id: address.id!, reaction_count: 0, reaction_weights_sum: 0, diff --git a/libs/model/src/comment/UpdateComment.command.ts b/libs/model/src/comment/UpdateComment.command.ts index d425b943b9e..bd41759783a 100644 --- a/libs/model/src/comment/UpdateComment.command.ts +++ b/libs/model/src/comment/UpdateComment.command.ts @@ -9,7 +9,6 @@ import { emitMentions, findMentionDiff, parseUserMentions, - quillToPlain, uniqueMentions, } from '../utils'; @@ -39,7 +38,6 @@ export function UpdateComment(): Command< if (currentVersion?.text !== payload.text) { const text = decodeContent(payload.text); - const plaintext = quillToPlain(text); const mentions = findMentionDiff( parseUserMentions(currentVersion?.text), uniqueMentions(parseUserMentions(text)), @@ -48,7 +46,7 @@ export function UpdateComment(): Command< // == mutation transaction boundary == await models.sequelize.transaction(async (transaction) => { await models.Comment.update( - { text, plaintext, search: getCommentSearchVector(text) }, + { text, search: getCommentSearchVector(text) }, { where: { id: comment.id }, transaction }, ); diff --git a/libs/model/src/models/comment.ts b/libs/model/src/models/comment.ts index e22ef9803be..4321f183065 100644 --- a/libs/model/src/models/comment.ts +++ b/libs/model/src/models/comment.ts @@ -36,7 +36,6 @@ export default ( address_id: { type: Sequelize.INTEGER, allowNull: true }, created_by: { type: Sequelize.STRING, allowNull: true }, text: { type: Sequelize.TEXT, allowNull: false }, - plaintext: { type: Sequelize.TEXT, allowNull: true }, // canvas-related columns canvas_signed_data: { type: Sequelize.JSONB, allowNull: true }, diff --git a/libs/schemas/src/entities/comment.schemas.ts b/libs/schemas/src/entities/comment.schemas.ts index a468df8570e..49493b064c1 100644 --- a/libs/schemas/src/entities/comment.schemas.ts +++ b/libs/schemas/src/entities/comment.schemas.ts @@ -9,7 +9,6 @@ export const Comment = z.object({ thread_id: PG_INT, address_id: PG_INT, text: z.string(), - plaintext: z.string(), parent_id: z.string().nullish(), canvas_signed_data: z.string().nullish(), From f346091814605523970ad7b1f9cd644015a35a47 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Mon, 16 Sep 2024 21:21:16 +0300 Subject: [PATCH 05/13] remove plaintext instances p1 --- .../client/scripts/state/api/comments/deleteComment.ts | 1 - packages/commonwealth/test/integration/databaseCleaner.spec.ts | 1 - .../commonwealth/test/integration/knock/commentCreated.spec.ts | 3 --- 3 files changed, 5 deletions(-) diff --git a/packages/commonwealth/client/scripts/state/api/comments/deleteComment.ts b/packages/commonwealth/client/scripts/state/api/comments/deleteComment.ts index 10360dbb0fc..c7cca53f8cd 100644 --- a/packages/commonwealth/client/scripts/state/api/comments/deleteComment.ts +++ b/packages/commonwealth/client/scripts/state/api/comments/deleteComment.ts @@ -49,7 +49,6 @@ const deleteComment = async ({ id: commentId, deleted: true, text: '[deleted]', - plaintext: '[deleted]', versionHistory: [], ...toCanvasSignedDataApiArgs(canvasSignedData), }, diff --git a/packages/commonwealth/test/integration/databaseCleaner.spec.ts b/packages/commonwealth/test/integration/databaseCleaner.spec.ts index 914820db2fe..2b4e7897f5a 100644 --- a/packages/commonwealth/test/integration/databaseCleaner.spec.ts +++ b/packages/commonwealth/test/integration/databaseCleaner.spec.ts @@ -198,7 +198,6 @@ describe('DatabaseCleaner Tests', async () => { text: 'Testing', reaction_count: 0, reaction_weights_sum: 0, - plaintext: 'Testing', search: getCommentSearchVector('Testing'), }); diff --git a/packages/commonwealth/test/integration/knock/commentCreated.spec.ts b/packages/commonwealth/test/integration/knock/commentCreated.spec.ts index cee1e97ba0d..4fa218a9fdc 100644 --- a/packages/commonwealth/test/integration/knock/commentCreated.spec.ts +++ b/packages/commonwealth/test/integration/knock/commentCreated.spec.ts @@ -103,9 +103,6 @@ describe('CommentCreated Event Handler', () => { text: `Hi [@${mentionedUser!.profile.name}](/profile/id/${ mentionedUser!.id }).`, - plaintext: `Hi [@${mentionedUser!.profile.name}](/profile/id/${ - mentionedUser!.id - }).`, parent_id: String(rootComment!.id), thread_id: thread!.id!, address_id: community!.Addresses![0].id, From 51dd3329dc546c5c773da51143205179662cfa00 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Tue, 17 Sep 2024 14:35:51 +0300 Subject: [PATCH 06/13] remove plaintext instances p2 --- libs/model/src/globalActivityCache.ts | 1 - libs/model/src/tester/e2eSeeds.ts | 4 ---- libs/model/test/util-tests/getCommentDepth.spec.ts | 4 +--- libs/schemas/src/entities/notification.schemas.ts | 1 - libs/schemas/src/entities/thread.schemas.ts | 10 +++++----- libs/schemas/src/queries/thread.schemas.ts | 1 - .../client/scripts/state/api/feeds/util.ts | 2 -- .../ThreadCard/ThreadOptions/ThreadOptions.tsx | 2 +- .../server_threads_methods/get_active_threads.ts | 1 - .../server_threads_methods/get_bulk_threads.ts | 4 ++-- .../commonwealth/server/util/sanitizeDeletedComment.ts | 1 - 11 files changed, 9 insertions(+), 22 deletions(-) diff --git a/libs/model/src/globalActivityCache.ts b/libs/model/src/globalActivityCache.ts index 90199d3f127..fc12887b8da 100644 --- a/libs/model/src/globalActivityCache.ts +++ b/libs/model/src/globalActivityCache.ts @@ -29,7 +29,6 @@ export async function getActivityFeed(models: DB, id = 0) { json_build_object( 'id', T.id, 'body', T.body, - 'plaintext', T.plaintext, 'title', T.title, 'numberOfComments', T.comment_count, 'created_at', T.created_at, diff --git a/libs/model/src/tester/e2eSeeds.ts b/libs/model/src/tester/e2eSeeds.ts index 55d05befb1d..cae953543d3 100644 --- a/libs/model/src/tester/e2eSeeds.ts +++ b/libs/model/src/tester/e2eSeeds.ts @@ -198,7 +198,6 @@ export const e2eTestEntities = async function ( community_id: 'cmntest', topic_id: -1, kind: 'discussion', - plaintext: 'text', stage: 'discussion', view_count: 0, reaction_count: 0, @@ -231,7 +230,6 @@ export const e2eTestEntities = async function ( community_id: 'cmntest', topic_id: -2, kind: 'discussion', - plaintext: 'text', stage: 'discussion', view_count: 0, reaction_count: 0, @@ -277,7 +275,6 @@ export const e2eTestEntities = async function ( address_id: -1, text: '', thread_id: -1, - plaintext: '', reaction_count: 0, search: getCommentSearchVector(''), }, @@ -300,7 +297,6 @@ export const e2eTestEntities = async function ( address_id: -2, text: '', thread_id: -2, - plaintext: '', reaction_count: 0, search: getCommentSearchVector(''), }, diff --git a/libs/model/test/util-tests/getCommentDepth.spec.ts b/libs/model/test/util-tests/getCommentDepth.spec.ts index 12d7590e4e4..99a02dca6c2 100644 --- a/libs/model/test/util-tests/getCommentDepth.spec.ts +++ b/libs/model/test/util-tests/getCommentDepth.spec.ts @@ -24,10 +24,8 @@ describe('getCommentDepth', () => { }); const thread = await models.Thread.create({ community_id, - // @ts-expect-error StrictNullChecks - address_id: address.id, + address_id: address!.id!, title: 'Testing', - plaintext: '', kind: 'discussion', search: getThreadSearchVector('Testing', ''), }); diff --git a/libs/schemas/src/entities/notification.schemas.ts b/libs/schemas/src/entities/notification.schemas.ts index 6ad18a4f48c..6de39758ada 100644 --- a/libs/schemas/src/entities/notification.schemas.ts +++ b/libs/schemas/src/entities/notification.schemas.ts @@ -84,7 +84,6 @@ export const CommentSubscription = z.object({ created_at: true, updated_at: true, text: true, - plaintext: true, }) .merge( z.object({ diff --git a/libs/schemas/src/entities/thread.schemas.ts b/libs/schemas/src/entities/thread.schemas.ts index bbb2c62c6b7..d3a3423e8e5 100644 --- a/libs/schemas/src/entities/thread.schemas.ts +++ b/libs/schemas/src/entities/thread.schemas.ts @@ -9,13 +9,13 @@ export const Thread = z.object({ address_id: PG_INT, title: z.string(), kind: z.string(), - stage: z.string(), + stage: z.string().optional(), body: z.string().nullish(), url: z.string().nullish(), topic_id: PG_INT.nullish(), pinned: z.boolean().nullish(), community_id: z.string(), - view_count: PG_INT, + view_count: PG_INT.optional(), links: z.object(linksSchema).array().nullish(), read_only: z.boolean().nullish(), @@ -36,9 +36,9 @@ export const Thread = z.object({ discord_meta: DiscordMetaSchema.nullish(), //counts - reaction_count: PG_INT, - reaction_weights_sum: PG_INT, - comment_count: PG_INT, + reaction_count: PG_INT.optional(), + reaction_weights_sum: PG_INT.optional(), + comment_count: PG_INT.optional().optional(), activity_rank_date: z.coerce.date().nullish(), diff --git a/libs/schemas/src/queries/thread.schemas.ts b/libs/schemas/src/queries/thread.schemas.ts index 5aeed8e39d9..2fa4e3c6880 100644 --- a/libs/schemas/src/queries/thread.schemas.ts +++ b/libs/schemas/src/queries/thread.schemas.ts @@ -34,7 +34,6 @@ export const BulkThread = z.object({ collaborators: z.any().array(), has_poll: z.boolean().nullable().optional(), last_commented_on: z.date().nullable().optional(), - plaintext: z.string().nullable().optional(), Address: z.object({ id: PG_INT, address: z.string(), diff --git a/packages/commonwealth/client/scripts/state/api/feeds/util.ts b/packages/commonwealth/client/scripts/state/api/feeds/util.ts index 764f2489ab8..47e7ef5b5f3 100644 --- a/packages/commonwealth/client/scripts/state/api/feeds/util.ts +++ b/packages/commonwealth/client/scripts/state/api/feeds/util.ts @@ -7,7 +7,6 @@ type ActivityResponse = { thread: { id: number; body: string; - plaintext: string; title: string; numberOfComments: number; created_at: string; @@ -54,7 +53,6 @@ export function formatActivityResponse(response: AxiosResponse) { body: x.thread.body, discord_meta: x.thread.discord_meta, numberOfComments: x.thread.numberOfComments, - plaintext: x.thread.plaintext, read_only: x.thread.read_only, archived_at: x.thread.archived_at, // @ts-expect-error diff --git a/packages/commonwealth/client/scripts/views/pages/discussions/ThreadCard/ThreadOptions/ThreadOptions.tsx b/packages/commonwealth/client/scripts/views/pages/discussions/ThreadCard/ThreadOptions/ThreadOptions.tsx index 1d6b1fc4f77..cd7c1ecfc3c 100644 --- a/packages/commonwealth/client/scripts/views/pages/discussions/ThreadCard/ThreadOptions/ThreadOptions.tsx +++ b/packages/commonwealth/client/scripts/views/pages/discussions/ThreadCard/ThreadOptions/ThreadOptions.tsx @@ -69,7 +69,7 @@ export const ThreadOptions = ({ const userStore = useUserStore(); const handleDownloadMarkdown = () => { - downloadDataAsFile(thread.plaintext, 'text/markdown', thread.title + '.md'); + downloadDataAsFile(thread.body, 'text/markdown', thread.title + '.md'); }; const [verifiedCanvasSignedData, setVerifiedCanvasSignedData] = diff --git a/packages/commonwealth/server/controllers/server_threads_methods/get_active_threads.ts b/packages/commonwealth/server/controllers/server_threads_methods/get_active_threads.ts index e2004868f4d..bbeef051d2c 100644 --- a/packages/commonwealth/server/controllers/server_threads_methods/get_active_threads.ts +++ b/packages/commonwealth/server/controllers/server_threads_methods/get_active_threads.ts @@ -63,7 +63,6 @@ export async function __getActiveThreads( 'id', 'address_id', 'text', - ['plaintext', 'plainText'], 'created_at', 'updated_at', 'deleted_at', diff --git a/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts b/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts index 1c957bc8533..a024a1822e1 100644 --- a/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts +++ b/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts @@ -120,9 +120,9 @@ export async function __getBulkThreads( top_threads AS ( SELECT id, title, url, body, kind, stage, read_only, discord_meta, pinned, community_id, T.created_at, updated_at, locked_at as thread_locked, links, - has_poll, last_commented_on, plaintext, comment_count as "numberOfComments", + has_poll, last_commented_on, comment_count as "numberOfComments", marked_as_spam_at, archived_at, topic_id, reaction_weights_sum, canvas_signed_data, - canvas_msg_id, plaintext, last_edited, address_id + canvas_msg_id, last_edited, address_id FROM "Threads" T WHERE community_id = :communityId AND diff --git a/packages/commonwealth/server/util/sanitizeDeletedComment.ts b/packages/commonwealth/server/util/sanitizeDeletedComment.ts index ab6f27f9658..04f0bb5fdb9 100644 --- a/packages/commonwealth/server/util/sanitizeDeletedComment.ts +++ b/packages/commonwealth/server/util/sanitizeDeletedComment.ts @@ -21,7 +21,6 @@ export function sanitizeDeletedComment( address_id: 0, canvas_msg_id: null, canvas_signed_data: null, - plaintext: '[deleted]', text: '[deleted]', }; } From 872df521e094bcd58c53c2ae31ca1dd06599c5af Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Tue, 17 Sep 2024 14:37:25 +0300 Subject: [PATCH 07/13] remove decode content script --- .../commonwealth/scripts/decode-db-content.ts | 332 ------------------ 1 file changed, 332 deletions(-) delete mode 100644 packages/commonwealth/scripts/decode-db-content.ts diff --git a/packages/commonwealth/scripts/decode-db-content.ts b/packages/commonwealth/scripts/decode-db-content.ts deleted file mode 100644 index 1ff2145278f..00000000000 --- a/packages/commonwealth/scripts/decode-db-content.ts +++ /dev/null @@ -1,332 +0,0 @@ -import { dispose, logger } from '@hicommonwealth/core'; -import { decodeContent, models } from '@hicommonwealth/model'; -import { Op, QueryTypes } from 'sequelize'; - -const log = logger(import.meta); -const BATCH_SIZE = 10; -const queryCase = 'WHEN id = ? THEN ? '; - -async function decodeThreads(lastId: number = 0) { - let lastThreadId = lastId; - while (true) { - const transaction = await models.sequelize.transaction(); - try { - const threads = await models.Thread.findAll({ - attributes: ['id', 'title', 'body'], - where: { - id: { - [Op.gt]: lastThreadId, - }, - }, - order: [['id', 'ASC']], - limit: BATCH_SIZE, - lock: transaction.LOCK.UPDATE, - transaction, - paranoid: false, - }); - if (threads.length === 0) { - await transaction.rollback(); - break; - } - - lastThreadId = threads.at(-1)!.id!; - - let queryTitleCases = ''; - let queryBodyCases = ''; - const titleReplacements: (number | string)[] = []; - const bodyReplacements: (number | string)[] = []; - const threadIds: number[] = []; - for (const { id, title, body } of threads) { - const decodedTitle = decodeContent(title); - queryTitleCases += queryCase; - titleReplacements.push(id!, decodedTitle); - - const decodedBody = body ? decodeContent(body) : ''; - queryBodyCases += queryCase; - bodyReplacements.push(id!, decodedBody); - - threadIds.push(id!); - } - - if (threadIds.length > 0) { - await models.sequelize.query( - ` - UPDATE "Threads" - SET title = CASE - ${queryTitleCases} - END, - body = CASE - ${queryBodyCases} - END - WHERE id IN (?); - `, - { - replacements: [ - ...titleReplacements, - ...bodyReplacements, - threadIds, - ], - type: QueryTypes.BULKUPDATE, - transaction, - }, - ); - } - await transaction.commit(); - log.info( - 'Successfully decoded threads' + - ` ${threads[0].id} to ${threads.at(-1)!.id}`, - ); - } catch (e) { - log.error('Failed to update', e); - await transaction.rollback(); - break; - } - } -} - -async function decodeThreadVersionHistory(lastId: number = 0) { - let lastVersionHistoryId = lastId; - while (true) { - const transaction = await models.sequelize.transaction(); - try { - const threads = await models.ThreadVersionHistory.findAll({ - attributes: ['id', 'body'], - where: { - id: { - [Op.gt]: lastVersionHistoryId, - }, - }, - order: [['id', 'ASC']], - limit: BATCH_SIZE, - lock: transaction.LOCK.UPDATE, - transaction, - }); - - if (threads.length === 0) { - await transaction.rollback(); - break; - } - - lastVersionHistoryId = threads.at(-1)!.id!; - - let queryCases = ''; - const replacements: (number | string)[] = []; - const threadVersionIds: number[] = []; - for (const { id, body } of threads) { - const decodedBody = decodeContent(body); - if (body === decodedBody) continue; - queryCases += queryCase; - replacements.push(id!, decodedBody); - threadVersionIds.push(id!); - } - - if (replacements.length > 0) { - await models.sequelize.query( - ` - UPDATE "ThreadVersionHistories" - SET body = CASE - ${queryCases} - END - WHERE id IN (?); - `, - { - replacements: [...replacements, threadVersionIds], - type: QueryTypes.BULKUPDATE, - transaction, - }, - ); - } - await transaction.commit(); - log.info( - 'Successfully decoded thread version histories ' + - `${threads[0].id} to ${threads.at(-1)!.id}`, - ); - } catch (e) { - log.error('Failed to update', e); - await transaction.rollback(); - break; - } - } -} - -async function decodeCommentVersionHistory(lastId: number = 0) { - let lastCommentVersionId = lastId; - while (true) { - const transaction = await models.sequelize.transaction(); - try { - const comments = await models.CommentVersionHistory.findAll({ - attributes: ['id', 'text'], - where: { - id: { - [Op.gt]: lastCommentVersionId, - }, - }, - order: [['id', 'ASC']], - limit: BATCH_SIZE, - lock: transaction.LOCK.UPDATE, - transaction, - }); - - if (comments.length === 0) { - await transaction.rollback(); - break; - } - - lastCommentVersionId = comments.at(-1)!.id!; - - let queryCases = ''; - const replacements: (number | string)[] = []; - const commentVersionIds: number[] = []; - for (const { id, text } of comments) { - const decodedBody = decodeContent(text); - if (text === decodedBody) continue; - queryCases += queryCase; - replacements.push(id!, decodedBody); - commentVersionIds.push(id!); - } - - if (replacements.length > 0) { - await models.sequelize.query( - ` - UPDATE "CommentVersionHistories" - SET text = CASE - ${queryCases} - END - WHERE id IN (?); - `, - { - replacements: [...replacements, commentVersionIds], - type: QueryTypes.BULKUPDATE, - transaction, - }, - ); - } - await transaction.commit(); - log.info( - 'Successfully decoded comment version histories' + - ` ${comments[0].id} to ${comments.at(-1)!.id}`, - ); - } catch (e) { - log.error('Failed to update', e); - await transaction.rollback(); - break; - } - } -} - -async function decodeComments(lastId: number = 0) { - let lastCommentId = lastId; - while (true) { - const transaction = await models.sequelize.transaction(); - try { - const comments = await models.Comment.findAll({ - attributes: ['id', 'text'], - where: { - id: { - [Op.gt]: lastCommentId, - }, - }, - order: [['id', 'ASC']], - limit: BATCH_SIZE, - lock: transaction.LOCK.UPDATE, - transaction, - paranoid: false, - }); - - if (comments.length === 0) { - await transaction.rollback(); - break; - } - - lastCommentId = comments.at(-1)!.id!; - - let queryCases = ''; - const replacements: (number | string)[] = []; - const commentIds: number[] = []; - for (const { id, text } of comments) { - const decodedBody = decodeContent(text); - if (text === decodedBody) continue; - queryCases += queryCase; - replacements.push(id!, decodedBody); - commentIds.push(id!); - } - - if (replacements.length > 0) { - await models.sequelize.query( - ` - UPDATE "Comments" - SET text = CASE - ${queryCases} - END - WHERE id IN (?); - `, - { - replacements: [...replacements, commentIds], - type: QueryTypes.BULKUPDATE, - transaction, - }, - ); - } - await transaction.commit(); - log.info( - 'Successfully decoded comments' + - ` ${comments[0].id} to ${comments.at(-1)!.id}`, - ); - } catch (e) { - log.error('Failed to update', e); - await transaction.rollback(); - break; - } - } -} - -async function main() { - const acceptedArgs = [ - 'threads', - 'thread-versions', - 'comments', - 'comment-versions', - ]; - if (!acceptedArgs.includes(process.argv[2])) { - log.error(`Must provide one of: ${JSON.stringify(acceptedArgs)}`); - } - - let lastId = 0; - if (process.argv[3]) { - lastId = parseInt(process.argv[3]); - } - - switch (process.argv[2]) { - case 'threads': - await decodeThreads(lastId); - log.info('Thread decoding finished.'); - break; - case 'thread-versions': - await decodeThreadVersionHistory(lastId); - log.info('Thread version history decoding finished.'); - break; - case 'comments': - await decodeComments(lastId); - log.info('Comment decoding finished.'); - break; - case 'comment-versions': - await decodeCommentVersionHistory(lastId); - log.info('Comment version history decoding finished'); - break; - default: - log.error('Invalid argument!'); - } -} - -if (import.meta.url.endsWith(process.argv[1])) { - main() - .then(() => { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - dispose()('EXIT', true); - }) - .catch((err) => { - console.error(err); - // eslint-disable-next-line @typescript-eslint/no-floating-promises - dispose()('ERROR', true); - }); -} From 7231e4b93a6eda5eb130aa77645d65648cdcd7c1 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Tue, 17 Sep 2024 18:54:33 +0300 Subject: [PATCH 08/13] fix migration --- .../migrations/20240916154134-remove-plaintext-and-backups.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/commonwealth/server/migrations/20240916154134-remove-plaintext-and-backups.js b/packages/commonwealth/server/migrations/20240916154134-remove-plaintext-and-backups.js index 4268c211daf..15c10ab8b1d 100644 --- a/packages/commonwealth/server/migrations/20240916154134-remove-plaintext-and-backups.js +++ b/packages/commonwealth/server/migrations/20240916154134-remove-plaintext-and-backups.js @@ -3,7 +3,7 @@ /** @type {import('sequelize-cli').Migration} */ module.exports = { async up(queryInterface, Sequelize) { - await queryInterface.sequelize.transaction(async () => { + await queryInterface.sequelize.transaction(async (transaction) => { await queryInterface.removeColumn('Threads', 'plaintext', { transaction, }); From d1dfe5cda6897ed4105b8c1c4422cfc468e0f610 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Tue, 17 Sep 2024 20:24:14 +0300 Subject: [PATCH 09/13] remove all plaintext references on the client --- .../pages/discussions/ThreadCard/ThreadCard.tsx | 13 +------------ .../views/pages/new_snapshot_proposal/index.tsx | 11 ++++------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/packages/commonwealth/client/scripts/views/pages/discussions/ThreadCard/ThreadCard.tsx b/packages/commonwealth/client/scripts/views/pages/discussions/ThreadCard/ThreadCard.tsx index f69e6b193f5..1132f9eaf3f 100644 --- a/packages/commonwealth/client/scripts/views/pages/discussions/ThreadCard/ThreadCard.tsx +++ b/packages/commonwealth/client/scripts/views/pages/discussions/ThreadCard/ThreadCard.tsx @@ -1,6 +1,5 @@ import clsx from 'clsx'; import { isDefaultStage, threadStageToLabel } from 'helpers'; -import { getBrowserInfo } from 'helpers/browser'; import { GetThreadActionTooltipTextResponse, filterLinks, @@ -118,16 +117,6 @@ export const ThreadCard = ({ (thread.stage && !isStageDefault) || linkedProposals?.length > 0; const stageLabel = threadStageToLabel(thread.stage); - // Future Ref: this fixes https://github.com/hicommonwealth/commonwealth/issues/8611 for iOS mobile - // where quill renders broken/cut-off/overlapping thread.plaintext in cases when there are multiple - //

tags in the quill delta for thread.plaintext or if thread.plaintext has \n characters which - // iOS devices don't seem to render correctly. - // Not updating it for desktop per a previous issue where markdown wasn't rendered correctly in - // preview because of .slice()'d content. - const bodyText = getBrowserInfo().isMobile - ? thread.plaintext.replaceAll(/\n/g, '').slice(0, 150) - : thread.plaintext; - return ( <> diff --git a/packages/commonwealth/client/scripts/views/pages/new_snapshot_proposal/index.tsx b/packages/commonwealth/client/scripts/views/pages/new_snapshot_proposal/index.tsx index 503bd407f45..743e31d0085 100644 --- a/packages/commonwealth/client/scripts/views/pages/new_snapshot_proposal/index.tsx +++ b/packages/commonwealth/client/scripts/views/pages/new_snapshot_proposal/index.tsx @@ -105,8 +105,8 @@ export const NewSnapshotProposalForm = ({ } const initialForm: ThreadForm = { - name: !thread ? '' : thread.title, - body: !thread ? '' : thread.plaintext, + name: thread?.title || '', + body: thread?.body || '', choices: ['Yes', 'No'], range: '5d', start: new Date().getTime(), @@ -129,13 +129,10 @@ export const NewSnapshotProposalForm = ({ const linkMarkdown = `${linkText}[here](${linkUrl})`; - const delta = createDeltaFromText( - thread.plaintext + linkMarkdown, - true, - ); + const delta = createDeltaFromText(thread.body + linkMarkdown, true); setContentDelta(delta); } else { - const delta = createDeltaFromText(thread.plaintext); + const delta = createDeltaFromText(thread.body); setContentDelta(delta); } } From 95f1eba5f53306652acf219eb1d7de2659482574 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Tue, 17 Sep 2024 20:31:18 +0300 Subject: [PATCH 10/13] type fixes --- .../pages/NotificationSettings/CommentSubscriptionEntry.tsx | 2 +- .../pages/NotificationSettings/ThreadSubscriptionEntry.tsx | 2 +- packages/commonwealth/server/routes/viewCount.ts | 2 +- .../commonwealth/test/integration/api/createComment.spec.ts | 4 ++-- .../commonwealth/test/integration/api/createReactions.spec.ts | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/commonwealth/client/scripts/views/pages/NotificationSettings/CommentSubscriptionEntry.tsx b/packages/commonwealth/client/scripts/views/pages/NotificationSettings/CommentSubscriptionEntry.tsx index e88611637fc..f5ca4c5573c 100644 --- a/packages/commonwealth/client/scripts/views/pages/NotificationSettings/CommentSubscriptionEntry.tsx +++ b/packages/commonwealth/client/scripts/views/pages/NotificationSettings/CommentSubscriptionEntry.tsx @@ -122,7 +122,7 @@ export const CommentSubscriptionEntry = (

{ e.preventDefault(); diff --git a/packages/commonwealth/client/scripts/views/pages/NotificationSettings/ThreadSubscriptionEntry.tsx b/packages/commonwealth/client/scripts/views/pages/NotificationSettings/ThreadSubscriptionEntry.tsx index fd28a7cdf89..882b5954187 100644 --- a/packages/commonwealth/client/scripts/views/pages/NotificationSettings/ThreadSubscriptionEntry.tsx +++ b/packages/commonwealth/client/scripts/views/pages/NotificationSettings/ThreadSubscriptionEntry.tsx @@ -107,7 +107,7 @@ export const ThreadSubscriptionEntry = (
{ e.preventDefault(); diff --git a/packages/commonwealth/server/routes/viewCount.ts b/packages/commonwealth/server/routes/viewCount.ts index b4c8776835d..c570d223386 100644 --- a/packages/commonwealth/server/routes/viewCount.ts +++ b/packages/commonwealth/server/routes/viewCount.ts @@ -48,7 +48,7 @@ const viewCount = async ( // add one to view count if not in cache and not newly created if (isNewView) { count = await count.update({ - view_count: count.view_count + 1, + view_count: count.view_count! + 1, }); } diff --git a/packages/commonwealth/test/integration/api/createComment.spec.ts b/packages/commonwealth/test/integration/api/createComment.spec.ts index 27de34ecdf3..6a2ec86306b 100644 --- a/packages/commonwealth/test/integration/api/createComment.spec.ts +++ b/packages/commonwealth/test/integration/api/createComment.spec.ts @@ -148,7 +148,7 @@ describe('createComment Integration Tests', () => { server.e2eTestEntities.testThreads[0].id, ); - chai.assert.equal(afterCommentCount, beforeCommentCount + 1); + chai.assert.equal(afterCommentCount, beforeCommentCount! + 1); chai.assert.isNotNull(comment); chai.assert.equal(response.status, 200); @@ -161,7 +161,7 @@ describe('createComment Integration Tests', () => { ); chai.assert.isNull(comment); - chai.assert.equal(afterCommentCount, beforeCommentCount); + chai.assert.equal(afterCommentCount, beforeCommentCount!); chai.assert.equal(deleteResponse.status, 200); }); }); diff --git a/packages/commonwealth/test/integration/api/createReactions.spec.ts b/packages/commonwealth/test/integration/api/createReactions.spec.ts index eaa00d6e444..1db76677b7a 100644 --- a/packages/commonwealth/test/integration/api/createReactions.spec.ts +++ b/packages/commonwealth/test/integration/api/createReactions.spec.ts @@ -172,7 +172,7 @@ describe('createReaction Integration Tests', () => { chai.assert.isNotNull(createReactionResponse); await thread!.reload(); - chai.assert.equal(thread!.reaction_count, beforeReactionCount + 1); + chai.assert.equal(thread!.reaction_count, beforeReactionCount! + 1); const reactionId = createReactionResponse.id; const deleteReactionResponse = await deleteReaction( @@ -184,6 +184,6 @@ describe('createReaction Integration Tests', () => { chai.assert.equal(deleteReactionResponse.status, 'Success'); await thread!.reload(); - chai.assert.equal(thread!.reaction_count, beforeReactionCount); + chai.assert.equal(thread!.reaction_count, beforeReactionCount!); }); }); From 74303ec2c8f22e9377d0adb8d7cac6d7add66768 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Wed, 18 Sep 2024 15:36:38 +0300 Subject: [PATCH 11/13] RSS feed fixes --- packages/commonwealth/server/routes/feed.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/commonwealth/server/routes/feed.ts b/packages/commonwealth/server/routes/feed.ts index f54ec48293a..b5bd724ff33 100644 --- a/packages/commonwealth/server/routes/feed.ts +++ b/packages/commonwealth/server/routes/feed.ts @@ -59,7 +59,7 @@ export const getFeedHandler = async ( queryValidationResult.data; if (active || search || thread_ids) { - throw new Error('Not implemented'); + throw new AppError('Not implemented'); } // get bulk threads @@ -137,7 +137,7 @@ export const getFeedHandler = async ( }); bulkThreads.threads.forEach((thread) => { - const title = decodeURIComponent(thread.title); + const title = thread.title; const slug = slugify(title); feed.addItem({ title: title, @@ -145,10 +145,7 @@ export const getFeedHandler = async ( id: thread.url, link: `https://common.xyz/${community_id}/discussions/${thread.id}-${slug}`, date: toDate(thread), - // @ts-expect-error StrictNullChecks - content: thread.body, - // @ts-expect-error StrictNullChecks - description: thread.plaintext, + content: thread.body || '', author: [ { // @ts-expect-error StrictNullChecks @@ -162,7 +159,8 @@ export const getFeedHandler = async ( // res.setHeader('content-type', 'text/xml.'); res.setHeader('content-type', 'application/atom+xml.'); + console.log(feed.atom1()); res.write(feed.atom1()); - res.end(); } + res.end(); }; From 16bfb222ede07c64e29ca8c454c8b63d53f8e740 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Wed, 18 Sep 2024 15:47:13 +0300 Subject: [PATCH 12/13] remove plaintext instance in activity feed --- libs/model/src/globalActivityCache.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/model/src/globalActivityCache.ts b/libs/model/src/globalActivityCache.ts index fc12887b8da..9fcd3b13854 100644 --- a/libs/model/src/globalActivityCache.ts +++ b/libs/model/src/globalActivityCache.ts @@ -62,7 +62,6 @@ export async function getActivityFeed(models: DB, id = 0) { 'id', C.id, 'address', A.address, 'text', C.text, - 'plainText', C.plainText, 'created_at', C.created_at::text, 'updated_at', C.updated_at::text, 'deleted_at', C.deleted_at::text, From 4425189160159620676b6a211d407e8a54b04568 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Wed, 2 Oct 2024 09:24:03 +0300 Subject: [PATCH 13/13] remove plaintext reference introduced after master merge --- .../controllers/server_threads_methods/get_bulk_threads.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts b/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts index a024a1822e1..a7a34616e2c 100644 --- a/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts +++ b/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts @@ -235,7 +235,6 @@ export async function __getBulkThreads( 'id', COM.id, 'address', A.address, 'text', COM.text, - 'plainText', COM.plainText, 'created_at', COM.created_at::text, 'updated_at', COM.updated_at::text, 'deleted_at', COM.deleted_at::text,