Merge pull request #9186 from hicommonwealth/tim/decode-db-contents-s…

…cripts

libs/model/package.json

@@ -47,6 +47,7 @@
     "node-fetch": "2",
     "node-object-hash": "^3.0.0",
     "pg": "^8.11.3",
+    "quill-delta-to-markdown": "^0.6.0",
     "sequelize": "^6.32.1",
     "umzug": "^3.7.0",
     "uuid": "^9.0.1",

libs/model/src/comment/CreateComment.command.ts

@@ -1,5 +1,5 @@
 import { EventNames, InvalidState, type Command } from '@hicommonwealth/core';
-import { getCommentSearchVector } from '@hicommonwealth/model';
+import { decodeContent, getCommentSearchVector } from '@hicommonwealth/model';
 import * as schemas from '@hicommonwealth/schemas';
 import { models } from '../database';
 import { isAuthorized, type AuthContext } from '../middleware';
@@ -10,7 +10,6 @@ import {
   emitMentions,
   parseUserMentions,
   quillToPlain,
-  sanitizeQuillText,
   uniqueMentions,
 } from '../utils';
 import { getCommentDepth } from '../utils/getCommentDepth';
@@ -53,7 +52,7 @@ export function CreateComment(): Command<
           throw new InvalidState(CreateCommentErrors.NestingTooDeep);
       }
 
-      const text = sanitizeQuillText(payload.text);
+      const text = decodeContent(payload.text);
       const plaintext = quillToPlain(text);
       const mentions = uniqueMentions(parseUserMentions(text));
 

libs/model/src/comment/UpdateComment.command.ts

@@ -5,11 +5,11 @@ import { isAuthorized, type AuthContext } from '../middleware';
 import { mustBeAuthorized } from '../middleware/guards';
 import { getCommentSearchVector } from '../models';
 import {
+  decodeContent,
   emitMentions,
   findMentionDiff,
   parseUserMentions,
   quillToPlain,
-  sanitizeQuillText,
   uniqueMentions,
 } from '../utils';
 
@@ -38,7 +38,7 @@ export function UpdateComment(): Command<
       });
 
       if (currentVersion?.text !== payload.text) {
-        const text = sanitizeQuillText(payload.text);
+        const text = decodeContent(payload.text);
         const plaintext = quillToPlain(text);
         const mentions = findMentionDiff(
           parseUserMentions(currentVersion?.text),

libs/model/src/thread/CreateThread.command.ts

@@ -17,10 +17,10 @@ import { mustBeAuthorized } from '../middleware/guards';
 import { getThreadSearchVector } from '../models/thread';
 import { tokenBalanceCache } from '../services';
 import {
+  decodeContent,
   emitMentions,
   parseUserMentions,
   quillToPlain,
-  sanitizeQuillText,
   uniqueMentions,
 } from '../utils';
 
@@ -112,7 +112,7 @@ export function CreateThread(): Command<
         checkContestLimits(activeContestManagers, actor.address!);
       }
 
-      const body = sanitizeQuillText(payload.body);
+      const body = decodeContent(payload.body);
       const plaintext = kind === 'discussion' ? quillToPlain(body) : body;
       const mentions = uniqueMentions(parseUserMentions(body));
 

libs/model/src/utils/decodeContent.ts

@@ -0,0 +1,27 @@
+// @ts-expect-error quill-delta-to-markdown doesn't have types
+import { deltaToMarkdown } from 'quill-delta-to-markdown';
+
+export function decodeContent(content: string) {
+  // decode if URI encoded
+  let decodedContent: string;
+  try {
+    decodedContent = decodeURIComponent(content);
+  } catch {
+    decodedContent = content;
+  }
+
+  // convert to Markdown if Quill Delta format
+  let rawMarkdown: string;
+  try {
+    const delta = JSON.parse(decodedContent);
+    if ('ops' in delta) {
+      rawMarkdown = deltaToMarkdown(delta.ops);
+    } else {
+      rawMarkdown = delta;
+    }
+  } catch (e) {
+    rawMarkdown = decodedContent;
+  }
+
+  return rawMarkdown.trim();
+}

libs/model/src/utils/index.ts

@@ -1,3 +1,4 @@
+export * from './decodeContent';
 export * from './denormalizedCountUtils';
 export * from './getDelta';
 export * from './parseUserMentions';

libs/shared/src/utils.ts

@@ -308,3 +308,11 @@ export function getWebhookDestination(webhookUrl = ''): string {
 
   return destination;
 }
+
+export function getDecodedString(str: string) {
+  try {
+    return decodeURIComponent(str);
+  } catch (err) {
+    return str;
+  }
+}

packages/commonwealth/client/scripts/models/Comment.ts

@@ -1,3 +1,4 @@
+import { getDecodedString } from '@hicommonwealth/shared';
 import type momentType from 'moment';
 import moment, { Moment } from 'moment';
 import AddressInfo from './AddressInfo';
@@ -64,7 +65,7 @@ export class Comment<T extends IUniqueId> {
     const versionHistory = CommentVersionHistories;
     this.communityId = community_id;
     this.author = Address?.address || author;
-    this.text = deleted_at?.length > 0 ? '[deleted]' : decodeURIComponent(text);
+    this.text = deleted_at?.length > 0 ? '[deleted]' : getDecodedString(text);
     this.plaintext = deleted_at?.length > 0 ? '[deleted]' : plaintext;
     this.versionHistory = versionHistory;
     this.threadId = thread_id;

packages/commonwealth/client/scripts/models/Thread.ts

@@ -4,7 +4,7 @@ import {
   ContestManager,
   ContestScore,
 } from '@hicommonwealth/schemas';
-import { ProposalType } from '@hicommonwealth/shared';
+import { ProposalType, getDecodedString } from '@hicommonwealth/shared';
 import { UserProfile, addressToUserProfile } from 'models/MinimumProfile';
 import moment, { Moment } from 'moment';
 import { z } from 'zod';
@@ -14,15 +14,6 @@ import Topic from './Topic';
 import type { IUniqueId } from './interfaces';
 import type { ThreadKind, ThreadStage } from './types';
 
-function getDecodedString(str: string) {
-  try {
-    return decodeURIComponent(str);
-  } catch (err) {
-    console.error(`Could not decode str: "${str}"`);
-    return str;
-  }
-}
-
 function processAssociatedContests(
   associatedContests?: AssociatedContest[] | null,
   contestActions?: ContestActionT[] | null,

packages/commonwealth/client/scripts/state/api/comments/createComment.ts

@@ -36,7 +36,7 @@ export const buildCreateCommentInput = async ({
   return {
     thread_id: threadId,
     parent_id: parentCommentId ?? undefined,
-    text: encodeURIComponent(unescapedText),
+    text: unescapedText,
     ...toCanvasSignedDataApiArgs(canvasSignedData),
   };
 };

packages/commonwealth/client/scripts/state/api/comments/editComment.ts

@@ -38,7 +38,7 @@ export const buildUpdateCommentInput = async ({
     author_community_id: communityId,
     comment_id: commentId,
     community_id: communityId,
-    text: encodeURIComponent(updatedBody),
+    text: updatedBody,
     jwt: userStore.getState().jwt,
     ...toCanvasSignedDataApiArgs(canvasSignedData),
   };
@@ -65,8 +65,6 @@ const useEditCommentMutation = ({
     onSuccess: async (updatedComment) => {
       // @ts-expect-error StrictNullChecks
       const comment = new Comment(updatedComment);
-      console.log({ comment, updatedComment });
-
       // update fetch comments query state with updated comment
       const key = [ApiEndpoints.FETCH_COMMENTS, communityId, threadId];
       queryClient.cancelQueries({ queryKey: key });

packages/commonwealth/client/scripts/state/api/threads/createThread.ts

@@ -41,8 +41,8 @@ export const buildCreateThreadInput = async ({
   return {
     community_id: communityId,
     topic_id: topic.id,
-    title: encodeURIComponent(title),
-    body: encodeURIComponent(body ?? ''),
+    title: title,
+    body: body ?? '',
     kind,
     stage,
     url,

packages/commonwealth/client/scripts/state/api/threads/editThread.ts

@@ -81,8 +81,8 @@ export const buildUpdateThreadInput = async ({
     jwt: userStore.getState().jwt,
     // for edit profile
     ...(url && { url }),
-    ...(newBody && { body: encodeURIComponent(newBody) }),
-    ...(newTitle && { title: encodeURIComponent(newTitle) }),
+    ...(newBody && { body: newBody }),
+    ...(newTitle && { title: newTitle }),
     ...(authorProfile && { author: JSON.stringify(authorProfile) }),
     // for editing thread locked status
     ...(readOnly !== undefined && { locked: readOnly }),

packages/commonwealth/package.json

@@ -158,6 +158,7 @@
     "buffer": "^6.0.3",
     "cheerio": "1.0.0-rc.3",
     "clsx": "^1.2.1",
+    "commonwealth-mdxeditor": "^0.0.3",
     "compression": "^1.7.4",
     "connect-session-sequelize": "^7.1.1",
     "cookie-parser": "^1.4.4",
@@ -261,8 +262,7 @@
     "web3-validator": "2.0.5",
     "yargs": "^17.7.2",
     "zod": "^3.22.4",
-    "zustand": "^4.3.8",
-    "commonwealth-mdxeditor": "^0.0.3"
+    "zustand": "^4.3.8"
   },
   "devDependencies": {
     "@ethersproject/keccak256": "5.7.0",