feat: clear node

yiwen-ai · Oct 10, 2023 · 7a7bdb5 · 7a7bdb5
1 parent be25a40
commit 7a7bdb5
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 1 deletion.
diff --git a/dist/app.js b/dist/app.js
@@ -34,6 +34,7 @@ async function initContext(ctx, next) {
     log.requestUri = ctx.originalUrl;
     log.remoteAddr = ctx.ip;
     log.xRequestID = ctx.get('x-request-id');
+    log.userAgent = ctx.get('user-agent');
     ctx.state.log = log;
     try {
         await next();

diff --git a/dist/ssr.js b/dist/ssr.js
@@ -176,6 +176,7 @@ async function getPublication(headers, cid, gid, language) {
         api.searchParams.append('language', language);
     }
     api.searchParams.append('fields', 'title,updated_at,from_language,authors,content');
+    api.searchParams.append('partial-content', '60'); // 读取 60% 的内容
     headers.accept = 'application/cbor';
     const res = await fetch(api, {
         headers,

diff --git a/dist/tiptap.js b/dist/tiptap.js
@@ -113,6 +113,17 @@ export class JSONDocumentAmender {
     }
     // https://prosemirror.net/docs/ref/#model.Document_Structure
     amendNode(node) {
+        if (!node || node.type === 'invalid') {
+            return;
+        }
+        if (node.type === 'image' && !node.attrs?.src) {
+            node.type = 'invalid';
+            return node;
+        }
+        if (node.type === 'paragraph' && !node.content?.length) {
+            node.type = 'invalid';
+            return node;
+        }
         // attrs: Attrs
         if (uidTypes.includes(node.type) && node.attrs == null) {
             node.attrs = { id: this.amendId('') };
@@ -136,9 +147,19 @@ export class JSONDocumentAmender {
         }
         // content: Node[]
         if (node.content != null) {
+            let prevHardBreak = false;
             for (const child of node.content) {
+                const isHardBreak = child.type === 'paragraph' &&
+                    child.content?.length === 1 &&
+                    child.content[0].type === 'hardBreak';
+                if (prevHardBreak && isHardBreak) {
+                    // 清理连续的 hardBreak
+                    child.type = 'invalid';
+                }
+                prevHardBreak = isHardBreak;
                 this.amendNode(child);
             }
+            node.content = node.content.filter((child) => child.type !== 'invalid');
         }
         return node;
     }

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "webscraper",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "",
   "private": true,
   "main": "dist/main.js",

diff --git a/src/app.ts b/src/app.ts
@@ -50,6 +50,7 @@ async function initContext(ctx: Koa.Context, next: Koa.Next): Promise<void> {
   log.requestUri = ctx.originalUrl
   log.remoteAddr = ctx.ip
   log.xRequestID = ctx.get('x-request-id')
+  log.userAgent = ctx.get('user-agent')
 
   ctx.state.log = log
 

diff --git a/src/ssr.ts b/src/ssr.ts
@@ -262,6 +262,7 @@ async function getPublication(
     'fields',
     'title,updated_at,from_language,authors,content'
   )
+  api.searchParams.append('partial-content', '60') // 读取 60% 的内容
 
   headers.accept = 'application/cbor'
   const res = await fetch(api, {

diff --git a/src/tiptap.ts b/src/tiptap.ts
@@ -132,6 +132,20 @@ export class JSONDocumentAmender {
 
   // https://prosemirror.net/docs/ref/#model.Document_Structure
   amendNode(node: Node): any {
+    if (!node || node.type === 'invalid') {
+      return
+    }
+
+    if (node.type === 'image' && !node.attrs?.src) {
+      node.type = 'invalid'
+      return node
+    }
+
+    if (node.type === 'paragraph' && !node.content?.length) {
+      node.type = 'invalid'
+      return node
+    }
+
     // attrs: Attrs
     if (uidTypes.includes(node.type) && node.attrs == null) {
       node.attrs = { id: this.amendId('') }
@@ -157,9 +171,21 @@ export class JSONDocumentAmender {
 
     // content: Node[]
     if (node.content != null) {
+      let prevHardBreak = false
       for (const child of node.content) {
+        const isHardBreak =
+          child.type === 'paragraph' &&
+          child.content?.length === 1 &&
+          child.content[0].type === 'hardBreak'
+        if (prevHardBreak && isHardBreak) {
+          // 清理连续的 hardBreak
+          child.type = 'invalid'
+        }
+        prevHardBreak = isHardBreak
         this.amendNode(child)
       }
+
+      node.content = node.content.filter((child) => child.type !== 'invalid')
     }
 
     return node