From 25f9b88623cf1ad03afb5b4f21440ff82c4ee686 Mon Sep 17 00:00:00 2001 From: as-op Date: Thu, 19 Sep 2024 16:57:54 +0200 Subject: [PATCH] avoid "growing" code block; avoid removal of empty code blocks --- src/commonmark/commonmarkdataprocessor.js | 52 +++++++++---- src/commonmark/utils/fix-code-blocks.js | 56 -------------- src/commonmark/utils/preprocessor.js | 1 - tests/commonmark/blockquotes.test.js | 4 +- tests/commonmark/code.test.js | 92 +++++++++++++---------- tests/commonmark/escaping.test.js | 2 +- 6 files changed, 92 insertions(+), 115 deletions(-) delete mode 100644 src/commonmark/utils/fix-code-blocks.js diff --git a/src/commonmark/commonmarkdataprocessor.js b/src/commonmark/commonmarkdataprocessor.js index 78b58c2..0bd371f 100644 --- a/src/commonmark/commonmarkdataprocessor.js +++ b/src/commonmark/commonmarkdataprocessor.js @@ -13,7 +13,6 @@ import {HtmlDataProcessor, DomConverter} from '@ckeditor/ckeditor5-engine'; import {highlightedCodeBlock} from 'turndown-plugin-gfm'; import TurndownService from 'turndown'; import {textNodesPreprocessor, linkPreprocessor, breaksPreprocessor} from './utils/preprocessor'; -import {fixBreaksInCodeBlocks, fixCodeBlocks} from "./utils/fix-code-blocks"; import {fixTasklistWhitespaces} from './utils/fix-tasklist-whitespaces'; import {fixBreaksInTables, fixBreaksInLists, fixBreaksOnRootLevel} from "./utils/fix-breaks"; import markdownIt from 'markdown-it'; @@ -21,10 +20,6 @@ import markdownItTaskLists from 'markdown-it-task-lists'; export const originalSrcAttribute = 'data-original-src'; -function debugOutFragment(fragment) { - console.log(Array.prototype.reduce.call(fragment.childNodes, (result, node) => result + (node.outerHTML || node.nodeValue), '')); -} - /** * This data processor implementation uses CommonMark as input/output data. * @@ -54,17 +49,18 @@ export default class CommonMarkDataProcessor { // Use tasklist plugin let parser = md.use(markdownItTaskLists, {label: true}); + const previousRenderer = parser.renderer.rules.code_block; + md.renderer.rules.code_block = function (tokens, idx, options, env, self) { + // markdown-it adds a newline to the end of code blocks, we need to remove it + tokens[idx].content = tokens[idx].content.replace(/\n$/, ''); + return previousRenderer(tokens, idx, options, env, self); + }; + const html = parser.render(data); // Convert input HTML data to DOM DocumentFragment. const domFragment = this._htmlDP._toDom(html); - // Fix empty line on the end of code blocks - fixBreaksInCodeBlocks(domFragment) - - // Fix empty code blocks - fixCodeBlocks(domFragment); - // Fix duplicate whitespace in task lists fixTasklistWhitespaces(domFragment); @@ -113,7 +109,20 @@ export default class CommonMarkDataProcessor { // Use Turndown to convert DOM fragment to markdown const turndownService = new TurndownService({ headingStyle: 'atx', - codeBlockStyle: 'fenced' + codeBlockStyle: 'fenced', + blankReplacement: function (content, node) { + if (node.tagName === 'CODE') { + // we don't want to remove code silently + const prefix = (node.getAttribute('class') || '').replace('language-', ''); + const textContent = node.textContent || ''; + + return "```" + prefix + '\n' + (textContent.length ? textContent : '\n') + "```\n"; + // we don't want to remove pre silently + } else if (node.tagName === 'PRE') { + return content; + } + return node.isBlock ? '\n\n' : '' + }, }); turndownService.use([ @@ -226,7 +235,7 @@ export default class CommonMarkDataProcessor { ) ); }, - replacement: (_content, node) => { + replacement: (_content, node) => { if (!node.parentElement && !node.nextSibling && !node.previousSibling) { //document with only one empty paragraph return ''; } else { @@ -235,9 +244,24 @@ export default class CommonMarkDataProcessor { }, }); + // turndownService.addRule('emptyCode', { + // filter: (node) => { + // console.log(node); + // // return ( + // // (node.nodeName === 'CODE' && node.textContent && node.textContent.includes('###turndown-ignore###')) + // // ); + // return false; + // }, + // replacement: (_content, node) => { + // const s = node.textContent.replace('###turndown-ignore###', ''); + // console.log(s); + // return s; + // }, + // }); + let turndown = turndownService.turndown(domFragment); // Escape non-breaking space characters - return turndown.replace(/\u00A0/, ' '); + return turndown.replace(/\u00A0/, ' ').replace('###turndown-ignore###\n', ''); } } diff --git a/src/commonmark/utils/fix-code-blocks.js b/src/commonmark/utils/fix-code-blocks.js deleted file mode 100644 index 67ade52..0000000 --- a/src/commonmark/utils/fix-code-blocks.js +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Empty code blocks break CKEditor so fix them for now - * https://community.openproject.com/work_packages/31749 - */ -export function fixCodeBlocks(root) { - let walker = document.createNodeIterator( - root, - // Only consider element nodes - NodeFilter.SHOW_ELEMENT, - // Only except text nodes whose parent is one of parents - { - acceptNode: function (node) { - if (node.tagName === 'CODE' && node.parentElement && node.parentElement.tagName === 'PRE') { - return NodeFilter.FILTER_ACCEPT; - } - } - }, - false - ); - - let node; - while (node = walker.nextNode()) { - if (node.children.length === 0 && !node.textContent) { - node.textContent = "\n" - } - } -} - -/** - * Remove forced last line feed in code blocks - */ -export function fixBreaksInCodeBlocks(root) { - let walker = document.createNodeIterator( - root, - // Only consider element nodes - NodeFilter.SHOW_TEXT, - // Only except text nodes whose parent is one of parents - { - acceptNode: function (node) { - if (!node.nextSibling && node.parentElement && node.parentElement.tagName === 'CODE') { - return NodeFilter.FILTER_ACCEPT; - } - } - }, - false - ); - - let node; - while (node = walker.nextNode()) { - if (node.textContent === "\n") { - node.remove(); - } else { - node.textContent = node.textContent.replace(/\n$/g, ''); - } - } -} diff --git a/src/commonmark/utils/preprocessor.js b/src/commonmark/utils/preprocessor.js index 4f47e76..a01127b 100644 --- a/src/commonmark/utils/preprocessor.js +++ b/src/commonmark/utils/preprocessor.js @@ -76,7 +76,6 @@ export function breaksPreprocessor(root, allowed_whitespace_nodes, allowed_raw_n } } - export function hasParentOfType(node, tagNames) { let parent = node.parentElement; diff --git a/tests/commonmark/blockquotes.test.js b/tests/commonmark/blockquotes.test.js index 30da03f..869bf42 100644 --- a/tests/commonmark/blockquotes.test.js +++ b/tests/commonmark/blockquotes.test.js @@ -107,13 +107,13 @@ describe('CommonMarkProcessor', () => { '

Example 1:

' + '
' +
 				'' +
-				'code 1' +
+				'code 1\n' +
 				'' +
 				'
' + '

Example 2:

' + '
' +
 				'' +
-				'code 2' +
+				'code 2\n' +
 				'' +
 				'
' + '', diff --git a/tests/commonmark/code.test.js b/tests/commonmark/code.test.js index 01a05b2..270a891 100644 --- a/tests/commonmark/code.test.js +++ b/tests/commonmark/code.test.js @@ -50,7 +50,9 @@ describe('CommonMarkProcessor', () => { '

some `backticks` inside

' ); }); + }); + describe('code block', () => { it('should process code blocks indented with tabs', () => { testDataProcessor( ' code block', @@ -161,7 +163,7 @@ describe('CommonMarkProcessor', () => { // We will need to handle this separately by some feature. '
var a = \'hello\';\n' +
-				'console.log(a + \' world\');
' + 'console.log(a + \' world\');\n' ); }); @@ -174,7 +176,7 @@ describe('CommonMarkProcessor', () => { // GitHub is rendering as special html with syntax highlighting. // We will need to handle this separately by some feature. - '
#!/bin/bash
', + '
#!/bin/bash\n
', // When converting back ~~~ are normalized to ```. @@ -195,7 +197,7 @@ describe('CommonMarkProcessor', () => { // We will need to handle this separately by some feature. '
var a = \'hello\';\n' +
-				'console.log(a + \' world\');
', + 'console.log(a + \' world\');\n', // When converting back ``````` are normalized to ```. @@ -217,7 +219,7 @@ describe('CommonMarkProcessor', () => { // We will need to handle this separately by some feature. '
var a = \'hello\';\n' +
-				'console.log(a + \' world\');
', + 'console.log(a + \' world\');\n', // When converting back ~~~~~~~~~~ are normalized to ```. @@ -228,41 +230,6 @@ describe('CommonMarkProcessor', () => { ); }); - it('should process empty code block', () => { - testDataProcessor( - '```js\n' + - '```', - - // GitHub is rendering as special html with syntax highlighting. - // We will need to handle this separately by some feature. - - '
\n
', - - // When converting back, empty code blocks will be removed. - // This might be an issue when switching from source to editor - // but changing this cannot be done in to-markdown converters. - '' - ); - }); - - it('should process code block with empty line', () => { - testDataProcessor( - '```js\n' + - '\n' + - '```', - - // GitHub is rendering as special html with syntax highlighting. - // We will need to handle this separately by some feature. - - '
\n
', - - // When converting back, empty code blocks will be removed. - // This might be an issue when switching from source to editor - // but changing this cannot be done in to-markdown converters. - '' - ); - }); - it('should process nested code', () => { testDataProcessor( '````` code `` code ``` `````', @@ -288,7 +255,7 @@ describe('CommonMarkProcessor', () => { '
' +
 				'```\n' +
 				'Code\n' +
-				'```' +
+				'```\n' +
 				'
' ); }); @@ -308,9 +275,52 @@ describe('CommonMarkProcessor', () => { '```\n' + 'Code\n' + '```\n' + - '````' + + '````\n' + '' ); }); + + it('should process empty code block', () => { + testDataProcessor( + '```js\n' + + '```', + '
', + // we always keep min one line in code block + '```js\n' + + '\n' + + '```', + ); + }); + + it('should process code block with empty line', () => { + testDataProcessor( + '```js\n' + + '\n' + + '```', + + // GitHub is rendering as special html with syntax highlighting. + // We will need to handle this separately by some feature. + + '
\n
', + + '```js\n' + + '\n' + + '```', + ); + }); + + it('should keep the amount of empty lines', () => { + testDataProcessor( + '```js\n' + + '\n\n\n' + + '```', + '
\n\n\n
', + + '```js\n' + + '\n\n\n' + + '```', + ); + }); + }); }); diff --git a/tests/commonmark/escaping.test.js b/tests/commonmark/escaping.test.js index cbf0961..a3bfca7 100644 --- a/tests/commonmark/escaping.test.js +++ b/tests/commonmark/escaping.test.js @@ -94,7 +94,7 @@ describe('Commonmark', () => { '\\

Test\\

\n' + '```', '
' +
-					'\\
Test\\
' +
+					'\\
Test\\
\n' +
 					'
'); });