From 7af0835221997f6ad1836e861d82e9d1dcf0c55b Mon Sep 17 00:00:00 2001 From: Ivan Posti Date: Wed, 13 Dec 2023 12:23:35 +0100 Subject: [PATCH] [#141] Add support for parser cancellation --- .../org/intellij/markdown/ExperimentalApi.kt | 13 ++++++++++ .../intellij/markdown/ast/ASTNodeBuilder.kt | 20 ++++++++++++-- .../markdown/parser/CancellationToken.kt | 12 +++++++++ .../intellij/markdown/parser/InlineBuilder.kt | 11 +++++++- .../markdown/parser/MarkdownParser.kt | 26 ++++++++++++++----- .../intellij/markdown/parser/TreeBuilder.kt | 13 ++++++++-- .../SequentialParserManager.kt | 19 +++++++++++++- 7 files changed, 101 insertions(+), 13 deletions(-) create mode 100644 src/commonMain/kotlin/org/intellij/markdown/ExperimentalApi.kt create mode 100644 src/commonMain/kotlin/org/intellij/markdown/parser/CancellationToken.kt diff --git a/src/commonMain/kotlin/org/intellij/markdown/ExperimentalApi.kt b/src/commonMain/kotlin/org/intellij/markdown/ExperimentalApi.kt new file mode 100644 index 00000000..bae671a3 --- /dev/null +++ b/src/commonMain/kotlin/org/intellij/markdown/ExperimentalApi.kt @@ -0,0 +1,13 @@ +package org.intellij.markdown + +/** + * API elements marked with this annotation should be considered unstable and might change + * in the future breaking source/binary compatibility. + */ +@RequiresOptIn( + message = "This API is experimental and might change in the future.", + level = RequiresOptIn.Level.ERROR +) +@Retention(AnnotationRetention.BINARY) +@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION, AnnotationTarget.CONSTRUCTOR) +annotation class ExperimentalApi diff --git a/src/commonMain/kotlin/org/intellij/markdown/ast/ASTNodeBuilder.kt b/src/commonMain/kotlin/org/intellij/markdown/ast/ASTNodeBuilder.kt index 589c1846..f792ebf5 100644 --- a/src/commonMain/kotlin/org/intellij/markdown/ast/ASTNodeBuilder.kt +++ b/src/commonMain/kotlin/org/intellij/markdown/ast/ASTNodeBuilder.kt @@ -1,17 +1,31 @@ package org.intellij.markdown.ast +import org.intellij.markdown.ExperimentalApi import org.intellij.markdown.IElementType import org.intellij.markdown.MarkdownElementTypes import org.intellij.markdown.MarkdownTokenTypes import org.intellij.markdown.ast.impl.ListCompositeNode import org.intellij.markdown.ast.impl.ListItemCompositeNode +import org.intellij.markdown.parser.CancellationToken -open class ASTNodeBuilder(protected val text: CharSequence) { +open class ASTNodeBuilder @ExperimentalApi constructor( + protected val text: CharSequence, + protected val cancellationToken: CancellationToken +) { + /** + * For compatibility only. + */ + @OptIn(ExperimentalApi::class) + constructor(text: CharSequence): this(text, CancellationToken.NonCancellable) + + @OptIn(ExperimentalApi::class) open fun createLeafNodes(type: IElementType, startOffset: Int, endOffset: Int): List { if (type == MarkdownTokenTypes.WHITE_SPACE) { val result = ArrayList() var lastEol = startOffset while (lastEol < endOffset) { + cancellationToken.checkCancelled() + val nextEol = indexOfSubSeq(text, lastEol, endOffset, '\n') if (nextEol == -1) { break @@ -32,7 +46,9 @@ open class ASTNodeBuilder(protected val text: CharSequence) { return listOf(LeafASTNode(type, startOffset, endOffset)) } + @OptIn(ExperimentalApi::class) open fun createCompositeNode(type: IElementType, children: List): CompositeASTNode { + cancellationToken.checkCancelled() when (type) { MarkdownElementTypes.UNORDERED_LIST, MarkdownElementTypes.ORDERED_LIST -> { @@ -57,4 +73,4 @@ open class ASTNodeBuilder(protected val text: CharSequence) { return -1 } } -} \ No newline at end of file +} diff --git a/src/commonMain/kotlin/org/intellij/markdown/parser/CancellationToken.kt b/src/commonMain/kotlin/org/intellij/markdown/parser/CancellationToken.kt new file mode 100644 index 00000000..5bd23866 --- /dev/null +++ b/src/commonMain/kotlin/org/intellij/markdown/parser/CancellationToken.kt @@ -0,0 +1,12 @@ +package org.intellij.markdown.parser + +import org.intellij.markdown.ExperimentalApi + +@ExperimentalApi +fun interface CancellationToken { + fun checkCancelled() + + object NonCancellable: CancellationToken { + override fun checkCancelled() = Unit + } +} diff --git a/src/commonMain/kotlin/org/intellij/markdown/parser/InlineBuilder.kt b/src/commonMain/kotlin/org/intellij/markdown/parser/InlineBuilder.kt index bbc7bfcd..da01d62b 100644 --- a/src/commonMain/kotlin/org/intellij/markdown/parser/InlineBuilder.kt +++ b/src/commonMain/kotlin/org/intellij/markdown/parser/InlineBuilder.kt @@ -1,11 +1,20 @@ package org.intellij.markdown.parser +import org.intellij.markdown.ExperimentalApi import org.intellij.markdown.ast.ASTNode import org.intellij.markdown.ast.ASTNodeBuilder import org.intellij.markdown.lexer.Compat.assert import org.intellij.markdown.parser.sequentialparsers.TokensCache -class InlineBuilder(nodeBuilder: ASTNodeBuilder, private val tokensCache: TokensCache) : TreeBuilder(nodeBuilder) { +@OptIn(ExperimentalApi::class) +class InlineBuilder @ExperimentalApi constructor( + nodeBuilder: ASTNodeBuilder, + private val tokensCache: TokensCache, + cancellationToken: CancellationToken +): TreeBuilder(nodeBuilder, cancellationToken) { + @OptIn(ExperimentalApi::class) + constructor(nodeBuilder: ASTNodeBuilder, tokensCache: TokensCache): this(nodeBuilder, tokensCache, CancellationToken.NonCancellable) + private var currentTokenPosition = -1 override fun flushEverythingBeforeEvent(event: MyEvent, currentNodeChildren: MutableList?) { diff --git a/src/commonMain/kotlin/org/intellij/markdown/parser/MarkdownParser.kt b/src/commonMain/kotlin/org/intellij/markdown/parser/MarkdownParser.kt index c67d5d4b..cf73eecb 100644 --- a/src/commonMain/kotlin/org/intellij/markdown/parser/MarkdownParser.kt +++ b/src/commonMain/kotlin/org/intellij/markdown/parser/MarkdownParser.kt @@ -11,12 +11,19 @@ import org.intellij.markdown.parser.sequentialparsers.LexerBasedTokensCache import org.intellij.markdown.parser.sequentialparsers.SequentialParser import org.intellij.markdown.parser.sequentialparsers.SequentialParserUtil -class MarkdownParser( +class MarkdownParser @ExperimentalApi constructor( private val flavour: MarkdownFlavourDescriptor, - private val assertionsEnabled: Boolean = true + private val assertionsEnabled: Boolean = true, + private val cancellationToken: CancellationToken = CancellationToken.NonCancellable ) { constructor(flavour: MarkdownFlavourDescriptor): this(flavour, true) + @OptIn(ExperimentalApi::class) + constructor( + flavour: MarkdownFlavourDescriptor, + assertionsEnabled: Boolean + ): this(flavour, assertionsEnabled, CancellationToken.NonCancellable) + fun buildMarkdownTreeFromString(text: String): ASTNode { return parse(MarkdownElementTypes.MARKDOWN_FILE, text, true) } @@ -45,6 +52,7 @@ class MarkdownParser( } } + @OptIn(ExperimentalApi::class) private fun doParse(root: IElementType, text: String, parseInlines: Boolean = true): ASTNode { val productionHolder = ProductionHolder() val markerProcessor = flavour.markerProcessorFactory.createMarkerProcessor(productionHolder) @@ -54,6 +62,7 @@ class MarkdownParser( val textHolder = LookaheadText(text) var pos: LookaheadText.Position? = textHolder.startPosition while (pos != null) { + cancellationToken.checkCancelled() productionHolder.updatePosition(pos.offset) pos = markerProcessor.processPosition(pos) } @@ -74,17 +83,21 @@ class MarkdownParser( return builder.buildTree(productionHolder.production) } + @OptIn(ExperimentalApi::class) private fun doParseInline(root: IElementType, text: CharSequence, textStart: Int, textEnd: Int): ASTNode { val lexer = flavour.createInlinesLexer() lexer.start(text, textStart, textEnd) val tokensCache = LexerBasedTokensCache(lexer) val wholeRange = 0..tokensCache.filteredTokens.size - val nodes = flavour.sequentialParserManager.runParsingSequence(tokensCache, - SequentialParserUtil.filterBlockquotes(tokensCache, wholeRange)) + val nodes = flavour.sequentialParserManager.runParsingSequence( + tokensCache = tokensCache, + rangesToParse = SequentialParserUtil.filterBlockquotes(tokensCache, wholeRange), + cancellationToken = cancellationToken + ) - return InlineBuilder(ASTNodeBuilder(text), tokensCache). - buildTree(nodes + listOf(SequentialParser.Node(wholeRange, root))) + val builder = InlineBuilder(ASTNodeBuilder(text, cancellationToken), tokensCache, cancellationToken) + return builder.buildTree(nodes + listOf(SequentialParser.Node(wholeRange, root))) } private fun topLevelFallback(root: IElementType, text: String): ASTNode { @@ -113,5 +126,4 @@ class MarkdownParser( } } } - } diff --git a/src/commonMain/kotlin/org/intellij/markdown/parser/TreeBuilder.kt b/src/commonMain/kotlin/org/intellij/markdown/parser/TreeBuilder.kt index 908bd4f7..e58a413a 100644 --- a/src/commonMain/kotlin/org/intellij/markdown/parser/TreeBuilder.kt +++ b/src/commonMain/kotlin/org/intellij/markdown/parser/TreeBuilder.kt @@ -1,13 +1,20 @@ package org.intellij.markdown.parser +import org.intellij.markdown.ExperimentalApi import org.intellij.markdown.ast.ASTNode import org.intellij.markdown.ast.ASTNodeBuilder import org.intellij.markdown.lexer.Compat.assert import org.intellij.markdown.lexer.Stack import org.intellij.markdown.parser.sequentialparsers.SequentialParser -abstract class TreeBuilder(protected val nodeBuilder: ASTNodeBuilder) { +abstract class TreeBuilder @ExperimentalApi constructor( + protected val nodeBuilder: ASTNodeBuilder, + protected val cancellationToken: CancellationToken +) { + @OptIn(ExperimentalApi::class) + constructor(nodeBuilder: ASTNodeBuilder): this(nodeBuilder, CancellationToken.NonCancellable) + @OptIn(ExperimentalApi::class) fun buildTree(production: List): ASTNode { val events = constructEvents(production) val markersStack = Stack>>() @@ -18,6 +25,7 @@ abstract class TreeBuilder(protected val nodeBuilder: ASTNodeBuilder) { } for (i in events.indices) { + cancellationToken.checkCancelled() val event = events[i] flushEverythingBeforeEvent(event, if (markersStack.isEmpty()) null else markersStack.peek().second) @@ -55,9 +63,11 @@ abstract class TreeBuilder(protected val nodeBuilder: ASTNodeBuilder) { protected abstract fun flushEverythingBeforeEvent(event: MyEvent, currentNodeChildren: MutableList?) + @OptIn(ExperimentalApi::class) private fun constructEvents(production: List): List { val events = ArrayList() for (index in production.indices) { + cancellationToken.checkCancelled() val result = production[index] val startTokenId = result.range.first val endTokenId = result.range.last @@ -113,5 +123,4 @@ abstract class TreeBuilder(protected val nodeBuilder: ASTNodeBuilder) { } protected class MyASTNodeWrapper(val astNode: ASTNode, val startTokenIndex: Int, val endTokenIndex: Int) - } diff --git a/src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/SequentialParserManager.kt b/src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/SequentialParserManager.kt index d48c960a..9178d179 100644 --- a/src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/SequentialParserManager.kt +++ b/src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/SequentialParserManager.kt @@ -1,15 +1,32 @@ package org.intellij.markdown.parser.sequentialparsers +import org.intellij.markdown.ExperimentalApi +import org.intellij.markdown.parser.CancellationToken + abstract class SequentialParserManager { abstract fun getParserSequence(): List - fun runParsingSequence(tokensCache: TokensCache, rangesToParse: List): Collection { + @OptIn(ExperimentalApi::class) + fun runParsingSequence( + tokensCache: TokensCache, + rangesToParse: List, + ): Collection { + return runParsingSequence(tokensCache, rangesToParse, CancellationToken.NonCancellable) + } + + @ExperimentalApi + fun runParsingSequence( + tokensCache: TokensCache, + rangesToParse: List, + cancellationToken: CancellationToken + ): Collection { val result = ArrayList() var parsingSpaces = ArrayList>() parsingSpaces.add(rangesToParse) for (sequentialParser in getParserSequence()) { + cancellationToken.checkCancelled() val nextLevelSpaces = ArrayList>() for (parsingSpace in parsingSpaces) {