[#141] Add support for parser cancellation

src/commonMain/kotlin/org/intellij/markdown/ExperimentalApi.kt

@@ -0,0 +1,13 @@
+package org.intellij.markdown
+
+/**
+ * API elements marked with this annotation should be considered unstable and might change
+ * in the future breaking source/binary compatibility.
+ */
+@RequiresOptIn(
+  message = "This API is experimental and might change in the future.",
+  level = RequiresOptIn.Level.ERROR
+)
+@Retention(AnnotationRetention.BINARY)
+@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION, AnnotationTarget.CONSTRUCTOR)
+annotation class ExperimentalApi

src/commonMain/kotlin/org/intellij/markdown/ast/ASTNodeBuilder.kt

@@ -1,17 +1,31 @@
 package org.intellij.markdown.ast
 
+import org.intellij.markdown.ExperimentalApi
 import org.intellij.markdown.IElementType
 import org.intellij.markdown.MarkdownElementTypes
 import org.intellij.markdown.MarkdownTokenTypes
 import org.intellij.markdown.ast.impl.ListCompositeNode
 import org.intellij.markdown.ast.impl.ListItemCompositeNode
+import org.intellij.markdown.parser.CancellationToken
 
-open class ASTNodeBuilder(protected val text: CharSequence) {
+open class ASTNodeBuilder @ExperimentalApi constructor(
+    protected val text: CharSequence,
+    protected val cancellationToken: CancellationToken
+) {
+    /**
+     * For compatibility only.
+     */
+    @OptIn(ExperimentalApi::class)
+    constructor(text: CharSequence): this(text, CancellationToken.NonCancellable)
+
+    @OptIn(ExperimentalApi::class)
     open fun createLeafNodes(type: IElementType, startOffset: Int, endOffset: Int): List<ASTNode> {
         if (type == MarkdownTokenTypes.WHITE_SPACE) {
             val result = ArrayList<ASTNode>()
             var lastEol = startOffset
             while (lastEol < endOffset) {
+                cancellationToken.checkCancelled()
+
                 val nextEol = indexOfSubSeq(text, lastEol, endOffset, '\n')
                 if (nextEol == -1) {
                     break
@@ -32,7 +46,9 @@ open class ASTNodeBuilder(protected val text: CharSequence) {
         return listOf(LeafASTNode(type, startOffset, endOffset))
     }
 
+    @OptIn(ExperimentalApi::class)
     open fun createCompositeNode(type: IElementType, children: List<ASTNode>): CompositeASTNode {
+        cancellationToken.checkCancelled()
         when (type) {
             MarkdownElementTypes.UNORDERED_LIST,
             MarkdownElementTypes.ORDERED_LIST -> {
@@ -57,4 +73,4 @@ open class ASTNodeBuilder(protected val text: CharSequence) {
             return -1
         }
     }
-}
+}

src/commonMain/kotlin/org/intellij/markdown/parser/CancellationToken.kt

@@ -0,0 +1,12 @@
+package org.intellij.markdown.parser
+
+import org.intellij.markdown.ExperimentalApi
+
+@ExperimentalApi
+fun interface CancellationToken {
+    fun checkCancelled()
+
+    object NonCancellable: CancellationToken {
+        override fun checkCancelled() = Unit
+    }
+}

src/commonMain/kotlin/org/intellij/markdown/parser/InlineBuilder.kt

@@ -1,11 +1,20 @@
 package org.intellij.markdown.parser
 
+import org.intellij.markdown.ExperimentalApi
 import org.intellij.markdown.ast.ASTNode
 import org.intellij.markdown.ast.ASTNodeBuilder
 import org.intellij.markdown.lexer.Compat.assert
 import org.intellij.markdown.parser.sequentialparsers.TokensCache
 
-class InlineBuilder(nodeBuilder: ASTNodeBuilder, private val tokensCache: TokensCache) : TreeBuilder(nodeBuilder) {
+@OptIn(ExperimentalApi::class)
+class InlineBuilder @ExperimentalApi constructor(
+    nodeBuilder: ASTNodeBuilder,
+    private val tokensCache: TokensCache,
+    cancellationToken: CancellationToken
+): TreeBuilder(nodeBuilder, cancellationToken) {
+    @OptIn(ExperimentalApi::class)
+    constructor(nodeBuilder: ASTNodeBuilder, tokensCache: TokensCache): this(nodeBuilder, tokensCache, CancellationToken.NonCancellable)
+
     private var currentTokenPosition = -1
 
     override fun flushEverythingBeforeEvent(event: MyEvent, currentNodeChildren: MutableList<MyASTNodeWrapper>?) {

src/commonMain/kotlin/org/intellij/markdown/parser/MarkdownParser.kt

@@ -11,12 +11,19 @@ import org.intellij.markdown.parser.sequentialparsers.LexerBasedTokensCache
 import org.intellij.markdown.parser.sequentialparsers.SequentialParser
 import org.intellij.markdown.parser.sequentialparsers.SequentialParserUtil
 
-class MarkdownParser(
+class MarkdownParser @ExperimentalApi constructor(
     private val flavour: MarkdownFlavourDescriptor,
-    private val assertionsEnabled: Boolean = true
+    private val assertionsEnabled: Boolean = true,
+    private val cancellationToken: CancellationToken = CancellationToken.NonCancellable
 ) {
     constructor(flavour: MarkdownFlavourDescriptor): this(flavour, true)
 
+    @OptIn(ExperimentalApi::class)
+    constructor(
+        flavour: MarkdownFlavourDescriptor,
+        assertionsEnabled: Boolean
+    ): this(flavour, assertionsEnabled, CancellationToken.NonCancellable)
+
     fun buildMarkdownTreeFromString(text: String): ASTNode {
         return parse(MarkdownElementTypes.MARKDOWN_FILE, text, true)
     }
@@ -45,6 +52,7 @@ class MarkdownParser(
         }
     }
 
+    @OptIn(ExperimentalApi::class)
     private fun doParse(root: IElementType, text: String, parseInlines: Boolean = true): ASTNode {
         val productionHolder = ProductionHolder()
         val markerProcessor = flavour.markerProcessorFactory.createMarkerProcessor(productionHolder)
@@ -54,6 +62,7 @@ class MarkdownParser(
         val textHolder = LookaheadText(text)
         var pos: LookaheadText.Position? = textHolder.startPosition
         while (pos != null) {
+            cancellationToken.checkCancelled()
             productionHolder.updatePosition(pos.offset)
             pos = markerProcessor.processPosition(pos)
         }
@@ -74,17 +83,21 @@ class MarkdownParser(
         return builder.buildTree(productionHolder.production)
     }
 
+    @OptIn(ExperimentalApi::class)
     private fun doParseInline(root: IElementType, text: CharSequence, textStart: Int, textEnd: Int): ASTNode {
         val lexer = flavour.createInlinesLexer()
         lexer.start(text, textStart, textEnd)
         val tokensCache = LexerBasedTokensCache(lexer)
 
         val wholeRange = 0..tokensCache.filteredTokens.size
-        val nodes = flavour.sequentialParserManager.runParsingSequence(tokensCache,
-                SequentialParserUtil.filterBlockquotes(tokensCache, wholeRange))
+        val nodes = flavour.sequentialParserManager.runParsingSequence(
+            tokensCache = tokensCache,
+            rangesToParse = SequentialParserUtil.filterBlockquotes(tokensCache, wholeRange),
+            cancellationToken = cancellationToken
+        )
 
-        return InlineBuilder(ASTNodeBuilder(text), tokensCache).
-                buildTree(nodes + listOf(SequentialParser.Node(wholeRange, root)))
+        val builder = InlineBuilder(ASTNodeBuilder(text, cancellationToken), tokensCache, cancellationToken)
+        return builder.buildTree(nodes + listOf(SequentialParser.Node(wholeRange, root)))
     }
 
     private fun topLevelFallback(root: IElementType, text: String): ASTNode {
@@ -113,5 +126,4 @@ class MarkdownParser(
             }
         }
     }
-
 }

src/commonMain/kotlin/org/intellij/markdown/parser/TreeBuilder.kt

@@ -1,13 +1,20 @@
 package org.intellij.markdown.parser
 
+import org.intellij.markdown.ExperimentalApi
 import org.intellij.markdown.ast.ASTNode
 import org.intellij.markdown.ast.ASTNodeBuilder
 import org.intellij.markdown.lexer.Compat.assert
 import org.intellij.markdown.lexer.Stack
 import org.intellij.markdown.parser.sequentialparsers.SequentialParser
 
-abstract class TreeBuilder(protected val nodeBuilder: ASTNodeBuilder) {
+abstract class TreeBuilder @ExperimentalApi constructor(
+    protected val nodeBuilder: ASTNodeBuilder,
+    protected val cancellationToken: CancellationToken
+) {
+    @OptIn(ExperimentalApi::class)
+    constructor(nodeBuilder: ASTNodeBuilder): this(nodeBuilder, CancellationToken.NonCancellable)
 
+    @OptIn(ExperimentalApi::class)
     fun buildTree(production: List<SequentialParser.Node>): ASTNode {
         val events = constructEvents(production)
         val markersStack = Stack<Pair<MyEvent, MutableList<MyASTNodeWrapper>>>()
@@ -18,6 +25,7 @@ abstract class TreeBuilder(protected val nodeBuilder: ASTNodeBuilder) {
         }
 
         for (i in events.indices) {
+            cancellationToken.checkCancelled()
             val event = events[i]
 
             flushEverythingBeforeEvent(event, if (markersStack.isEmpty()) null else markersStack.peek().second)
@@ -55,9 +63,11 @@ abstract class TreeBuilder(protected val nodeBuilder: ASTNodeBuilder) {
 
     protected abstract fun flushEverythingBeforeEvent(event: MyEvent, currentNodeChildren: MutableList<MyASTNodeWrapper>?)
 
+    @OptIn(ExperimentalApi::class)
     private fun constructEvents(production: List<SequentialParser.Node>): List<MyEvent> {
         val events = ArrayList<MyEvent>()
         for (index in production.indices) {
+            cancellationToken.checkCancelled()
             val result = production[index]
             val startTokenId = result.range.first
             val endTokenId = result.range.last
@@ -113,5 +123,4 @@ abstract class TreeBuilder(protected val nodeBuilder: ASTNodeBuilder) {
     }
 
     protected class MyASTNodeWrapper(val astNode: ASTNode, val startTokenIndex: Int, val endTokenIndex: Int)
-
 }

src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/SequentialParserManager.kt

@@ -1,15 +1,32 @@
 package org.intellij.markdown.parser.sequentialparsers
 
+import org.intellij.markdown.ExperimentalApi
+import org.intellij.markdown.parser.CancellationToken
+
 abstract class SequentialParserManager {
     abstract fun getParserSequence(): List<SequentialParser>
 
-    fun runParsingSequence(tokensCache: TokensCache, rangesToParse: List<IntRange>): Collection<SequentialParser.Node> {
+    @OptIn(ExperimentalApi::class)
+    fun runParsingSequence(
+        tokensCache: TokensCache,
+        rangesToParse: List<IntRange>,
+    ): Collection<SequentialParser.Node> {
+        return runParsingSequence(tokensCache, rangesToParse, CancellationToken.NonCancellable)
+    }
+
+    @ExperimentalApi
+    fun runParsingSequence(
+        tokensCache: TokensCache,
+        rangesToParse: List<IntRange>,
+        cancellationToken: CancellationToken
+    ): Collection<SequentialParser.Node> {
         val result = ArrayList<SequentialParser.Node>()
 
         var parsingSpaces = ArrayList<List<IntRange>>()
         parsingSpaces.add(rangesToParse)
 
         for (sequentialParser in getParserSequence()) {
+            cancellationToken.checkCancelled()
             val nextLevelSpaces = ArrayList<List<IntRange>>()
 
             for (parsingSpace in parsingSpaces) {