From b5a8ca9d1fdeb0de58e6dea2ac2fe789114d72ba Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:44:13 -0600 Subject: [PATCH] Rework Async `tree-sitter` Model, Fix Strong Ref Cycle (#225) --- Package.resolved | 18 +- Package.swift | 2 +- .../CodeEditSourceEditor.swift | 2 +- .../TextViewController+HighlightBracket.swift | 6 +- .../TextViewController+Highlighter.swift | 7 +- .../Controller/TextViewController.swift | 7 +- .../Enums/CaptureName.swift | 2 +- .../NSRange+/NSRange+InputEdit.swift | 23 +- .../Extensions/Parser+createTree.swift | 20 -- .../Extensions/TextView+/TextView+Point.swift | 18 + .../TextView+createReadBlock.swift} | 17 +- .../Highlighting/HighlightProviding.swift | 37 +- .../Highlighting/HighlightRange.swift | 9 +- .../Highlighter+NSTextStorageDelegate.swift | 36 ++ .../Highlighting/Highlighter.swift | 225 +++++++----- .../Highlighting/HighlighterTextView.swift | 10 +- .../TreeSitter/LanguageLayer.swift | 47 +-- .../TreeSitter/PthreadLock.swift | 33 -- .../TreeSitter/TreeSitterClient+Edit.swift | 86 +---- .../TreeSitterClient+Highlight.swift | 71 ++-- .../TreeSitter/TreeSitterClient.swift | 332 ++++++++++-------- .../TreeSitter/TreeSitterState.swift | 90 +++-- .../TextViewControllerTests.swift | 2 +- .../TreeSitterClientTests.swift | 111 ++---- 24 files changed, 582 insertions(+), 629 deletions(-) delete mode 100644 Sources/CodeEditSourceEditor/Extensions/Parser+createTree.swift create mode 100644 Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+Point.swift rename Sources/CodeEditSourceEditor/Extensions/{HighlighterTextView+createReadBlock.swift => TextView+/TextView+createReadBlock.swift} (53%) create mode 100644 Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift delete mode 100644 Sources/CodeEditSourceEditor/TreeSitter/PthreadLock.swift diff --git a/Package.resolved b/Package.resolved index 7e2a1391e..5d97a3e82 100644 --- a/Package.resolved +++ b/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/CodeEditApp/CodeEditLanguages.git", "state" : { - "revision" : "af29ab4a15474a0a38ef88ef65c20e58a0812e43", - "version" : "0.1.17" + "revision" : "620b463c88894741e20d4711c9435b33547de5d2", + "version" : "0.1.18" } }, { @@ -14,8 +14,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/CodeEditApp/CodeEditTextView.git", "state" : { - "revision" : "c867fed329b2b4ce91a13742e20626f50cf233bb", - "version" : "0.7.0" + "revision" : "6abce20f1827a3665a5159195157f592352e38b4", + "version" : "0.7.1" } }, { @@ -41,8 +41,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "a902f1823a7ff3c9ab2fba0f992396b948eda307", - "version" : "1.0.5" + "revision" : "d029d9d39c87bed85b1c50adee7c41795261a192", + "version" : "1.0.6" } }, { @@ -57,10 +57,10 @@ { "identity" : "swifttreesitter", "kind" : "remoteSourceControl", - "location" : "https://github.com/ChimeHQ/SwiftTreeSitter", + "location" : "https://github.com/ChimeHQ/SwiftTreeSitter.git", "state" : { - "revision" : "df25a52f72ebc5b50ae20d26d1363793408bb28b", - "version" : "0.7.1" + "revision" : "2599e95310b3159641469d8a21baf2d3d200e61f", + "version" : "0.8.0" } }, { diff --git a/Package.swift b/Package.swift index 8d8837275..2dfe6a02a 100644 --- a/Package.swift +++ b/Package.swift @@ -22,7 +22,7 @@ let package = Package( // tree-sitter languages .package( url: "https://github.com/CodeEditApp/CodeEditLanguages.git", - exact: "0.1.17" + exact: "0.1.18" ), // SwiftLint .package( diff --git a/Sources/CodeEditSourceEditor/CodeEditSourceEditor.swift b/Sources/CodeEditSourceEditor/CodeEditSourceEditor.swift index 7308efa6c..446ec7a8b 100644 --- a/Sources/CodeEditSourceEditor/CodeEditSourceEditor.swift +++ b/Sources/CodeEditSourceEditor/CodeEditSourceEditor.swift @@ -212,7 +212,7 @@ public struct CodeEditSourceEditor: NSViewControllerRepresentable { @MainActor public class Coordinator: NSObject { var parent: CodeEditSourceEditor - var controller: TextViewController? + weak var controller: TextViewController? var isUpdatingFromRepresentable: Bool = false var isUpdateFromTextView: Bool = false diff --git a/Sources/CodeEditSourceEditor/Controller/TextViewController+HighlightBracket.swift b/Sources/CodeEditSourceEditor/Controller/TextViewController+HighlightBracket.swift index 86851f080..0622d5e92 100644 --- a/Sources/CodeEditSourceEditor/Controller/TextViewController+HighlightBracket.swift +++ b/Sources/CodeEditSourceEditor/Controller/TextViewController+HighlightBracket.swift @@ -15,11 +15,11 @@ extension TextViewController { for range in textView.selectionManager.textSelections.map({ $0.range }) { if range.isEmpty, range.location > 0, // Range is not the beginning of the document - let preceedingCharacter = textView.textStorage.substring( + let precedingCharacter = textView.textStorage.substring( from: NSRange(location: range.location - 1, length: 1) // The preceding character exists ) { for pair in BracketPairs.allValues { - if preceedingCharacter == pair.0 { + if precedingCharacter == pair.0 { // Walk forwards if let characterIndex = findClosingPair( pair.0, @@ -34,7 +34,7 @@ extension TextViewController { highlightCharacter(range.location - 1) } } - } else if preceedingCharacter == pair.1 && range.location - 1 > 0 { + } else if precedingCharacter == pair.1 && range.location - 1 > 0 { // Walk backwards if let characterIndex = findClosingPair( pair.1, diff --git a/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift b/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift index e56a2292a..0ad8597e9 100644 --- a/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift +++ b/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift @@ -32,11 +32,8 @@ extension TextViewController { if let highlightProvider = highlightProvider { provider = highlightProvider } else { - let textProvider: ResolvingQueryCursor.TextProvider = { [weak self] range, _ -> String? in - return self?.textView.textStorage.mutableString.substring(with: range) - } - - provider = TreeSitterClient(textProvider: textProvider) + self.treeSitterClient = TreeSitterClient() + provider = self.treeSitterClient! } if let provider = provider { diff --git a/Sources/CodeEditSourceEditor/Controller/TextViewController.swift b/Sources/CodeEditSourceEditor/Controller/TextViewController.swift index 69d4e59ca..2d5dc4b4d 100644 --- a/Sources/CodeEditSourceEditor/Controller/TextViewController.swift +++ b/Sources/CodeEditSourceEditor/Controller/TextViewController.swift @@ -149,7 +149,12 @@ public class TextViewController: NSViewController { } } - internal var highlighter: Highlighter? + var highlighter: Highlighter? + + /// The tree sitter client managed by the source editor. + /// + /// This will be `nil` if another highlighter provider is passed to the source editor. + internal(set) public var treeSitterClient: TreeSitterClient? private var fontCharWidth: CGFloat { (" " as NSString).size(withAttributes: [.font: font]).width } diff --git a/Sources/CodeEditSourceEditor/Enums/CaptureName.swift b/Sources/CodeEditSourceEditor/Enums/CaptureName.swift index 941d1b6a5..b73a9a251 100644 --- a/Sources/CodeEditSourceEditor/Enums/CaptureName.swift +++ b/Sources/CodeEditSourceEditor/Enums/CaptureName.swift @@ -6,7 +6,7 @@ // /// A collection of possible capture names for `tree-sitter` with their respected raw values. -public enum CaptureName: String, CaseIterable { +public enum CaptureName: String, CaseIterable, Sendable { case include case constructor case keyword diff --git a/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift b/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift index 7252541e1..3bc073686 100644 --- a/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift +++ b/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift @@ -6,10 +6,11 @@ // import Foundation +import CodeEditTextView import SwiftTreeSitter extension InputEdit { - init?(range: NSRange, delta: Int, oldEndPoint: Point) { + init?(range: NSRange, delta: Int, oldEndPoint: Point, textView: TextView) { let newEndLocation = NSMaxRange(range) + delta if newEndLocation < 0 { @@ -17,16 +18,18 @@ extension InputEdit { return nil } - // TODO: - Ask why Neon only uses .zero for these - let startPoint: Point = .zero - let newEndPoint: Point = .zero + let newRange = NSRange(location: range.location, length: range.length + delta) + let startPoint = textView.pointForLocation(newRange.location) ?? .zero + let newEndPoint = textView.pointForLocation(newEndLocation) ?? .zero - self.init(startByte: UInt32(range.location * 2), - oldEndByte: UInt32(NSMaxRange(range) * 2), - newEndByte: UInt32(newEndLocation * 2), - startPoint: startPoint, - oldEndPoint: oldEndPoint, - newEndPoint: newEndPoint) + self.init( + startByte: UInt32(range.location * 2), + oldEndByte: UInt32(NSMaxRange(range) * 2), + newEndByte: UInt32(newEndLocation * 2), + startPoint: startPoint, + oldEndPoint: oldEndPoint, + newEndPoint: newEndPoint + ) } } diff --git a/Sources/CodeEditSourceEditor/Extensions/Parser+createTree.swift b/Sources/CodeEditSourceEditor/Extensions/Parser+createTree.swift deleted file mode 100644 index 514c5b5d7..000000000 --- a/Sources/CodeEditSourceEditor/Extensions/Parser+createTree.swift +++ /dev/null @@ -1,20 +0,0 @@ -// -// Parser+createTree.swift -// CodeEditSourceEditor -// -// Created by Khan Winter on 5/20/23. -// - -import Foundation -import SwiftTreeSitter - -extension Parser { - /// Creates a tree-sitter tree. - /// - Parameters: - /// - parser: The parser object to use to parse text. - /// - readBlock: A callback for fetching blocks of text. - /// - Returns: A tree if it could be parsed. - internal func createTree(readBlock: @escaping Parser.ReadBlock) -> Tree? { - return parse(tree: nil, readBlock: readBlock) - } -} diff --git a/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+Point.swift b/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+Point.swift new file mode 100644 index 000000000..3008ae489 --- /dev/null +++ b/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+Point.swift @@ -0,0 +1,18 @@ +// +// TextView+Point.swift +// CodeEditSourceEditor +// +// Created by Khan Winter on 1/18/24. +// + +import Foundation +import CodeEditTextView +import SwiftTreeSitter + +extension TextView { + func pointForLocation(_ location: Int) -> Point? { + guard let linePosition = layoutManager.textLineForOffset(location) else { return nil } + let column = location - linePosition.range.location + return Point(row: linePosition.index, column: column) + } +} diff --git a/Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift b/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+createReadBlock.swift similarity index 53% rename from Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift rename to Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+createReadBlock.swift index ac5976d0a..89eb4d16a 100644 --- a/Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift +++ b/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+createReadBlock.swift @@ -6,20 +6,27 @@ // import Foundation +import CodeEditTextView import SwiftTreeSitter -extension HighlighterTextView { +extension TextView { func createReadBlock() -> Parser.ReadBlock { - return { byteOffset, _ in - let limit = self.documentRange.length + return { [weak self] byteOffset, _ in + let limit = self?.documentRange.length ?? 0 let location = byteOffset / 2 let end = min(location + (1024), limit) - if location > end { + if location > end || self == nil { // Ignore and return nothing, tree-sitter's internal tree can be incorrect in some situations. return nil } let range = NSRange(location.. SwiftTreeSitter.Predicate.TextProvider { + return { [weak self] range, _ in + return self?.stringForRange(range) } } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift b/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift index ec8fe89f1..3d23beb8a 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift @@ -6,42 +6,43 @@ // import Foundation +import CodeEditTextView import CodeEditLanguages import AppKit /// The protocol a class must conform to to be used for highlighting. public protocol HighlightProviding: AnyObject { - /// A unique identifier for the highlighter object. - /// Example: `"CodeEdit.TreeSitterHighlighter"` - /// - Note: This does not need to be *globally* unique, merely unique across all the highlighters used. - var identifier: String { get } - /// Called once to set up the highlight provider with a data source and language. /// - Parameters: /// - textView: The text view to use as a text source. - /// - codeLanguage: The langugage that should be used by the highlighter. - func setUp(textView: HighlighterTextView, codeLanguage: CodeLanguage) + /// - codeLanguage: The language that should be used by the highlighter. + func setUp(textView: TextView, codeLanguage: CodeLanguage) + + /// Notifies the highlighter that an edit is going to happen in the given range. + /// - Parameters: + /// - textView: The text view to use. + /// - range: The range of the incoming edit. + func willApplyEdit(textView: TextView, range: NSRange) /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. /// The returned `IndexSet` should include all indexes that need to be highlighted, including any inserted text. /// - Parameters: - /// - textView:The text view to use. + /// - textView: The text view to use. /// - range: The range of the edit. /// - delta: The length of the edit, can be negative for deletions. - /// - completion: The function to call with an `IndexSet` containing all Indices to invalidate. - func applyEdit(textView: HighlighterTextView, - range: NSRange, - delta: Int, - completion: @escaping ((IndexSet) -> Void)) + /// - Returns: an `IndexSet` containing all Indices to invalidate. + func applyEdit(textView: TextView, range: NSRange, delta: Int, completion: @escaping (IndexSet) -> Void) /// Queries the highlight provider for any ranges to apply highlights to. The highlight provider should return an /// array containing all ranges to highlight, and the capture type for the range. Any ranges or indexes /// excluded from the returned array will be treated as plain text and highlighted as such. /// - Parameters: /// - textView: The text view to use. - /// - range: The range to operate on. - /// - completion: Function to call with all ranges to highlight - func queryHighlightsFor(textView: HighlighterTextView, - range: NSRange, - completion: @escaping (([HighlightRange]) -> Void)) + /// - range: The range to query. + /// - Returns: All highlight ranges for the queried ranges. + func queryHighlightsFor(textView: TextView, range: NSRange, completion: @escaping ([HighlightRange]) -> Void) +} + +extension HighlightProviding { + public func willApplyEdit(textView: TextView, range: NSRange) { } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/HighlightRange.swift b/Sources/CodeEditSourceEditor/Highlighting/HighlightRange.swift index 710e206f0..ffb2837fd 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/HighlightRange.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/HighlightRange.swift @@ -7,13 +7,8 @@ import Foundation -/// This class represents a range to highlight, as well as the capture name for syntax coloring. -public class HighlightRange { - init(range: NSRange, capture: CaptureName?) { - self.range = range - self.capture = capture - } - +/// This struct represents a range to highlight, as well as the capture name for syntax coloring. +public struct HighlightRange: Sendable { let range: NSRange let capture: CaptureName? } diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift new file mode 100644 index 000000000..096302641 --- /dev/null +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift @@ -0,0 +1,36 @@ +// +// Highlighter+NSTextStorageDelegate.swift +// CodeEditSourceEditor +// +// Created by Khan Winter on 1/18/24. +// + +import AppKit + +extension Highlighter: NSTextStorageDelegate { + /// Processes an edited range in the text. + /// Will query tree-sitter for any updated indices and re-highlight only the ranges that need it. + func textStorage( + _ textStorage: NSTextStorage, + didProcessEditing editedMask: NSTextStorageEditActions, + range editedRange: NSRange, + changeInLength delta: Int + ) { + // This method is called whenever attributes are updated, so to avoid re-highlighting the entire document + // each time an attribute is applied, we check to make sure this is in response to an edit. + guard editedMask.contains(.editedCharacters) else { return } + + self.storageDidEdit(editedRange: editedRange, delta: delta) + } + + func textStorage( + _ textStorage: NSTextStorage, + willProcessEditing editedMask: NSTextStorageEditActions, + range editedRange: NSRange, + changeInLength delta: Int + ) { + guard editedMask.contains(.editedCharacters) else { return } + + self.storageWillEdit(editedRange: editedRange) + } +} diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift index 9277b16d5..8beebf067 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift @@ -18,6 +18,7 @@ import CodeEditLanguages /// memory and it will listen for bounds changes, text changes, etc. However, to completely invalidate all /// highlights use the ``invalidate()`` method to re-highlight all (visible) text, and the ``setLanguage`` /// method to update the highlighter with a new language if needed. +@MainActor class Highlighter: NSObject { // MARK: - Index Sets @@ -30,20 +31,19 @@ class Highlighter: NSObject { /// The set of valid indexes private var validSet: IndexSet = .init() - /// The range of the entire document - private var entireTextRange: Range { - return 0..<(textView.textStorage.length) - } - /// The set of visible indexes in tht text view lazy private var visibleSet: IndexSet = { - return IndexSet(integersIn: textView.visibleTextRange ?? NSRange()) + return IndexSet(integersIn: textView?.visibleTextRange ?? NSRange()) }() + // MARK: - Tasks + + private var runningTasks: [UUID: Task] = [:] + // MARK: - UI /// The text view to highlight - private unowned var textView: TextView + private weak var textView: TextView? /// The editor theme private var theme: EditorTheme @@ -55,10 +55,10 @@ class Highlighter: NSObject { private var language: CodeLanguage /// Calculates invalidated ranges given an edit. - private weak var highlightProvider: HighlightProviding? + private(set) weak var highlightProvider: HighlightProviding? /// The length to chunk ranges into when passing to the highlighter. - fileprivate let rangeChunkLimit = 256 + private let rangeChunkLimit = 1024 // MARK: - Init @@ -85,15 +85,19 @@ class Highlighter: NSObject { highlightProvider?.setUp(textView: textView, codeLanguage: language) if let scrollView = textView.enclosingScrollView { - NotificationCenter.default.addObserver(self, - selector: #selector(visibleTextChanged(_:)), - name: NSView.frameDidChangeNotification, - object: scrollView) - - NotificationCenter.default.addObserver(self, - selector: #selector(visibleTextChanged(_:)), - name: NSView.boundsDidChangeNotification, - object: scrollView.contentView) + NotificationCenter.default.addObserver( + self, + selector: #selector(visibleTextChanged(_:)), + name: NSView.frameDidChangeNotification, + object: scrollView + ) + + NotificationCenter.default.addObserver( + self, + selector: #selector(visibleTextChanged(_:)), + name: NSView.boundsDidChangeNotification, + object: scrollView.contentView + ) } } @@ -101,13 +105,15 @@ class Highlighter: NSObject { /// Invalidates all text in the textview. Useful for updating themes. public func invalidate() { - updateVisibleSet() - invalidate(range: NSRange(entireTextRange)) + guard let textView else { return } + updateVisibleSet(textView: textView) + invalidate(range: textView.documentRange) } /// Sets the language and causes a re-highlight of the entire text. /// - Parameter language: The language to update to. public func setLanguage(language: CodeLanguage) { + guard let textView = self.textView else { return } highlightProvider?.setUp(textView: textView, codeLanguage: language) invalidate() } @@ -116,12 +122,15 @@ class Highlighter: NSObject { /// - Parameter provider: The provider to use for future syntax highlights. public func setHighlightProvider(_ provider: HighlightProviding) { self.highlightProvider = provider - highlightProvider?.setUp(textView: textView, codeLanguage: language) + guard let textView = self.textView else { return } + highlightProvider?.setUp(textView: textView, codeLanguage: self.language) invalidate() } deinit { self.attributeProvider = nil + self.textView = nil + self.highlightProvider = nil } } @@ -140,77 +149,100 @@ private extension Highlighter { validSet.subtract(set) - highlightNextRange() + highlightInvalidRanges() } /// Begins highlighting any invalid ranges - func highlightNextRange() { + func highlightInvalidRanges() { // If there aren't any more ranges to highlight, don't do anything, otherwise continue highlighting // any available ranges. - guard let range = getNextRange() else { - return + var rangesToQuery: [NSRange] = [] + while let range = getNextRange() { + rangesToQuery.append(range) + pendingSet.insert(range: range) } - highlight(range: range) - - highlightNextRange() + queryHighlights(for: rangesToQuery) } - /// Highlights the given range - /// - Parameter range: The range to request highlights for. - func highlight(range rangeToHighlight: NSRange) { - pendingSet.insert(integersIn: rangeToHighlight) - - highlightProvider?.queryHighlightsFor( - textView: self.textView, - range: rangeToHighlight - ) { [weak self] highlightRanges in - guard let attributeProvider = self?.attributeProvider, - let textView = self?.textView else { return } - - self?.pendingSet.remove(integersIn: rangeToHighlight) - guard self?.visibleSet.intersects(integersIn: rangeToHighlight) ?? false else { - return + /// Highlights the given ranges + /// - Parameter ranges: The ranges to request highlights for. + func queryHighlights(for rangesToHighlight: [NSRange]) { + guard let textView else { return } + + if !Thread.isMainThread { + DispatchQueue.main.async { [weak self] in + for range in rangesToHighlight { + self?.highlightProvider?.queryHighlightsFor( + textView: textView, + range: range + ) { [weak self] highlights in + self?.applyHighlightResult(highlights, rangeToHighlight: range) + } + } } - self?.validSet.formUnion(IndexSet(integersIn: rangeToHighlight)) + } else { + for range in rangesToHighlight { + highlightProvider?.queryHighlightsFor(textView: textView, range: range) { [weak self] highlights in + self?.applyHighlightResult(highlights, rangeToHighlight: range) + } + } + } + } + + /// Applies a highlight query result to the text view. + /// - Parameters: + /// - results: The result of a highlight query. + /// - rangeToHighlight: The range to apply the highlight to. + private func applyHighlightResult(_ results: [HighlightRange], rangeToHighlight: NSRange) { + guard let attributeProvider = self.attributeProvider else { + return + } - // Loop through each highlight and modify the textStorage accordingly. - textView.layoutManager.beginTransaction() - textView.textStorage.beginEditing() + pendingSet.remove(integersIn: rangeToHighlight) + guard visibleSet.intersects(integersIn: rangeToHighlight) else { + return + } + validSet.formUnion(IndexSet(integersIn: rangeToHighlight)) - // Create a set of indexes that were not highlighted. - var ignoredIndexes = IndexSet(integersIn: rangeToHighlight) + // Loop through each highlight and modify the textStorage accordingly. + textView?.layoutManager.beginTransaction() + textView?.textStorage.beginEditing() - // Apply all highlights that need color - for highlight in highlightRanges { - textView.textStorage.setAttributes( - attributeProvider.attributesFor(highlight.capture), - range: highlight.range - ) + // Create a set of indexes that were not highlighted. + var ignoredIndexes = IndexSet(integersIn: rangeToHighlight) - // Remove highlighted indexes from the "ignored" indexes. - ignoredIndexes.remove(integersIn: highlight.range) - } + // Apply all highlights that need color + for highlight in results + where textView?.documentRange.upperBound ?? 0 > highlight.range.upperBound { + textView?.textStorage.setAttributes( + attributeProvider.attributesFor(highlight.capture), + range: highlight.range + ) - // For any indices left over, we need to apply normal attributes to them - // This fixes the case where characters are changed to have a non-text color, and then are skipped when - // they need to be changed back. - for ignoredRange in ignoredIndexes.rangeView { - textView.textStorage.setAttributes( - attributeProvider.attributesFor(nil), - range: NSRange(ignoredRange) - ) - } + // Remove highlighted indexes from the "ignored" indexes. + ignoredIndexes.remove(integersIn: highlight.range) + } - textView.textStorage.endEditing() - textView.layoutManager.endTransaction() + // For any indices left over, we need to apply normal attributes to them + // This fixes the case where characters are changed to have a non-text color, and then are skipped when + // they need to be changed back. + for ignoredRange in ignoredIndexes.rangeView + where textView?.documentRange.upperBound ?? 0 > ignoredRange.upperBound { + textView?.textStorage.setAttributes( + attributeProvider.attributesFor(nil), + range: NSRange(ignoredRange) + ) } + + textView?.textStorage.endEditing() + textView?.layoutManager.endTransaction() } /// Gets the next `NSRange` to highlight based on the invalid set, visible set, and pending set. /// - Returns: An `NSRange` to highlight if it could be fetched. func getNextRange() -> NSRange? { - let set: IndexSet = IndexSet(integersIn: entireTextRange) // All text + let set: IndexSet = IndexSet(integersIn: textView?.documentRange ?? .zero) // All text .subtracting(validSet) // Subtract valid = Invalid set .intersection(visibleSet) // Only visible indexes .subtracting(pendingSet) // Don't include pending indexes @@ -220,8 +252,10 @@ private extension Highlighter { } // Chunk the ranges in sets of rangeChunkLimit characters. - return NSRange(location: range.lowerBound, - length: min(rangeChunkLimit, range.upperBound - range.lowerBound)) + return NSRange( + location: range.lowerBound, + length: min(rangeChunkLimit, range.upperBound - range.lowerBound) + ) } } @@ -229,7 +263,7 @@ private extension Highlighter { // MARK: - Visible Content Updates private extension Highlighter { - private func updateVisibleSet() { + private func updateVisibleSet(textView: TextView) { if let newVisibleRange = textView.visibleTextRange { visibleSet = IndexSet(integersIn: newVisibleRange) } @@ -237,7 +271,18 @@ private extension Highlighter { /// Updates the view to highlight newly visible text when the textview is scrolled or bounds change. @objc func visibleTextChanged(_ notification: Notification) { - updateVisibleSet() + let textView: TextView + if let clipView = notification.object as? NSClipView, + let documentView = clipView.enclosingScrollView?.documentView as? TextView { + textView = documentView + } else if let scrollView = notification.object as? NSScrollView, + let documentView = scrollView.documentView as? TextView { + textView = documentView + } else { + return + } + + updateVisibleSet(textView: textView) // Any indices that are both *not* valid and in the visible text range should be invalidated let newlyInvalidSet = visibleSet.subtracting(validSet) @@ -248,37 +293,31 @@ private extension Highlighter { } } -// MARK: - NSTextStorageDelegate - -extension Highlighter: NSTextStorageDelegate { - /// Processes an edited range in the text. - /// Will query tree-sitter for any updated indices and re-highlight only the ranges that need it. - func textStorage(_ textStorage: NSTextStorage, - didProcessEditing editedMask: NSTextStorageEditActions, - range editedRange: NSRange, - changeInLength delta: Int) { - // This method is called whenever attributes are updated, so to avoid re-highlighting the entire document - // each time an attribute is applied, we check to make sure this is in response to an edit. - guard editedMask.contains(.editedCharacters) else { - return - } +// MARK: - Editing + +extension Highlighter { + func storageDidEdit(editedRange: NSRange, delta: Int) { + guard let textView else { return } let range = NSRange(location: editedRange.location, length: editedRange.length - delta) if delta > 0 { visibleSet.insert(range: editedRange) } - highlightProvider?.applyEdit(textView: self.textView, - range: range, - delta: delta) { [weak self] invalidatedIndexSet in - let indexSet = invalidatedIndexSet + highlightProvider?.applyEdit(textView: textView, range: range, delta: delta) { [weak self] invalidIndexSet in + let indexSet = invalidIndexSet .union(IndexSet(integersIn: editedRange)) // Only invalidate indices that are visible. - .intersection(self?.visibleSet ?? .init()) + .intersection(self?.visibleSet ?? IndexSet()) for range in indexSet.rangeView { self?.invalidate(range: NSRange(range)) } } } + + func storageWillEdit(editedRange: NSRange) { + guard let textView else { return } + highlightProvider?.willApplyEdit(textView: textView, range: editedRange) + } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/HighlighterTextView.swift b/Sources/CodeEditSourceEditor/Highlighting/HighlighterTextView.swift index ed8b4e355..e4e8930e2 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/HighlighterTextView.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/HighlighterTextView.swift @@ -9,15 +9,7 @@ import Foundation import AppKit import CodeEditTextView -/// The object `HighlightProviding` objects are given when asked for highlights. -public protocol HighlighterTextView: AnyObject { - /// The entire range of the document. - var documentRange: NSRange { get } - /// A substring for the requested range. - func stringForRange(_ nsRange: NSRange) -> String? -} - -extension TextView: HighlighterTextView { +extension TextView { public func stringForRange(_ nsRange: NSRange) -> String? { textStorage.substring(from: nsRange) } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift b/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift index 8054a6290..8d8444523 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift @@ -22,7 +22,7 @@ public class LanguageLayer: Hashable { id: TreeSitterLanguage, parser: Parser, supportsInjections: Bool, - tree: Tree? = nil, + tree: MutableTree? = nil, languageQuery: Query? = nil, ranges: [NSRange] ) { @@ -39,7 +39,7 @@ public class LanguageLayer: Hashable { let id: TreeSitterLanguage let parser: Parser let supportsInjections: Bool - var tree: Tree? + var tree: MutableTree? var languageQuery: Query? var ranges: [NSRange] @@ -48,7 +48,7 @@ public class LanguageLayer: Hashable { id: id, parser: parser, supportsInjections: supportsInjections, - tree: tree?.copy(), + tree: tree?.mutableCopy(), languageQuery: languageQuery, ranges: ranges ) @@ -65,38 +65,36 @@ public class LanguageLayer: Hashable { /// Calculates a series of ranges that have been invalidated by a given edit. /// - Parameters: - /// - textView: The text view to use for text. /// - edit: The edit to act on. /// - timeout: The maximum time interval the parser can run before halting. /// - readBlock: A callback for fetching blocks of text. /// - Returns: An array of distinct `NSRanges` that need to be re-highlighted. func findChangedByteRanges( - textView: HighlighterTextView, edit: InputEdit, timeout: TimeInterval?, readBlock: @escaping Parser.ReadBlock ) throws -> [NSRange] { parser.timeout = timeout ?? 0 - let (oldTree, newTree) = calculateNewState( - tree: self.tree, + let newTree = calculateNewState( + tree: self.tree?.mutableCopy(), parser: self.parser, edit: edit, readBlock: readBlock ) - if oldTree == nil && newTree == nil { + if self.tree == nil && newTree == nil { // There was no existing tree, make a new one and return all indexes. - tree = parser.createTree(readBlock: readBlock) - return [NSRange(textView.documentRange.intRange)] - } else if oldTree != nil && newTree == nil { + self.tree = parser.parse(tree: nil as Tree?, readBlock: readBlock) + return [self.tree?.rootNode?.range ?? .zero] + } else if self.tree != nil && newTree == nil { // The parser timed out, throw Error.parserTimeout } - let ranges = changedByteRanges(oldTree, rhs: newTree).map { $0.range } + let ranges = changedByteRanges(self.tree, newTree).map { $0.range } - tree = newTree + self.tree = newTree return ranges } @@ -110,21 +108,26 @@ public class LanguageLayer: Hashable { /// - readBlock: The block to use to read text. /// - Returns: (The old state, the new state). internal func calculateNewState( - tree: Tree?, + tree: MutableTree?, parser: Parser, edit: InputEdit, readBlock: @escaping Parser.ReadBlock - ) -> (Tree?, Tree?) { - guard let oldTree = tree else { - return (nil, nil) + ) -> MutableTree? { + guard let tree else { + return nil } // Apply the edit to the old tree - oldTree.edit(edit) + tree.edit(edit) - let newTree = parser.parse(tree: oldTree, readBlock: readBlock) + // Check every timeout to see if the task is canceled to avoid parsing after the editor has been closed. + // We can continue a parse after a timeout causes it to cancel by calling parse on the same tree. + var newTree: MutableTree? + while newTree == nil && !Task.isCancelled { + newTree = parser.parse(tree: tree, readBlock: readBlock) + } - return (oldTree.copy(), newTree) + return newTree } /// Calculates the changed byte ranges between two trees. @@ -132,7 +135,7 @@ public class LanguageLayer: Hashable { /// - lhs: The first (older) tree. /// - rhs: The second (newer) tree. /// - Returns: Any changed ranges. - internal func changedByteRanges(_ lhs: Tree?, rhs: Tree?) -> [Range] { + internal func changedByteRanges(_ lhs: MutableTree?, _ rhs: MutableTree?) -> [Range] { switch (lhs, rhs) { case (let tree1?, let tree2?): return tree1.changedRanges(from: tree2).map({ $0.bytes }) @@ -145,7 +148,7 @@ public class LanguageLayer: Hashable { } } - enum Error: Swift.Error { + enum Error: Swift.Error, LocalizedError { case parserTimeout } } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/PthreadLock.swift b/Sources/CodeEditSourceEditor/TreeSitter/PthreadLock.swift deleted file mode 100644 index c34e3fd42..000000000 --- a/Sources/CodeEditSourceEditor/TreeSitter/PthreadLock.swift +++ /dev/null @@ -1,33 +0,0 @@ -// -// PthreadLock.swift -// CodeEditSourceEditor -// -// Created by Khan Winter on 6/2/23. -// - -import Foundation - -/// A thread safe, atomic lock that wraps a `pthread_mutex_t` -class PthreadLock { - private var _lock: pthread_mutex_t - - /// Initializes the lock - init() { - _lock = .init() - pthread_mutex_init(&_lock, nil) - } - - /// Locks the lock, if the lock is already locked it will block the current thread until it unlocks. - func lock() { - pthread_mutex_lock(&_lock) - } - - /// Unlocks the lock. - func unlock() { - pthread_mutex_unlock(&_lock) - } - - deinit { - pthread_mutex_destroy(&_lock) - } -} diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient+Edit.swift index fae473040..ba212b99e 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient+Edit.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient+Edit.swift @@ -10,102 +10,54 @@ import SwiftTreeSitter import CodeEditLanguages extension TreeSitterClient { - /// This class contains an edit state that can be resumed if a parser hits a timeout. - class EditState { - var edit: InputEdit - var rangeSet: IndexSet - var layerSet: Set - var touchedLayers: Set - var completion: ((IndexSet) -> Void) - - init( - edit: InputEdit, - minimumCapacity: Int = 0, - completion: @escaping (IndexSet) -> Void - ) { - self.edit = edit - self.rangeSet = IndexSet() - self.layerSet = Set(minimumCapacity: minimumCapacity) - self.touchedLayers = Set(minimumCapacity: minimumCapacity) - self.completion = completion - } - } - /// Applies the given edit to the current state and calls the editState's completion handler. - /// - Parameters: - /// - editState: The edit state to apply. - /// - startAtLayerIndex: An optional layer index to start from if some work has already been done on this edit - /// state object. - /// - runningAsync: Determine whether or not to timeout long running parse tasks. - internal func applyEdit(editState: EditState, startAtLayerIndex: Int? = nil, runningAsync: Bool = false) { - guard let readBlock, let textView, let state else { return } - stateLock.lock() + /// - Parameter edit: The edit to apply to the internal tree sitter state. + /// - Returns: The set of ranges invalidated by the edit operation. + func applyEdit(edit: InputEdit) -> IndexSet { + guard let state, let readBlock, let readCallback else { return IndexSet() } - // Loop through all layers, apply edits & find changed byte ranges. - let startIdx = startAtLayerIndex ?? 0 - for layerIdx in (startIdx..() + // Loop through all layers, apply edits & find changed byte ranges. + for (idx, layer) in state.layers.enumerated().reversed() { if layer.id != state.primaryLayer.id { // Reversed for safe removal while looping for rangeIdx in (0.. Void) - ) { - stateLock.lock() - guard let textView, let state = state?.copy() else { return } - stateLock.unlock() + func queryHighlightsForRange(range: NSRange) -> [HighlightRange] { + guard let state = self.state, let readCallback else { return [] } var highlights: [HighlightRange] = [] var injectedSet = IndexSet(integersIn: range) @@ -26,12 +20,13 @@ extension TreeSitterClient { // Query injected only if a layer's ranges intersects with `range` for layerRange in layer.ranges { if let rangeIntersection = range.intersection(layerRange) { - highlights.append(contentsOf: queryLayerHighlights( + let queryResult = queryLayerHighlights( layer: layer, - textView: textView, - range: rangeIntersection - )) + range: rangeIntersection, + readCallback: readCallback + ) + highlights.append(contentsOf: queryResult) injectedSet.remove(integersIn: rangeIntersection) } } @@ -39,43 +34,26 @@ extension TreeSitterClient { // Query primary for any ranges that weren't used in the injected layers. for range in injectedSet.rangeView { - highlights.append(contentsOf: queryLayerHighlights( + let queryResult = queryLayerHighlights( layer: state.layers[0], - textView: textView, - range: NSRange(range) - )) + range: NSRange(range), + readCallback: readCallback + ) + highlights.append(contentsOf: queryResult) } - stateLock.unlock() - if !runningAsync { - completion(highlights) - } else { - DispatchQueue.main.async { - completion(highlights) - } - } - } - - internal func queryHighlightsForRangeAsync( - range: NSRange, - completion: @escaping (([HighlightRange]) -> Void) - ) { - queuedQueries.append { - self.queryHighlightsForRange(range: range, runningAsync: true, completion: completion) - } - beginTasksIfNeeded() + return highlights } /// Queries the given language layer for any highlights. /// - Parameters: /// - layer: The layer to query. - /// - textView: A text view to use for contextual data. /// - range: The range to query for. /// - Returns: Any ranges to highlight. internal func queryLayerHighlights( layer: LanguageLayer, - textView: HighlighterTextView, - range: NSRange + range: NSRange, + readCallback: SwiftTreeSitter.Predicate.TextProvider ) -> [HighlightRange] { guard let tree = layer.tree, let rootNode = tree.rootNode else { @@ -84,13 +62,13 @@ extension TreeSitterClient { // This needs to be on the main thread since we're going to use the `textProvider` in // the `highlightsFromCursor` method, which uses the textView's text storage. - guard let cursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { + guard let queryCursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { return [] } - cursor.setRange(range) - cursor.matchLimit = Constants.treeSitterMatchLimit + queryCursor.setRange(range) + queryCursor.matchLimit = Constants.treeSitterMatchLimit - return highlightsFromCursor(cursor: ResolvingQueryCursor(cursor: cursor), includedRange: range) + return highlightsFromCursor(cursor: queryCursor, includedRange: range, readCallback: readCallback) } /// Resolves a query cursor to the highlight ranges it contains. @@ -99,15 +77,18 @@ extension TreeSitterClient { /// - cursor: The cursor to resolve. /// - includedRange: The range to include highlights from. /// - Returns: Any highlight ranges contained in the cursor. - internal func highlightsFromCursor(cursor: ResolvingQueryCursor, includedRange: NSRange) -> [HighlightRange] { - cursor.prepare(with: self.textProvider) + internal func highlightsFromCursor( + cursor: QueryCursor, + includedRange: NSRange, + readCallback: SwiftTreeSitter.Predicate.TextProvider + ) -> [HighlightRange] { return cursor .flatMap { $0.captures } .compactMap { - // Sometimes `cursor.setRange` just doesnt work :( so we have to do a redundant check for a valid range + // Sometimes `cursor.setRange` just doesn't work :( so we have to do a redundant check for a valid range // in the included range let intersectionRange = $0.range.intersection(includedRange) ?? .zero - // Check that the capture name is one CETV can parse. If not, ignore it completely. + // Check that the capture name is one CESE can parse. If not, ignore it completely. if intersectionRange.length > 0, let captureName = CaptureName.fromString($0.name ?? "") { return HighlightRange(range: intersectionRange, capture: captureName) } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift index 1f8c94098..42ce2263b 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift @@ -6,49 +6,59 @@ // import Foundation +import CodeEditTextView import CodeEditLanguages import SwiftTreeSitter +import OSLog -/// `TreeSitterClient` is a class that manages applying edits for and querying captures for a syntax tree. -/// It handles queuing edits, processing them with the given text, and invalidating indices in the text for efficient +/// # TreeSitterClient +/// +/// ``TreeSitterClient`` is an class that manages a tree-sitter syntax tree and provides an API for notifying that +/// tree of edits and querying the tree. This type also conforms to ``HighlightProviding`` to provide syntax /// highlighting. /// -/// Use the `init` method to set up the client initially. If text changes it should be able to be read through the -/// `textProvider` callback. You can optionally update the text manually using the `setText` method. -/// However, the `setText` method will re-compile the entire corpus so should be used sparingly. +/// The APIs this object provides can perform either asynchronously or synchronously. All calls to this object must +/// first be dispatched from the main queue to ensure serial access to internal properties. Any synchronous methods +/// can throw an ``TreeSitterClient/Error/syncUnavailable`` error if an asynchronous or synchronous call is already +/// being made on the object. In those cases it is up to the caller to decide whether or not to retry asynchronously. +/// +/// The only exception to the above rule is the ``HighlightProviding`` conformance methods. The methods for that +/// implementation may return synchronously or asynchronously depending on a variety of factors such as document +/// length, edit length, highlight length and if the object is available for a synchronous call. public final class TreeSitterClient: HighlightProviding { - typealias AsyncCallback = @Sendable () -> Void + static let logger: Logger = Logger(subsystem: "com.CodeEdit.CodeEditSourceEditor", category: "TreeSitterClient") - // MARK: - Properties + /// The number of operations running or enqueued to run on the dispatch queue. This variable **must** only be + /// changed from the main thread or race conditions are very likely. + private var runningOperationCount = 0 - public var identifier: String { - "CodeEdit.TreeSitterClient" - } + /// The number of times the object has been set up. Used to cancel async tasks if + /// ``TreeSitterClient/setUp(textView:codeLanguage:)`` is called. + private var setUpCount = 0 - /// The text view to use as a data source for text. - internal weak var textView: HighlighterTextView? - /// A callback to use to efficiently fetch portions of text. - internal var readBlock: Parser.ReadBlock? + /// The concurrent queue to perform operations on. + private let operationQueue = DispatchQueue( + label: "CodeEditSourceEditor.TreeSitter.EditQueue", + qos: .userInteractive + ) - /// The running background task. - internal var runningTask: Task? - /// An array of all edits queued for execution. - internal var queuedEdits: [AsyncCallback] = [] - /// An array of all highlight queries queued for execution. - internal var queuedQueries: [AsyncCallback] = [] + // MARK: - Properties + + /// A callback to use to efficiently fetch portions of text. + var readBlock: Parser.ReadBlock? - /// A lock that must be obtained whenever `state` is modified - internal var stateLock: PthreadLock = PthreadLock() - /// A lock that must be obtained whenever either `queuedEdits` or `queuedHighlights` is modified - internal var queueLock: PthreadLock = PthreadLock() + /// A callback used to fetch text for queries. + var readCallback: SwiftTreeSitter.Predicate.TextProvider? /// The internal tree-sitter layer tree object. - internal var state: TreeSitterState? - internal var textProvider: ResolvingQueryCursor.TextProvider + var state: TreeSitterState? + + /// The end point of the previous edit. + private var oldEndPoint: Point? // MARK: - Constants - internal enum Constants { + enum Constants { /// The maximum amount of limits a cursor can match during a query. /// Used to ensure performance in large files, even though we generally limit the query to the visible range. /// Neovim encountered this issue and uses 64 for their limit. Helix uses 256 due to issues with some @@ -57,8 +67,9 @@ public final class TreeSitterClient: HighlightProviding { /// And: https://github.com/helix-editor/helix/pull/4830 static let treeSitterMatchLimit = 256 - /// The timeout for parsers. - static let parserTimeout: TimeInterval = 0.005 + /// The timeout for parsers to re-check if a task is canceled. This constant represents the period between + /// checks. + static let parserTimeout: TimeInterval = 0.1 /// The maximum length of an edit before it must be processed asynchronously static let maxSyncEditLength: Int = 1024 @@ -68,20 +79,10 @@ public final class TreeSitterClient: HighlightProviding { /// The maximum length a query can be before it must be performed asynchronously. static let maxSyncQueryLength: Int = 4096 - - /// The maximum number of highlight queries that can be performed in parallel. - static let simultaneousHighlightLimit: Int = 5 } - // MARK: - Init/Config - - /// Initializes the `TreeSitterClient` with the given parameters. - /// - Parameters: - /// - textView: The text view to use as a data source. - /// - codeLanguage: The language to set up the parser with. - /// - textProvider: The text provider callback to read any text. - public init(textProvider: @escaping ResolvingQueryCursor.TextProvider) { - self.textProvider = textProvider + public enum Error: Swift.Error { + case syncUnavailable } // MARK: - HighlightProviding @@ -91,46 +92,142 @@ public final class TreeSitterClient: HighlightProviding { /// - textView: The text view to use as a data source. /// A weak reference will be kept for the lifetime of this object. /// - codeLanguage: The language to use for parsing. - public func setUp(textView: HighlighterTextView, codeLanguage: CodeLanguage) { - cancelAllRunningTasks() - queueLock.lock() - self.textView = textView + public func setUp(textView: TextView, codeLanguage: CodeLanguage) { + Self.logger.debug("TreeSitterClient setting up with language: \(codeLanguage.id.rawValue, privacy: .public)") + self.readBlock = textView.createReadBlock() - queuedEdits.append { - self.stateLock.lock() - self.state = TreeSitterState(codeLanguage: codeLanguage, textView: textView) - self.stateLock.unlock() + self.readCallback = textView.createReadCallback() + + self.setState( + language: codeLanguage, + readCallback: self.readCallback!, + readBlock: self.readBlock! + ) + } + + /// Sets the client's new state. + /// - Parameters: + /// - language: The language to use. + /// - readCallback: The callback to use to read text from the document. + /// - readBlock: The callback to use to read blocks of text from the document. + private func setState( + language: CodeLanguage, + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock + ) { + setUpCount += 1 + performAsync { [weak self] in + self?.state = TreeSitterState(codeLanguage: language, readCallback: readCallback, readBlock: readBlock) + } + } + + // MARK: - Async Operations + + /// Performs the given operation asynchronously. + /// + /// All completion handlers passed to this function will be enqueued on the `operationQueue` dispatch queue, + /// ensuring serial access to this class. + /// + /// This function will handle ensuring balanced increment/decrements are made to the `runningOperationCount` in + /// a safe manner. + /// + /// - Note: While in debug mode, this method will throw an assertion failure if not called from the Main thread. + /// - Parameter operation: The operation to perform + private func performAsync(_ operation: @escaping () -> Void) { + assertMain() + runningOperationCount += 1 + let setUpCountCopy = setUpCount + operationQueue.async { [weak self] in + guard self != nil && self?.setUpCount == setUpCountCopy else { return } + operation() + DispatchQueue.main.async { + self?.runningOperationCount -= 1 + } + } + } + + /// Attempts to perform a synchronous operation on the client. + /// + /// The operation will be dispatched synchronously to the `operationQueue`, this function will return once the + /// operation is finished. + /// + /// - Note: While in debug mode, this method will throw an assertion failure if not called from the Main thread. + /// - Parameter operation: The operation to perform synchronously. + /// - Throws: Can throw an ``TreeSitterClient/Error/syncUnavailable`` error if it's determined that an async + /// operation is unsafe. + private func performSync(_ operation: @escaping () -> Void) throws { + assertMain() + + guard runningOperationCount == 0 else { + throw Error.syncUnavailable + } + + runningOperationCount += 1 + + operationQueue.sync { + operation() + } + + self.runningOperationCount -= 1 + } + + /// Assert that the caller is calling from the main thread. + private func assertMain() { +#if DEBUG + if !Thread.isMainThread { + assertionFailure("TreeSitterClient used from non-main queue. This will cause race conditions.") } - beginTasksIfNeeded() - queueLock.unlock() +#endif } + // MARK: - HighlightProviding + /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. /// The returned `IndexSet` should include all indexes that need to be highlighted, including any inserted text. /// - Parameters: - /// - textView:The text view to use. + /// - textView: The text view to use. /// - range: The range of the edit. /// - delta: The length of the edit, can be negative for deletions. /// - completion: The function to call with an `IndexSet` containing all Indices to invalidate. - public func applyEdit( - textView: HighlighterTextView, - range: NSRange, - delta: Int, - completion: @escaping ((IndexSet) -> Void) - ) { - guard let edit = InputEdit(range: range, delta: delta, oldEndPoint: .zero) else { return } - - queueLock.lock() - let longEdit = range.length > Constants.maxSyncEditLength - let longDocument = textView.documentRange.length > Constants.maxSyncContentLength + public func applyEdit(textView: TextView, range: NSRange, delta: Int, completion: @escaping (IndexSet) -> Void) { + let oldEndPoint: Point - if hasOutstandingWork || longEdit || longDocument { - applyEditAsync(editState: EditState(edit: edit, completion: completion), startAtLayerIndex: 0) - queueLock.unlock() + if self.oldEndPoint != nil { + oldEndPoint = self.oldEndPoint! } else { - queueLock.unlock() - applyEdit(editState: EditState(edit: edit, completion: completion)) + oldEndPoint = textView.pointForLocation(range.max) ?? .zero } + + guard let edit = InputEdit( + range: range, + delta: delta, + oldEndPoint: oldEndPoint, + textView: textView + ) else { + completion(IndexSet()) + return + } + + let operation = { [weak self] in + let invalidatedRanges = self?.applyEdit(edit: edit) ?? IndexSet() + completion(invalidatedRanges) + } + + do { + let longEdit = range.length > Constants.maxSyncEditLength + let longDocument = textView.documentRange.length > Constants.maxSyncContentLength + + if longEdit || longDocument { + + } + try performSync(operation) + } catch { + performAsync(operation) + } + } + + public func willApplyEdit(textView: TextView, range: NSRange) { + oldEndPoint = textView.pointForLocation(range.max) } /// Initiates a highlight query. @@ -139,96 +236,27 @@ public final class TreeSitterClient: HighlightProviding { /// - range: The range to limit the highlights to. /// - completion: Called when the query completes. public func queryHighlightsFor( - textView: HighlighterTextView, + textView: TextView, range: NSRange, - completion: @escaping (([HighlightRange]) -> Void) + completion: @escaping ([HighlightRange]) -> Void ) { - queueLock.lock() - let longQuery = range.length > Constants.maxSyncQueryLength - let longDocument = textView.documentRange.length > Constants.maxSyncContentLength - - if hasOutstandingWork || longQuery || longDocument { - queryHighlightsForRangeAsync(range: range, completion: completion) - queueLock.unlock() - } else { - queueLock.unlock() - queryHighlightsForRange(range: range, runningAsync: false, completion: completion) - } - } - - // MARK: - Async - - /// Use to determine if there are any queued or running async tasks. - var hasOutstandingWork: Bool { - runningTask != nil || queuedEdits.count > 0 || queuedQueries.count > 0 - } - - private enum QueuedTaskType { - case edit(job: AsyncCallback) - case highlight(jobs: [AsyncCallback]) - } - - /// Spawn the running task if one is needed and doesn't already exist. - /// - /// The task will run until `determineNextTask` returns nil. It will run any highlight jobs in parallel. - internal func beginTasksIfNeeded() { - guard runningTask == nil && (queuedEdits.count > 0 || queuedQueries.count > 0) else { return } - runningTask = Task.detached(priority: .userInitiated) { - defer { - self.runningTask = nil + let operation = { [weak self] in + let highlights = self?.queryHighlightsForRange(range: range) + DispatchQueue.main.async { + completion(highlights ?? []) } - - do { - while let nextQueuedJob = self.determineNextJob() { - try Task.checkCancellation() - switch nextQueuedJob { - case .edit(let job): - job() - case .highlight(let jobs): - await withTaskGroup(of: Void.self, body: { taskGroup in - for job in jobs { - taskGroup.addTask { - job() - } - } - }) - } - } - } catch { } } - } - /// Determines the next async job to run and returns it if it exists. - /// Greedily returns queued highlight jobs determined by `Constants.simultaneousHighlightLimit` - private func determineNextJob() -> QueuedTaskType? { - queueLock.lock() - defer { - queueLock.unlock() - } + do { + let longQuery = range.length > Constants.maxSyncQueryLength + let longDocument = textView.documentRange.length > Constants.maxSyncContentLength - // Get an edit task if any, otherwise get a highlight task if any. - if queuedEdits.count > 0 { - return .edit(job: queuedEdits.removeFirst()) - } else if queuedQueries.count > 0 { - let jobCount = min(queuedQueries.count, Constants.simultaneousHighlightLimit) - let jobs = Array(queuedQueries[0..(arrayLiteral: layers[0]) - var touchedLayers = Set() - - var idx = 0 - while idx < layers.count { - updateInjectedLanguageLayer( - textView: textView, - layer: layers[idx], - layerSet: &layerSet, - touchedLayers: &touchedLayers - ) - - idx += 1 - } + self.parseDocument(readCallback: readCallback, readBlock: readBlock) } - /// Private initilizer used by `copy` + /// Private initializer used by `copy` private init(codeLanguage: CodeLanguage, layers: [LanguageLayer]) { self.primaryLayer = codeLanguage self.layers = layers @@ -53,7 +39,7 @@ public class TreeSitterState { /// Sets the language for the state. Removing all existing layers. /// - Parameter codeLanguage: The language to use. - public func setLanguage(_ codeLanguage: CodeLanguage) { + private func setLanguage(_ codeLanguage: CodeLanguage) { layers.removeAll() primaryLayer = codeLanguage @@ -72,10 +58,32 @@ public class TreeSitterState { try? layers[0].parser.setLanguage(treeSitterLanguage) } - /// Creates a copy of this state object. - /// - Returns: The copied object - public func copy() -> TreeSitterState { - return TreeSitterState(codeLanguage: primaryLayer, layers: layers.map { $0.copy() }) + /// Performs the initial document parse for the primary layer. + /// - Parameters: + /// - readCallback: The callback to use to read content from the document. + /// - readBlock: The callback to use to read blocks of content from the document. + private func parseDocument( + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock + ) { + layers[0].parser.timeout = 0.0 + layers[0].tree = layers[0].parser.parse(tree: nil as Tree?, readBlock: readBlock) + + var layerSet = Set(arrayLiteral: layers[0]) + var touchedLayers = Set() + + var idx = 0 + while idx < layers.count { + updateInjectedLanguageLayer( + readCallback: readCallback, + readBlock: readBlock, + layer: layers[idx], + layerSet: &layerSet, + touchedLayers: &touchedLayers + ) + + idx += 1 + } } // MARK: - Layer Management @@ -89,7 +97,7 @@ public class TreeSitterState { /// Removes all languagel ayers in the given set. /// - Parameter set: A set of all language layers to remove. public func removeLanguageLayers(in set: Set) { - layers.removeAll(where: { set.contains($0 )}) + layers.removeAll(where: { set.contains($0) }) } /// Attempts to create a language layer and load a highlights file. @@ -130,13 +138,15 @@ public class TreeSitterState { /// Inserts any new language layers, and removes any that may have been deleted after an edit. /// - Parameters: - /// - textView: The data source for text ranges. + /// - readCallback: Callback used to read text for a specific range. + /// - readBlock: Callback used to read blocks of text. /// - touchedLayers: A set of layers. Each time a layer is visited, it will be removed from this set. /// Use this to determine if any layers were not modified after this method was run. /// Those layers should be removed. /// - Returns: A set of indices of any new layers. This set indicates ranges that should be re-highlighted. public func updateInjectedLayers( - textView: HighlighterTextView, + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock, touchedLayers: Set ) -> IndexSet { var layerSet = Set(layers) @@ -152,7 +162,8 @@ public class TreeSitterState { if layer.supportsInjections { rangeSet.formUnion( updateInjectedLanguageLayer( - textView: textView, + readCallback: readCallback, + readBlock: readBlock, layer: layer, layerSet: &layerSet, touchedLayers: &touchedLayers @@ -172,7 +183,8 @@ public class TreeSitterState { /// Performs an injections query on the given language layer. /// Updates any existing layers with new ranges and adds new layers if needed. /// - Parameters: - /// - textView: The text view to use. + /// - readCallback: Callback used to read text for a specific range. + /// - readBlock: Callback used to read blocks of text. /// - layer: The language layer to perform the query on. /// - layerSet: The set of layers that exist in the document. /// Used for efficient lookup of existing `(language, range)` pairs @@ -181,7 +193,8 @@ public class TreeSitterState { /// - Returns: An index set of any updated indexes. @discardableResult private func updateInjectedLanguageLayer( - textView: HighlighterTextView, + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock, layer: LanguageLayer, layerSet: inout Set, touchedLayers: inout Set @@ -194,8 +207,8 @@ public class TreeSitterState { cursor.matchLimit = TreeSitterClient.Constants.treeSitterMatchLimit - let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in - return textView.stringForRange(range) + let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, point in + return readCallback(range, point) } var updatedRanges = IndexSet() @@ -222,12 +235,11 @@ public class TreeSitterState { // If we've found this layer, it means it should exist after an edit. touchedLayers.remove(layer) } else { - let readBlock = textView.createReadBlock() // New range, make a new layer! if let addedLayer = addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) { addedLayer.ranges = [range.range] addedLayer.parser.includedRanges = addedLayer.ranges.map { $0.tsRange } - addedLayer.tree = addedLayer.parser.createTree(readBlock: readBlock) + addedLayer.tree = addedLayer.parser.parse(tree: nil as Tree?, readBlock: readBlock) layerSet.insert(addedLayer) updatedRanges.insert(range: range.range) @@ -247,7 +259,7 @@ public class TreeSitterState { /// - Returns: A map of each language to all the ranges they have been injected into. private func injectedLanguagesFrom( cursor: QueryCursor, - textProvider: @escaping ResolvingQueryCursor.TextProvider + textProvider: @escaping SwiftTreeSitter.Predicate.TextProvider ) -> [String: [NamedRange]] { var languages: [String: [NamedRange]] = [:] diff --git a/Tests/CodeEditSourceEditorTests/TextViewControllerTests.swift b/Tests/CodeEditSourceEditorTests/TextViewControllerTests.swift index 9c84bdb40..fc5e340fa 100644 --- a/Tests/CodeEditSourceEditorTests/TextViewControllerTests.swift +++ b/Tests/CodeEditSourceEditorTests/TextViewControllerTests.swift @@ -237,7 +237,7 @@ final class TextViewControllerTests: XCTestCase { controller.scrollView.setFrameSize(NSSize(width: 500, height: 500)) controller.viewDidLoad() controller.bracketPairHighlight = nil - controller.textView.string = "{ Loren Ipsum {} }" + controller.setText("{ Loren Ipsum {} }") controller.setCursorPositions([CursorPosition(line: 1, column: 2)]) // After first opening { XCTAssert(controller.highlightLayers.isEmpty, "Controller added highlight layer when setting is set to `nil`") controller.setCursorPositions([CursorPosition(line: 1, column: 3)]) diff --git a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift index 13467e8a0..425dbc81d 100644 --- a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift +++ b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift @@ -1,109 +1,46 @@ import XCTest +import CodeEditTextView @testable import CodeEditSourceEditor // swiftlint:disable all -fileprivate class TestTextView: HighlighterTextView { - var testString: NSMutableString = "func testSwiftFunc() -> Int {\n\tprint(\"\")\n}" - - var documentRange: NSRange { - NSRange(location: 0, length: testString.length) - } - - func stringForRange(_ nsRange: NSRange) -> String? { - testString.substring(with: nsRange) - } -} final class TreeSitterClientTests: XCTestCase { - fileprivate var textView = TestTextView() + class Delegate: TextViewDelegate { } + + fileprivate var textView = TextView( + string: "func testSwiftFunc() -> Int {\n\tprint(\"\")\n}", + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + textColor: .labelColor, + lineHeightMultiplier: 1.0, + wrapLines: true, + isEditable: true, + isSelectable: true, + letterSpacing: 1.0, + delegate: Delegate() + ) var client: TreeSitterClient! override func setUp() { - client = TreeSitterClient { nsRange, _ in - self.textView.stringForRange(nsRange) - } + client = TreeSitterClient() } func test_clientSetup() { client.setUp(textView: textView, codeLanguage: .swift) - XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") - - let editExpectation = expectation(description: "Edit work should never return") - editExpectation.isInverted = true // Expect to never happen - - textView.testString.insert("let int = 0\n", at: 0) - client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in - editExpectation.fulfill() - } - - client.setUp(textView: textView, codeLanguage: .swift) - XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") - XCTAssert(client.queuedEdits.count == 1, "Client should cancel all queued work when setUp is called.") - - waitForExpectations(timeout: 1.0, handler: nil) - } - - // Test async language loading with edits and highlights queued before loading completes. - func test_languageLoad() { - textView = TestTextView() - client.setUp(textView: textView, codeLanguage: .swift) - - XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") - - let editExpectation = expectation(description: "Edit work should return first.") - let highlightExpectation = expectation(description: "Highlight should return last.") - - client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in - highlightExpectation.fulfill() - } - - textView.testString.insert("let int = 0\n", at: 0) - client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in - editExpectation.fulfill() - } - - wait(for: [editExpectation, highlightExpectation], timeout: 10.0, enforceOrder: true) - } - - // Edits should be consumed before highlights. - func test_queueOrder() { - textView = TestTextView() - client.setUp(textView: textView, codeLanguage: .swift) - - let editExpectation = expectation(description: "Edit work should return first.") - let editExpectation2 = expectation(description: "Edit2 should return 2nd.") - let highlightExpectation = expectation(description: "Highlight should return 3rd.") - - // Do initial query while language loads. - client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in - print("highlightExpectation") - highlightExpectation.fulfill() - } - // Queue another edit - textView.testString.insert("let int = 0\n", at: 0) - client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in - print("editExpectation") - editExpectation.fulfill() + let now = Date() + while client.state == nil && abs(now.timeIntervalSinceNow) < 5 { + usleep(1000) } - // One more edit - textView.testString.insert("let int = 0\n", at: 0) - client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in - print("editExpectation2") - editExpectation2.fulfill() + if abs(now.timeIntervalSinceNow) >= 5 { + XCTFail("Client took more than 5 seconds to set up.") } - wait( - for: [ - editExpectation, - editExpectation2, - highlightExpectation, - ], - timeout: 10.0, - enforceOrder: true - ) + let primaryLanguage = client.state?.primaryLayer.id + let layerCount = client.state?.layers.count + XCTAssertEqual(primaryLanguage, .swift, "Client set up incorrect language") + XCTAssertEqual(layerCount, 1, "Client set up too many layers") } } // swiftlint:enable all