From 61f61ac7e470bd9ce764a2ec038b58932bdd918a Mon Sep 17 00:00:00 2001 From: pcr Date: Wed, 5 Oct 2022 10:55:26 +0200 Subject: [PATCH 1/5] Initial implementation of a semantic tokens reconciler Initial implementation of a semantic tokens reconciler which supports only SemanticTokensFull. The reconciler uses the theme defined for TM4E so that the same styles are used for the same token types. In addition to the tokens defined for TM4E, also the deprecated modifier is supported by marking striking out the regions. --- org.eclipse.lsp4e/META-INF/MANIFEST.MF | 1 + org.eclipse.lsp4e/plugin.xml | 17 + .../SemanticHighlightReconciler.java | 36 ++ .../SemanticHighlightReconcilerStrategy.java | 376 ++++++++++++++++++ .../target-platform-latest.target | 4 + 5 files changed, 434 insertions(+) create mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java create mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java diff --git a/org.eclipse.lsp4e/META-INF/MANIFEST.MF b/org.eclipse.lsp4e/META-INF/MANIFEST.MF index 158dd9d53..32f3638bf 100644 --- a/org.eclipse.lsp4e/META-INF/MANIFEST.MF +++ b/org.eclipse.lsp4e/META-INF/MANIFEST.MF @@ -25,6 +25,7 @@ Require-Bundle: org.eclipse.core.runtime;bundle-version="3.12.0", org.eclipse.debug.ui;bundle-version="3.11.200", org.eclipse.swt, org.eclipse.jdt.annotation;bundle-version="2.1.0";resolution:=optional, + org.eclipse.tm4e.ui, org.eclipse.ui.editors, org.eclipse.ui.navigator;bundle-version="3.6.100", org.eclipse.lsp4j;bundle-version="[0.19.0,0.20.0)", diff --git a/org.eclipse.lsp4e/plugin.xml b/org.eclipse.lsp4e/plugin.xml index fad248dce..67fdadd1c 100644 --- a/org.eclipse.lsp4e/plugin.xml +++ b/org.eclipse.lsp4e/plugin.xml @@ -657,4 +657,21 @@ + + + + + + + + + + + + + + diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java new file mode 100644 index 000000000..076344ec4 --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Evolution AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import org.eclipse.jface.text.IDocument; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.jface.text.reconciler.MonoReconciler; + + +public class SemanticHighlightReconciler extends MonoReconciler { + + public SemanticHighlightReconciler() { + super(new SemanticHighlightReconcilerStrategy(), false); + } + + @Override + public void install(final ITextViewer textViewer) { + super.install(textViewer); + // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted + ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).install(textViewer); + } + + @Override + public void uninstall() { + super.uninstall(); + // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted + ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).uninstall(); + } + +} \ No newline at end of file diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java new file mode 100644 index 000000000..fd1d4e730 --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java @@ -0,0 +1,376 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Evolution AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import org.eclipse.core.resources.IFile; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.jface.text.BadLocationException; +import org.eclipse.jface.text.IDocument; +import org.eclipse.jface.text.IRegion; +import org.eclipse.jface.text.ITextPresentationListener; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.jface.text.TextAttribute; +import org.eclipse.jface.text.TextPresentation; +import org.eclipse.jface.text.TextViewer; +import org.eclipse.jface.text.reconciler.DirtyRegion; +import org.eclipse.jface.text.reconciler.IReconcilingStrategy; +import org.eclipse.jface.text.reconciler.IReconcilingStrategyExtension; +import org.eclipse.jface.text.rules.IToken; +import org.eclipse.lsp4e.LSPEclipseUtils; +import org.eclipse.lsp4e.LanguageServerPlugin; +import org.eclipse.lsp4e.LanguageServerWrapper; +import org.eclipse.lsp4e.LanguageServersRegistry.LanguageServerDefinition; +import org.eclipse.lsp4e.LanguageServiceAccessor; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.SemanticTokenModifiers; +import org.eclipse.lsp4j.SemanticTokens; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.lsp4j.SemanticTokensParams; +import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; +import org.eclipse.lsp4j.ServerCapabilities; +import org.eclipse.lsp4j.TextDocumentIdentifier; +import org.eclipse.lsp4j.services.LanguageServer; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.StyleRange; +import org.eclipse.tm4e.ui.TMUIPlugin; +import org.eclipse.tm4e.ui.themes.ITheme; + +/** + * A reconciler strategy using semantic highlighting as defined by LSP. + */ +public class SemanticHighlightReconcilerStrategy + implements IReconcilingStrategy, IReconcilingStrategyExtension, ITextPresentationListener { + + private @Nullable ITextViewer viewer; + + private @Nullable ITheme theme; + + private IDocument document; + + private Map semanticTokensLegendMap; + + private List previousRanges; + + /** + * Installs the reconciler on the given text viewer. After this method has been + * finished, the reconciler is operational, i.e., it works without requesting + * further client actions until uninstall is called. + * + * @param textViewer + * the viewer on which the reconciler is installed + */ + public void install(final ITextViewer textViewer) { + viewer = textViewer; + theme = TMUIPlugin.getThemeManager().getDefaultTheme(); + if (textViewer instanceof TextViewer viewer) { + viewer.addTextPresentationListener(this); + } + previousRanges = new ArrayList<>(); + } + + /** + * Removes the reconciler from the text viewer it has previously been installed + * on. + */ + public void uninstall() { + theme = null; + ITextViewer textViewer = viewer; + if (textViewer instanceof TextViewer viewer) { + viewer.removeTextPresentationListener(this); + } + viewer = null; + previousRanges = null; + semanticTokensLegendMap = null; + + } + + private void initSemanticTokensLegendMap() { + IFile file = LSPEclipseUtils.getFile(document); + if (file != null) { + try { + semanticTokensLegendMap = new HashMap<>(); + for (LanguageServerWrapper wrapper: LanguageServiceAccessor.getLSWrappers(file, x -> true)) { + ServerCapabilities serverCapabilities = wrapper.getServerCapabilities(); + if (serverCapabilities != null) { + SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities.getSemanticTokensProvider(); + if (semanticTokensProvider != null) { + semanticTokensLegendMap.put(wrapper.serverDefinition.id, semanticTokensProvider.getLegend()); + } + } + } + } catch (IOException e) { + LanguageServerPlugin.logError(e); + } + } + } + + private SemanticTokensParams getSemanticTokensParams() { + URI uri = LSPEclipseUtils.toUri(document); + if (uri != null) { + SemanticTokensParams semanticTokensParams = new SemanticTokensParams(); + semanticTokensParams.setTextDocument(new TextDocumentIdentifier(uri.toString())); + return semanticTokensParams; + } + return null; + } + + private void saveStyle(final SemanticTokens semanticTokens, final SemanticTokensLegend semanticTokensLegend) { + if (semanticTokens == null || semanticTokensLegend == null) { + return; + } + List dataStream = semanticTokens.getData(); + if (!dataStream.isEmpty()) { + try { + List styleRanges = getStyleRanges(dataStream, semanticTokensLegend); + saveStyles(styleRanges); + } catch (BadLocationException e) { + LanguageServerPlugin.logError(e); + } + } + } + + private StyleRange clone(final StyleRange styleRange) { + StyleRange clonedStyleRange = new StyleRange(styleRange.start, styleRange.length, styleRange.foreground, + styleRange.background, styleRange.fontStyle); + clonedStyleRange.strikeout = styleRange.strikeout; + return clonedStyleRange; + } + + private void mergeStyles(final TextPresentation textPresentation, final List styleRanges) { + StyleRange[] array = new StyleRange[styleRanges.size()]; + array = styleRanges.toArray(array); + textPresentation.replaceStyleRanges(array); + } + + private boolean overlaps(final StyleRange range, final IRegion region) { + return isContained(range.start, region) || isContained(range.start + range.length, region) + || isContained(region.getOffset(), range); + } + + private boolean isContained(final int offset, final StyleRange range) { + return offset >= range.start && offset < (range.start + range.length); + } + + private boolean isContained(final int offset, final IRegion region) { + return offset >= region.getOffset() && offset < (region.getOffset() + region.getLength()); + } + + private void saveStyles(final List styleRanges) { + synchronized (previousRanges) { + previousRanges.clear(); + previousRanges.addAll(styleRanges); + previousRanges.sort(Comparator.comparing(s -> s.start)); + } + } + + private List getStyleRanges(final List dataStream, + final SemanticTokensLegend semanticTokensLegend) throws BadLocationException { + List styleRanges = new ArrayList<>(dataStream.size() / 5); + + int idx = 0; + int prevLine = 0; + int line = 0; + int offset = 0; + int length = 0; + String tokenType = null; + for (Integer data : dataStream) { + switch (idx % 5) { + case 0: // line + line += data; + break; + case 1: // offset + if (line == prevLine) { + offset += data; + } else { + offset = LSPEclipseUtils.toOffset(new Position(line, data), document); + } + break; + case 2: // length + length = data; + break; + case 3: // token type + tokenType = tokenType(data, semanticTokensLegend.getTokenTypes()); + break; + case 4: // token modifier + prevLine = line; + List tokenModifiers = tokenModifiers(data, semanticTokensLegend.getTokenModifiers()); + StyleRange styleRange = getStyleRange(offset, length, textAttribute(tokenType)); + if (tokenModifiers.stream().anyMatch(x -> x.equals(SemanticTokenModifiers.Deprecated))) { + styleRange.strikeout = true; + } + styleRanges.add(styleRange); + break; + } + idx++; + } + return styleRanges; + } + + private String tokenType(final Integer data, final List legend) { + try { + return legend.get(data - 1); + } catch (IndexOutOfBoundsException e) { + return null; // no match + } + } + + private List tokenModifiers(final Integer data, final List legend) { + if (data.intValue() == 0) { + return Collections.emptyList(); + } + BitSet bitSet = BitSet.valueOf(new long[] { data }); + List tokenModifiers = new ArrayList<>(); + for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { + try { + tokenModifiers.add(legend.get(i)); + } catch (IndexOutOfBoundsException e) { + // no match + } + } + + return tokenModifiers; + } + + private TextAttribute textAttribute(final String tokenType) { + ITheme localTheme = theme; + if (localTheme != null && tokenType != null) { + IToken token = localTheme.getToken(tokenType); + if (token != null) { + Object data = token.getData(); + if (data instanceof TextAttribute textAttribute) { + return textAttribute; + } + } + } + return null; + } + + /** + * Gets a style range for the given inputs. + * + * @param offset + * the offset of the range to be styled + * @param length + * the length of the range to be styled + * @param attr + * the attribute describing the style of the range to be styled + */ + private StyleRange getStyleRange(final int offset, final int length, final TextAttribute attr) { + final StyleRange styleRange; + if (attr != null) { + final int style = attr.getStyle(); + final int fontStyle = style & (SWT.ITALIC | SWT.BOLD | SWT.NORMAL); + styleRange = new StyleRange(offset, length, attr.getForeground(), attr.getBackground(), fontStyle); + styleRange.strikeout = (style & TextAttribute.STRIKETHROUGH) != 0; + styleRange.underline = (style & TextAttribute.UNDERLINE) != 0; + styleRange.font = attr.getFont(); + return styleRange; + } else { + styleRange = new StyleRange(); + styleRange.start = offset; + styleRange.length = length; + } + return styleRange; + } + + @Override + public void setProgressMonitor(final IProgressMonitor monitor) { + } + + @Override + public void setDocument(final IDocument document) { + this.document = document; + initSemanticTokensLegendMap(); + } + + private SemanticTokensLegend getSemanticTokensLegend(final LanguageServer languageSever) { + Optional serverDefinition = LanguageServiceAccessor + .resolveServerDefinition(languageSever); + if (serverDefinition.isPresent()) { + return semanticTokensLegendMap.get(serverDefinition.get().id); + } + return null; + } + + private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilities) { + return serverCapabilities.getSemanticTokensProvider() != null + && serverCapabilities.getSemanticTokensProvider().getFull().getLeft(); + } + + private CompletableFuture semanticTokensFull(final List languageServers) { + return CompletableFuture + .allOf(languageServers.stream().map(this::semanticTokensFull).toArray(CompletableFuture[]::new)); + } + + private CompletableFuture semanticTokensFull(final LanguageServer languageServer) { + SemanticTokensParams semanticTokensParams = getSemanticTokensParams(); + return languageServer.getTextDocumentService().semanticTokensFull(semanticTokensParams) + .thenAccept(semanticTokens -> { + saveStyle(semanticTokens, getSemanticTokensLegend(languageServer)); + }).exceptionally(e -> { + LanguageServerPlugin.logError(e); + return null; + }); + } + + private void fullReconcile() { + try { + LanguageServiceAccessor.getLanguageServers(document, this::hasSemanticTokensFull)// + .thenAccept(this::semanticTokensFull).get(); + } catch (InterruptedException | ExecutionException e) { + LanguageServerPlugin.logError(e); + } + } + + @Override + public void initialReconcile() { + fullReconcile(); + } + + @Override + public void reconcile(final DirtyRegion dirtyRegion, final IRegion subRegion) { + fullReconcile(); + } + + @Override + public void reconcile(final IRegion partition) { + fullReconcile(); + } + + private List appliedRanges(final TextPresentation textPresentation) { + synchronized (previousRanges) { + // we need to create new styles because the text presentation might change a + // style when applied to the presentation + // and we want the ones saved from the reconciling as immutable + return previousRanges.stream()// + .filter(r -> overlaps(r, textPresentation.getExtent()))// + .map(this::clone).collect(Collectors.toList()); + } + } + + @Override + public void applyTextPresentation(final TextPresentation textPresentation) { + mergeStyles(textPresentation, appliedRanges(textPresentation)); + } +} \ No newline at end of file diff --git a/target-platforms/target-platform-latest/target-platform-latest.target b/target-platforms/target-platform-latest/target-platform-latest.target index 1cae84f65..703b30c58 100644 --- a/target-platforms/target-platform-latest/target-platform-latest.target +++ b/target-platforms/target-platform-latest/target-platform-latest.target @@ -26,6 +26,10 @@ + + + + \ No newline at end of file From 4c64bbc6f6cbf4f3dff8d6da30a1616982027b30 Mon Sep 17 00:00:00 2001 From: rubenporras Date: Tue, 13 Dec 2022 08:54:13 +0100 Subject: [PATCH 2/5] Split the SemanticHighlightReconciler class into smaller classes which can be unit tested. --- .../SemanticHighlightReconciler.java | 36 +- .../SemanticHighlightReconcilerStrategy.java | 361 ++++++------------ .../SemanticTokensDataStreamProcessor.java | 163 ++++++++ .../SemanticTokensLegendProvider.java | 89 +++++ .../semanticTokens/StyleRangeHolder.java | 103 +++++ .../semanticTokens/TokenTypeMapper.java | 44 +++ 6 files changed, 545 insertions(+), 251 deletions(-) create mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java create mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java create mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java create mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java index 076344ec4..0a542d6c8 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2022 Avaloq Evolution AG. + * Copyright (c) 2022 Avaloq Group AG. * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ @@ -12,25 +12,27 @@ import org.eclipse.jface.text.ITextViewer; import org.eclipse.jface.text.reconciler.MonoReconciler; - public class SemanticHighlightReconciler extends MonoReconciler { - public SemanticHighlightReconciler() { - super(new SemanticHighlightReconcilerStrategy(), false); - } + public SemanticHighlightReconciler() { + super(new SemanticHighlightReconcilerStrategy(), false); + } - @Override - public void install(final ITextViewer textViewer) { - super.install(textViewer); - // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted - ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).install(textViewer); - } + @Override + public void install(final ITextViewer textViewer) { + super.install(textViewer); + // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is + // accepted + ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)) + .install(textViewer); + } - @Override - public void uninstall() { - super.uninstall(); - // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted - ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).uninstall(); - } + @Override + public void uninstall() { + super.uninstall(); + // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is + // accepted + ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).uninstall(); + } } \ No newline at end of file diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java index fd1d4e730..46331ae96 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java @@ -8,69 +8,95 @@ *******************************************************************************/ package org.eclipse.lsp4e.operations.semanticTokens; -import java.io.IOException; import java.net.URI; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; +import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; +import java.util.function.Function; -import org.eclipse.core.resources.IFile; import org.eclipse.core.runtime.IProgressMonitor; -import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jface.text.BadLocationException; import org.eclipse.jface.text.IDocument; import org.eclipse.jface.text.IRegion; +import org.eclipse.jface.text.ITextListener; import org.eclipse.jface.text.ITextPresentationListener; import org.eclipse.jface.text.ITextViewer; -import org.eclipse.jface.text.TextAttribute; import org.eclipse.jface.text.TextPresentation; import org.eclipse.jface.text.TextViewer; import org.eclipse.jface.text.reconciler.DirtyRegion; import org.eclipse.jface.text.reconciler.IReconcilingStrategy; import org.eclipse.jface.text.reconciler.IReconcilingStrategyExtension; -import org.eclipse.jface.text.rules.IToken; import org.eclipse.lsp4e.LSPEclipseUtils; import org.eclipse.lsp4e.LanguageServerPlugin; -import org.eclipse.lsp4e.LanguageServerWrapper; -import org.eclipse.lsp4e.LanguageServersRegistry.LanguageServerDefinition; import org.eclipse.lsp4e.LanguageServiceAccessor; +import org.eclipse.lsp4e.LanguageServiceAccessor.LSPDocumentInfo; import org.eclipse.lsp4j.Position; -import org.eclipse.lsp4j.SemanticTokenModifiers; import org.eclipse.lsp4j.SemanticTokens; import org.eclipse.lsp4j.SemanticTokensLegend; import org.eclipse.lsp4j.SemanticTokensParams; -import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; import org.eclipse.lsp4j.ServerCapabilities; import org.eclipse.lsp4j.TextDocumentIdentifier; +import org.eclipse.lsp4j.jsonrpc.ResponseErrorException; +import org.eclipse.lsp4j.jsonrpc.messages.ResponseError; +import org.eclipse.lsp4j.jsonrpc.messages.ResponseErrorCode; import org.eclipse.lsp4j.services.LanguageServer; -import org.eclipse.swt.SWT; import org.eclipse.swt.custom.StyleRange; -import org.eclipse.tm4e.ui.TMUIPlugin; -import org.eclipse.tm4e.ui.themes.ITheme; +import org.eclipse.swt.custom.StyledText; /** - * A reconciler strategy using semantic highlighting as defined by LSP. + * A semantic reconciler strategy using LSP. + *

+ * This strategy applies semantic highlight on top of the syntactic highlight + * provided by TM4E by implementing {@link ITextPresentationListener}. Because + * semantic highlight involves remote calls, it is expected to be slower than + * syntactic highlight. Thus we execute semantic highlight processing in a + * separate reconciler thread that will never block TM4E. + *

+ * This strategy records the version of the document when the semantic highlight + * is sent to the LS and when TM4E highlight is applied. If the response for a + * particular document version comes after the TM4E highlight has been applied, + * the text presentation is invalidated so that highlight is extended with the + * results. + *

+ * To avoid flickering, {@link StyleRangeHolder} implement {@link ITextListener} + * to adapt recorded semantic highlights and apply those instead of nothing + * where needed. + *

+ * If the response comes before, the data is saved and applied later on top of + * the presentation provided by TM4E. The results from our reconciler are + * recorded + *

+ * For simplicity, out-dated responses are discarded, as we know we shall get + * newer ones. */ public class SemanticHighlightReconcilerStrategy implements IReconcilingStrategy, IReconcilingStrategyExtension, ITextPresentationListener { - private @Nullable ITextViewer viewer; - - private @Nullable ITheme theme; + private ITextViewer viewer; private IDocument document; - private Map semanticTokensLegendMap; + private StyleRangeHolder styleRangeHolder; + + private SemanticTokensDataStreamProcessor semanticTokensDataStreamProcessor; + + private SemanticTokensLegendProvider semanticTokensLegendProvider; + + /** + * Written in {@link this.class#applyTextPresentation(TextPresentation)} + * applyTextPresentation and read in the lambda in + * {@link this.class#semanticTokensFull(LanguageServer, int)}, the lambda and + * the former method are executed in the display thread, thus serializing access + * using volatile without using explicit synchronized blocks is enough to avoid + * that org.eclipse.jface.text.ITextViewer.invalidateTextPresentation() is + * called by use while the presentation is being updated. + */ + private volatile int documentVersionAtLastAppliedTextPresentation; - private List previousRanges; + private CompletableFuture semanticTokensFullFuture; /** * Installs the reconciler on the given text viewer. After this method has been @@ -82,11 +108,15 @@ public class SemanticHighlightReconcilerStrategy */ public void install(final ITextViewer textViewer) { viewer = textViewer; - theme = TMUIPlugin.getThemeManager().getDefaultTheme(); - if (textViewer instanceof TextViewer viewer) { - viewer.addTextPresentationListener(this); + styleRangeHolder = new StyleRangeHolder(); + semanticTokensDataStreamProcessor = new SemanticTokensDataStreamProcessor(new TokenTypeMapper(viewer), + offsetMapper()); + semanticTokensLegendProvider = new SemanticTokensLegendProvider(); + + if (viewer instanceof TextViewer) { + ((TextViewer) viewer).addTextPresentationListener(this); } - previousRanges = new ArrayList<>(); + viewer.addTextListener(styleRangeHolder); } /** @@ -94,35 +124,24 @@ public void install(final ITextViewer textViewer) { * on. */ public void uninstall() { - theme = null; - ITextViewer textViewer = viewer; - if (textViewer instanceof TextViewer viewer) { - viewer.removeTextPresentationListener(this); + semanticTokensDataStreamProcessor = null; + if (viewer instanceof TextViewer) { + ((TextViewer) viewer).removeTextPresentationListener(this); } + viewer.removeTextListener(styleRangeHolder); viewer = null; - previousRanges = null; - semanticTokensLegendMap = null; - + styleRangeHolder = null; + semanticTokensLegendProvider = null; } - private void initSemanticTokensLegendMap() { - IFile file = LSPEclipseUtils.getFile(document); - if (file != null) { + private Function offsetMapper() { + return (p) -> { try { - semanticTokensLegendMap = new HashMap<>(); - for (LanguageServerWrapper wrapper: LanguageServiceAccessor.getLSWrappers(file, x -> true)) { - ServerCapabilities serverCapabilities = wrapper.getServerCapabilities(); - if (serverCapabilities != null) { - SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities.getSemanticTokensProvider(); - if (semanticTokensProvider != null) { - semanticTokensLegendMap.put(wrapper.serverDefinition.id, semanticTokensProvider.getLegend()); - } - } - } - } catch (IOException e) { - LanguageServerPlugin.logError(e); + return LSPEclipseUtils.toOffset(p, document); + } catch (BadLocationException e) { + throw new RuntimeException(e); } - } + }; } private SemanticTokensParams getSemanticTokensParams() { @@ -141,159 +160,12 @@ private void saveStyle(final SemanticTokens semanticTokens, final SemanticTokens } List dataStream = semanticTokens.getData(); if (!dataStream.isEmpty()) { - try { - List styleRanges = getStyleRanges(dataStream, semanticTokensLegend); - saveStyles(styleRanges); - } catch (BadLocationException e) { - LanguageServerPlugin.logError(e); - } + List styleRanges = semanticTokensDataStreamProcessor.getStyleRanges(dataStream, + semanticTokensLegend); + styleRangeHolder.saveStyles(styleRanges); } } - private StyleRange clone(final StyleRange styleRange) { - StyleRange clonedStyleRange = new StyleRange(styleRange.start, styleRange.length, styleRange.foreground, - styleRange.background, styleRange.fontStyle); - clonedStyleRange.strikeout = styleRange.strikeout; - return clonedStyleRange; - } - - private void mergeStyles(final TextPresentation textPresentation, final List styleRanges) { - StyleRange[] array = new StyleRange[styleRanges.size()]; - array = styleRanges.toArray(array); - textPresentation.replaceStyleRanges(array); - } - - private boolean overlaps(final StyleRange range, final IRegion region) { - return isContained(range.start, region) || isContained(range.start + range.length, region) - || isContained(region.getOffset(), range); - } - - private boolean isContained(final int offset, final StyleRange range) { - return offset >= range.start && offset < (range.start + range.length); - } - - private boolean isContained(final int offset, final IRegion region) { - return offset >= region.getOffset() && offset < (region.getOffset() + region.getLength()); - } - - private void saveStyles(final List styleRanges) { - synchronized (previousRanges) { - previousRanges.clear(); - previousRanges.addAll(styleRanges); - previousRanges.sort(Comparator.comparing(s -> s.start)); - } - } - - private List getStyleRanges(final List dataStream, - final SemanticTokensLegend semanticTokensLegend) throws BadLocationException { - List styleRanges = new ArrayList<>(dataStream.size() / 5); - - int idx = 0; - int prevLine = 0; - int line = 0; - int offset = 0; - int length = 0; - String tokenType = null; - for (Integer data : dataStream) { - switch (idx % 5) { - case 0: // line - line += data; - break; - case 1: // offset - if (line == prevLine) { - offset += data; - } else { - offset = LSPEclipseUtils.toOffset(new Position(line, data), document); - } - break; - case 2: // length - length = data; - break; - case 3: // token type - tokenType = tokenType(data, semanticTokensLegend.getTokenTypes()); - break; - case 4: // token modifier - prevLine = line; - List tokenModifiers = tokenModifiers(data, semanticTokensLegend.getTokenModifiers()); - StyleRange styleRange = getStyleRange(offset, length, textAttribute(tokenType)); - if (tokenModifiers.stream().anyMatch(x -> x.equals(SemanticTokenModifiers.Deprecated))) { - styleRange.strikeout = true; - } - styleRanges.add(styleRange); - break; - } - idx++; - } - return styleRanges; - } - - private String tokenType(final Integer data, final List legend) { - try { - return legend.get(data - 1); - } catch (IndexOutOfBoundsException e) { - return null; // no match - } - } - - private List tokenModifiers(final Integer data, final List legend) { - if (data.intValue() == 0) { - return Collections.emptyList(); - } - BitSet bitSet = BitSet.valueOf(new long[] { data }); - List tokenModifiers = new ArrayList<>(); - for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { - try { - tokenModifiers.add(legend.get(i)); - } catch (IndexOutOfBoundsException e) { - // no match - } - } - - return tokenModifiers; - } - - private TextAttribute textAttribute(final String tokenType) { - ITheme localTheme = theme; - if (localTheme != null && tokenType != null) { - IToken token = localTheme.getToken(tokenType); - if (token != null) { - Object data = token.getData(); - if (data instanceof TextAttribute textAttribute) { - return textAttribute; - } - } - } - return null; - } - - /** - * Gets a style range for the given inputs. - * - * @param offset - * the offset of the range to be styled - * @param length - * the length of the range to be styled - * @param attr - * the attribute describing the style of the range to be styled - */ - private StyleRange getStyleRange(final int offset, final int length, final TextAttribute attr) { - final StyleRange styleRange; - if (attr != null) { - final int style = attr.getStyle(); - final int fontStyle = style & (SWT.ITALIC | SWT.BOLD | SWT.NORMAL); - styleRange = new StyleRange(offset, length, attr.getForeground(), attr.getBackground(), fontStyle); - styleRange.strikeout = (style & TextAttribute.STRIKETHROUGH) != 0; - styleRange.underline = (style & TextAttribute.UNDERLINE) != 0; - styleRange.font = attr.getFont(); - return styleRange; - } else { - styleRange = new StyleRange(); - styleRange.start = offset; - styleRange.length = length; - } - return styleRange; - } - @Override public void setProgressMonitor(final IProgressMonitor monitor) { } @@ -301,16 +173,7 @@ public void setProgressMonitor(final IProgressMonitor monitor) { @Override public void setDocument(final IDocument document) { this.document = document; - initSemanticTokensLegendMap(); - } - - private SemanticTokensLegend getSemanticTokensLegend(final LanguageServer languageSever) { - Optional serverDefinition = LanguageServiceAccessor - .resolveServerDefinition(languageSever); - if (serverDefinition.isPresent()) { - return semanticTokensLegendMap.get(serverDefinition.get().id); - } - return null; + semanticTokensLegendProvider.setDocument(document); } private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilities) { @@ -318,26 +181,66 @@ private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilitie && serverCapabilities.getSemanticTokensProvider().getFull().getLeft(); } - private CompletableFuture semanticTokensFull(final List languageServers) { - return CompletableFuture - .allOf(languageServers.stream().map(this::semanticTokensFull).toArray(CompletableFuture[]::new)); + private CompletableFuture semanticTokensFull(final List languageServers, final int version) { + return CompletableFuture.allOf( + languageServers.stream().map(ls -> semanticTokensFull(ls, version)).toArray(CompletableFuture[]::new)); } - private CompletableFuture semanticTokensFull(final LanguageServer languageServer) { + private boolean isRequestCancelledException(final Throwable throwable) { + if (throwable instanceof CompletionException) { + Throwable cause = ((CompletionException) throwable).getCause(); + if (cause instanceof ResponseErrorException) { + ResponseError responseError = ((ResponseErrorException) cause).getResponseError(); + return responseError != null + && responseError.getCode() == ResponseErrorCode.RequestCancelled.getValue(); + } + } + return false; + } + + private CompletableFuture semanticTokensFull(final LanguageServer languageServer, final int version) { SemanticTokensParams semanticTokensParams = getSemanticTokensParams(); return languageServer.getTextDocumentService().semanticTokensFull(semanticTokensParams) .thenAccept(semanticTokens -> { - saveStyle(semanticTokens, getSemanticTokensLegend(languageServer)); + if (getDocumentVersion() == version) { + saveStyle(semanticTokens, semanticTokensLegendProvider.getSemanticTokensLegend(languageServer)); + StyledText textWidget = viewer.getTextWidget(); + textWidget.getDisplay().asyncExec(() -> { + if (!textWidget.isDisposed() && documentVersionAtLastAppliedTextPresentation == version) { + viewer.invalidateTextPresentation(); + } + }); + } }).exceptionally(e -> { - LanguageServerPlugin.logError(e); + if (!isRequestCancelledException(e)) { + LanguageServerPlugin.logError(e); + } return null; }); } + private int getDocumentVersion() { + Iterator<@NonNull LSPDocumentInfo> iterator = LanguageServiceAccessor + .getLSPDocumentInfosFor(document, this::hasSemanticTokensFull).iterator(); + if (iterator.hasNext()) { + return iterator.next().getVersion(); + } + return -1; + } + + private void cancelSemanticTokensFull() { + if (semanticTokensFullFuture != null) { + semanticTokensFullFuture.cancel(true); + } + } + private void fullReconcile() { + cancelSemanticTokensFull(); try { - LanguageServiceAccessor.getLanguageServers(document, this::hasSemanticTokensFull)// - .thenAccept(this::semanticTokensFull).get(); + int version = getDocumentVersion(); + semanticTokensFullFuture = LanguageServiceAccessor.getLanguageServers(document, this::hasSemanticTokensFull)// + .thenAccept(ls -> semanticTokensFull(ls, version)); + semanticTokensFullFuture.get(); } catch (InterruptedException | ExecutionException e) { LanguageServerPlugin.logError(e); } @@ -358,19 +261,9 @@ public void reconcile(final IRegion partition) { fullReconcile(); } - private List appliedRanges(final TextPresentation textPresentation) { - synchronized (previousRanges) { - // we need to create new styles because the text presentation might change a - // style when applied to the presentation - // and we want the ones saved from the reconciling as immutable - return previousRanges.stream()// - .filter(r -> overlaps(r, textPresentation.getExtent()))// - .map(this::clone).collect(Collectors.toList()); - } - } - @Override public void applyTextPresentation(final TextPresentation textPresentation) { - mergeStyles(textPresentation, appliedRanges(textPresentation)); + documentVersionAtLastAppliedTextPresentation = getDocumentVersion(); + textPresentation.replaceStyleRanges(styleRangeHolder.overlappingRanges(textPresentation.getExtent())); } -} \ No newline at end of file +} diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java new file mode 100644 index 000000000..efad1f0cc --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.TextAttribute; +import org.eclipse.jface.text.rules.IToken; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.SemanticTokenModifiers; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.StyleRange; + +/** + * The Class SemanticTokensDataStreamProcessor translates a stream of integers + * as defined by the LSP SemanticTokenRequests into a list of StyleRanges. + */ +public class SemanticTokensDataStreamProcessor { + + private final Function offsetMapper; + private final Function tokenTypeMapper; + + /** + * Creates a new instance of {@link SemanticTokensDataStreamProcessor}. + * + * @param tokenTypeMapper + * @param offsetMapper + */ + public SemanticTokensDataStreamProcessor(@NonNull final Function tokenTypeMapper, + @NonNull final Function offsetMapper) { + this.tokenTypeMapper = tokenTypeMapper; + this.offsetMapper = offsetMapper; + } + + /** + * Get the StyleRanges for the given data stream and tokens legend. + * + * @param dataStream + * @param semanticTokensLegend + * @return + */ + public @NonNull List getStyleRanges(@NonNull final List dataStream, + @NonNull final SemanticTokensLegend semanticTokensLegend) { + List styleRanges = new ArrayList<>(dataStream.size() / 5); + + int idx = 0; + int prevLine = 0; + int line = 0; + int offset = 0; + int length = 0; + String tokenType = null; + for (Integer data : dataStream) { + switch (idx % 5) { + case 0: // line + line += data; + break; + case 1: // offset + if (line == prevLine) { + offset += data; + } else { + offset = offsetMapper.apply(new Position(line, data)); + } + break; + case 2: // length + length = data; + break; + case 3: // token type + tokenType = tokenType(data, semanticTokensLegend.getTokenTypes()); + break; + case 4: // token modifier + prevLine = line; + List tokenModifiers = tokenModifiers(data, semanticTokensLegend.getTokenModifiers()); + StyleRange styleRange = getStyleRange(offset, length, textAttribute(tokenType)); + if (tokenModifiers.stream().anyMatch(x -> x.equals(SemanticTokenModifiers.Deprecated))) { + styleRange.strikeout = true; + } + styleRanges.add(styleRange); + break; + } + idx++; + } + return styleRanges; + } + + private String tokenType(final Integer data, final List legend) { + try { + return legend.get(data - 1); + } catch (IndexOutOfBoundsException e) { + return null; // no match + } + } + + private List tokenModifiers(final Integer data, final List legend) { + if (data.intValue() == 0) { + return Collections.emptyList(); + } + BitSet bitSet = BitSet.valueOf(new long[] { data }); + List tokenModifiers = new ArrayList<>(); + for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { + try { + tokenModifiers.add(legend.get(i)); + } catch (IndexOutOfBoundsException e) { + // no match + } + } + + return tokenModifiers; + } + + private TextAttribute textAttribute(final String tokenType) { + if (tokenType != null) { + IToken token = tokenTypeMapper.apply(tokenType); + if (token != null) { + Object data = token.getData(); + if (data instanceof TextAttribute) { + return (TextAttribute) data; + } + } + } + return null; + } + + /** + * Gets a style range for the given inputs. + * + * @param offset + * the offset of the range to be styled + * @param length + * the length of the range to be styled + * @param attr + * the attribute describing the style of the range to be styled + */ + private StyleRange getStyleRange(final int offset, final int length, final TextAttribute attr) { + final StyleRange styleRange; + if (attr != null) { + final int style = attr.getStyle(); + final int fontStyle = style & (SWT.ITALIC | SWT.BOLD | SWT.NORMAL); + styleRange = new StyleRange(offset, length, attr.getForeground(), attr.getBackground(), fontStyle); + styleRange.strikeout = (style & TextAttribute.STRIKETHROUGH) != 0; + styleRange.underline = (style & TextAttribute.UNDERLINE) != 0; + styleRange.font = attr.getFont(); + return styleRange; + } else { + styleRange = new StyleRange(); + styleRange.start = offset; + styleRange.length = length; + } + return styleRange; + } +} \ No newline at end of file diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java new file mode 100644 index 000000000..da662c2b3 --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java @@ -0,0 +1,89 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.eclipse.core.resources.IFile; +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.IDocument; +import org.eclipse.lsp4e.LSPEclipseUtils; +import org.eclipse.lsp4e.LanguageServerPlugin; +import org.eclipse.lsp4e.LanguageServerWrapper; +import org.eclipse.lsp4e.LanguageServersRegistry.LanguageServerDefinition; +import org.eclipse.lsp4e.LanguageServiceAccessor; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; +import org.eclipse.lsp4j.ServerCapabilities; +import org.eclipse.lsp4j.services.LanguageServer; + + +/** + * A provider for {@link SemanticTokensLegend}. + */ +public class SemanticTokensLegendProvider { +private Map semanticTokensLegendMap; +private IDocument document; + +/** + * Tells the provider strategy on which document it will + * work. + * + * @param document + * the document on which this mapper will work + */ +public void setDocument(final IDocument document) { + this.document = document; +} + +private void initSemanticTokensLegendMap() { + IFile file = LSPEclipseUtils.getFile(document); + if (file != null) { + try { + semanticTokensLegendMap = new HashMap<>(); + for (LanguageServerWrapper wrapper: LanguageServiceAccessor.getLSWrappers(file, x -> true)) { + ServerCapabilities serverCapabilities = wrapper.getServerCapabilities(); + if (serverCapabilities != null) { + SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities.getSemanticTokensProvider(); + if (semanticTokensProvider != null) { + semanticTokensLegendMap.put(wrapper.serverDefinition.id, semanticTokensProvider.getLegend()); + } + } + } + } catch (IOException e) { + semanticTokensLegendMap = Collections.emptyMap(); + LanguageServerPlugin.logError(e); + } + } else { + semanticTokensLegendMap = Collections.emptyMap(); + } +} + +/** + * Gets the {@link SemanticTokensLegend} for the given {@link LanguageServer}. + * + * @param languageSever + * @return + */ +public SemanticTokensLegend getSemanticTokensLegend(@NonNull final LanguageServer languageSever) { + Optional serverDefinition = LanguageServiceAccessor.resolveServerDefinition(languageSever); + if (serverDefinition.isPresent()) { + if (semanticTokensLegendMap == null) { + initSemanticTokensLegendMap(); + } + return semanticTokensLegendMap.get(serverDefinition.get().id); + } + return null; +} +} diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java new file mode 100644 index 000000000..d03b4a04d --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.IRegion; +import org.eclipse.jface.text.ITextListener; +import org.eclipse.jface.text.Region; +import org.eclipse.jface.text.TextEvent; +import org.eclipse.swt.custom.StyleRange; + +/** + * The Class SemanticTokensDataStreamProcessor holds a list of StyleRanges. + *

+ * To avoid flickering, we also implement {@link ITextListener} to adapt (the + * only adaptation currently supported shifting ranges) recorded semantic + * highlights When the user writes a single or multi-line comments shifting is + * not enough. That could be improved if we can access + * org.eclipse.tm4e.languageconfiguration.ILanguageConfiguration.getComments() + * (still unclear on how to do that). + */ +public class StyleRangeHolder implements ITextListener { + private final List previousRanges; + + public StyleRangeHolder() { + previousRanges = new ArrayList<>(); + } + + /** + * save the styles. + * + * @param styleRanges + */ + public void saveStyles(@NonNull final List styleRanges) { + synchronized (previousRanges) { + previousRanges.clear(); + previousRanges.addAll(styleRanges); + previousRanges.sort(Comparator.comparing(s -> s.start)); + } + } + + /** + * return a copy of the saved styles that overlap the given region. + * + * @param region + * @return + */ + public StyleRange[] overlappingRanges(@NonNull final IRegion region) { + synchronized (previousRanges) { + // we need to create new styles because the text presentation might change a + // style when applied to the presentation + // and we want the ones saved from the reconciling as immutable + return previousRanges.stream()// + .filter(r -> overlaps(r, region))// + .map(this::clone).toArray(StyleRange[]::new); + } + } + + private StyleRange clone(final StyleRange styleRange) { + StyleRange clonedStyleRange = new StyleRange(styleRange.start, styleRange.length, styleRange.foreground, + styleRange.background, styleRange.fontStyle); + clonedStyleRange.strikeout = styleRange.strikeout; + return clonedStyleRange; + } + + private boolean isContained(final int offset, final StyleRange range) { + return offset >= range.start && offset < (range.start + range.length); + } + + private boolean isContained(final int offset, final IRegion region) { + return offset >= region.getOffset() && offset < (region.getOffset() + region.getLength()); + } + + private boolean overlaps(final StyleRange range, final IRegion region) { + return isContained(range.start, region) || isContained(range.start + range.length, region) + || isContained(region.getOffset(), range); + } + + @Override + public void textChanged(final TextEvent event) { + if (event.getDocumentEvent() != null) { // if null, it is an internal event, not a changed text + String replacedText = event.getReplacedText(); + String text = event.getText(); + int delta = (text != null ? text.length() : 0) - (replacedText != null ? replacedText.length() : 0); + synchronized (previousRanges) { + previousRanges.removeIf(r -> isContained(event.getOffset(), new Region(r.start, r.length))); + previousRanges.stream().filter(r -> r.start >= event.getOffset()).forEach(r -> r.start += delta); + } + } + } + +} diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java new file mode 100644 index 000000000..bf7b2900c --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.util.function.Function; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.jface.text.rules.IToken; +import org.eclipse.tm4e.ui.TMUIPlugin; +import org.eclipse.tm4e.ui.text.TMPresentationReconciler; +import org.eclipse.tm4e.ui.themes.ITokenProvider; + +/** + * A Class that maps TokenTypes to {@link IToken}. + */ +public class TokenTypeMapper implements Function { + private @NonNull final ITextViewer viewer; + + public TokenTypeMapper(final ITextViewer viewer) { + this.viewer = viewer; + } + + @Override + public IToken apply(final String tokenType) { + TMPresentationReconciler tmPresentationReconciler = TMPresentationReconciler + .getTMPresentationReconciler(viewer); + + if (tmPresentationReconciler != null) { + ITokenProvider tokenProvider = tmPresentationReconciler.getTokenProvider(); + if (tokenProvider != null) { + tokenProvider.getToken(tokenType); + } + } + return TMUIPlugin.getThemeManager().getDefaultTheme().getToken(tokenType); + } +} From 60ef8b84720da9c3e44b3fd1bf9af66e2688f24e Mon Sep 17 00:00:00 2001 From: rubenporras Date: Tue, 13 Dec 2022 09:47:59 +0100 Subject: [PATCH 3/5] Fix JDT Null warnings --- .../SemanticHighlightReconciler.java | 3 ++ .../SemanticHighlightReconcilerStrategy.java | 43 ++++++++++++------- .../semanticTokens/TokenTypeMapper.java | 5 ++- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java index 0a542d6c8..78771850b 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java @@ -8,6 +8,8 @@ *******************************************************************************/ package org.eclipse.lsp4e.operations.semanticTokens; +import java.util.Objects; + import org.eclipse.jface.text.IDocument; import org.eclipse.jface.text.ITextViewer; import org.eclipse.jface.text.reconciler.MonoReconciler; @@ -23,6 +25,7 @@ public void install(final ITextViewer textViewer) { super.install(textViewer); // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is // accepted + Objects.requireNonNull(textViewer); ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)) .install(textViewer); } diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java index 46331ae96..826306a4c 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java @@ -18,6 +18,7 @@ import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.Nullable; import org.eclipse.jface.text.BadLocationException; import org.eclipse.jface.text.IDocument; import org.eclipse.jface.text.IRegion; @@ -106,10 +107,10 @@ public class SemanticHighlightReconcilerStrategy * @param textViewer * the viewer on which the reconciler is installed */ - public void install(final ITextViewer textViewer) { + public void install(@NonNull final ITextViewer textViewer) { viewer = textViewer; styleRangeHolder = new StyleRangeHolder(); - semanticTokensDataStreamProcessor = new SemanticTokensDataStreamProcessor(new TokenTypeMapper(viewer), + semanticTokensDataStreamProcessor = new SemanticTokensDataStreamProcessor(new TokenTypeMapper(textViewer), offsetMapper()); semanticTokensLegendProvider = new SemanticTokensLegendProvider(); @@ -134,7 +135,7 @@ public void uninstall() { semanticTokensLegendProvider = null; } - private Function offsetMapper() { + private @NonNull Function offsetMapper() { return (p) -> { try { return LSPEclipseUtils.toOffset(p, document); @@ -220,10 +221,16 @@ private CompletableFuture semanticTokensFull(final LanguageServer language } private int getDocumentVersion() { - Iterator<@NonNull LSPDocumentInfo> iterator = LanguageServiceAccessor - .getLSPDocumentInfosFor(document, this::hasSemanticTokensFull).iterator(); - if (iterator.hasNext()) { - return iterator.next().getVersion(); + IDocument theDocument = document; + if (theDocument != null) { + Iterator<@NonNull LSPDocumentInfo> iterator = LanguageServiceAccessor + .getLSPDocumentInfosFor(theDocument, this::hasSemanticTokensFull).iterator(); + if (iterator.hasNext()) { + @Nullable LSPDocumentInfo documentInfo = iterator.next(); + if (documentInfo != null) { + return documentInfo.getVersion(); + } + } } return -1; } @@ -235,14 +242,17 @@ private void cancelSemanticTokensFull() { } private void fullReconcile() { + IDocument theDocument = document; cancelSemanticTokensFull(); - try { - int version = getDocumentVersion(); - semanticTokensFullFuture = LanguageServiceAccessor.getLanguageServers(document, this::hasSemanticTokensFull)// - .thenAccept(ls -> semanticTokensFull(ls, version)); - semanticTokensFullFuture.get(); - } catch (InterruptedException | ExecutionException e) { - LanguageServerPlugin.logError(e); + if (theDocument != null) { + try { + int version = getDocumentVersion(); + semanticTokensFullFuture = LanguageServiceAccessor.getLanguageServers(theDocument, this::hasSemanticTokensFull)// + .thenAccept(ls -> semanticTokensFull(ls, version)); + semanticTokensFullFuture.get(); + } catch (InterruptedException | ExecutionException e) { + LanguageServerPlugin.logError(e); + } } } @@ -264,6 +274,9 @@ public void reconcile(final IRegion partition) { @Override public void applyTextPresentation(final TextPresentation textPresentation) { documentVersionAtLastAppliedTextPresentation = getDocumentVersion(); - textPresentation.replaceStyleRanges(styleRangeHolder.overlappingRanges(textPresentation.getExtent())); + IRegion extent = textPresentation.getExtent(); + if (extent != null) { + textPresentation.replaceStyleRanges(styleRangeHolder.overlappingRanges(extent)); + } } } diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java index bf7b2900c..00fdda7bf 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java @@ -24,12 +24,15 @@ public class TokenTypeMapper implements Function { private @NonNull final ITextViewer viewer; - public TokenTypeMapper(final ITextViewer viewer) { + public TokenTypeMapper(@NonNull final ITextViewer viewer) { this.viewer = viewer; } @Override public IToken apply(final String tokenType) { + if (tokenType == null) { + return null; + } TMPresentationReconciler tmPresentationReconciler = TMPresentationReconciler .getTMPresentationReconciler(viewer); From 9609c8318f95b1ca724ec885655c6decc7921216 Mon Sep 17 00:00:00 2001 From: rubenporras Date: Tue, 13 Dec 2022 13:31:24 +0100 Subject: [PATCH 4/5] Add tests --- .../src/org/eclipse/lsp4e/test/AllTests.java | 10 +- ...manticHighlightReconcilerStrategyTest.java | 73 +++++++++++ ...SemanticTokensDataStreamProcessorTest.java | 54 ++++++++ .../SemanticTokensLegendProviderTest.java | 63 +++++++++ .../semanticTokens/SemanticTokensUtil.java | 122 ++++++++++++++++++ .../semanticTokens/StyleRangeHolderTest.java | 77 +++++++++++ .../tests/mock/MockTextDocumentService.java | 11 ++ .../lsp4e/LanguageServiceAccessor.java | 11 +- .../SemanticHighlightReconcilerStrategy.java | 29 ++++- .../SemanticTokensLegendProvider.java | 89 ------------- 10 files changed, 438 insertions(+), 101 deletions(-) create mode 100644 org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java create mode 100644 org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java create mode 100644 org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java create mode 100644 org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensUtil.java create mode 100644 org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/StyleRangeHolderTest.java delete mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java index b6a731965..bef29aafa 100644 --- a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java @@ -44,6 +44,10 @@ import org.eclipse.lsp4e.test.references.FindReferencesTest; import org.eclipse.lsp4e.test.rename.LSPTextChangeTest; import org.eclipse.lsp4e.test.rename.RenameTest; +import org.eclipse.lsp4e.test.semanticTokens.SemanticHighlightReconcilerStrategyTest; +import org.eclipse.lsp4e.test.semanticTokens.SemanticTokensDataStreamProcessorTest; +import org.eclipse.lsp4e.test.semanticTokens.SemanticTokensLegendProviderTest; +import org.eclipse.lsp4e.test.semanticTokens.StyleRangeHolderTest; import org.eclipse.lsp4e.test.symbols.SymbolsModelTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -88,7 +92,11 @@ LSPCodeMiningTest.class, ShowMessageTest.class, WorkspaceFoldersTest.class, - DebugTest.class + DebugTest.class, + SemanticHighlightReconcilerStrategyTest.class, + SemanticTokensDataStreamProcessorTest.class, + SemanticTokensLegendProviderTest.class, + StyleRangeHolderTest.class }) public class AllTests { diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java new file mode 100644 index 000000000..b1466fbcb --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import static org.junit.Assert.assertArrayEquals; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import org.eclipse.core.resources.IFile; +import org.eclipse.core.resources.IProject; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.lsp4e.test.AllCleanRule; +import org.eclipse.lsp4e.test.TestUtils; +import org.eclipse.lsp4e.tests.mock.MockLanguageServer; +import org.eclipse.lsp4j.SemanticTokens; +import org.eclipse.swt.custom.StyleRange; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.ui.tests.harness.util.DisplayHelper; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +public class SemanticHighlightReconcilerStrategyTest { + @Rule + public AllCleanRule clear = new AllCleanRule(); + + private IProject project; + private Shell shell; + + @Before + public void setUp() throws CoreException { + project = TestUtils.createProject(getClass().getName() + System.currentTimeMillis()); + shell = new Shell(); + + // Setup Server Capabilities + List tokenTypes = Arrays.asList("keyword"); + List tokenModifiers = Arrays.asList("obsolete"); + SemanticTokensUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers); + } + + @Test + public void testKeyword() throws InterruptedException, ExecutionException, CoreException { + SemanticTokens semanticTokens = new SemanticTokens(); + semanticTokens.setData(SemanticTokensUtil.keywordSemanticTokens()); + + MockLanguageServer.INSTANCE.getTextDocumentService().setSemanticTokens(semanticTokens); + + IFile file = TestUtils.createUniqueTestFile(project, "lspt", SemanticTokensUtil.keywordText); + ITextViewer textViewer = TestUtils.openTextViewer(file); + + Display display = shell.getDisplay(); + DisplayHelper.sleep(display, 2_000); // Give some time to the editor to update + + StyleRange[] styleRanges = textViewer.getTextWidget().getStyleRanges(); + + List expectedStyleRanges = Arrays.asList(// + new StyleRange(0, 4, SemanticTokensUtil.GREEN, null), // + new StyleRange(15, 4, SemanticTokensUtil.GREEN, null), // + new StyleRange(24, 7, SemanticTokensUtil.GREEN, null)// + ); + assertArrayEquals(expectedStyleRanges.toArray(), styleRanges); + } +} diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java new file mode 100644 index 000000000..176940aba --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import org.eclipse.jface.text.Document; +import org.eclipse.lsp4e.operations.semanticTokens.SemanticTokensDataStreamProcessor; +import org.eclipse.lsp4e.test.AllCleanRule; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.swt.custom.StyleRange; +import org.junit.Rule; +import org.junit.Test; + +public class SemanticTokensDataStreamProcessorTest { + @Rule + public AllCleanRule clear = new AllCleanRule(); + + @Test + public void testKeyword() throws InterruptedException, ExecutionException { + Document document = new Document(SemanticTokensUtil.keywordText); + + SemanticTokensDataStreamProcessor processor = new SemanticTokensDataStreamProcessor(SemanticTokensUtil + .keywordTokenTypeMapper(SemanticTokensUtil.RED_TOKEN), SemanticTokensUtil.offsetMapper(document)); + + List expectedStream = SemanticTokensUtil.keywordSemanticTokens(); + List expectedStyleRanges = Arrays.asList(// + new StyleRange(0, 4, SemanticTokensUtil.RED, null), // + new StyleRange(15, 4, SemanticTokensUtil.RED, null), // + new StyleRange(24, 7, SemanticTokensUtil.RED, null)// + ); + + List styleRanges = processor.getStyleRanges(expectedStream, getSemanticTokensLegend()); + + assertEquals(expectedStyleRanges, styleRanges); + } + + private SemanticTokensLegend getSemanticTokensLegend() { + SemanticTokensLegend semanticTokensLegend = new SemanticTokensLegend(); + semanticTokensLegend.setTokenTypes(Arrays.asList("keyword","other")); + semanticTokensLegend.setTokenModifiers(Arrays.asList("obsolete")); + return semanticTokensLegend; + } +} diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java new file mode 100644 index 000000000..db17202c7 --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java @@ -0,0 +1,63 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import org.eclipse.core.resources.IFile; +import org.eclipse.core.resources.IProject; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.jface.text.BadLocationException; +import org.eclipse.lsp4e.LanguageServiceAccessor; +import org.eclipse.lsp4e.operations.semanticTokens.SemanticHighlightReconcilerStrategy; +import org.eclipse.lsp4e.test.AllCleanRule; +import org.eclipse.lsp4e.test.TestUtils; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.lsp4j.services.LanguageServer; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +public class SemanticTokensLegendProviderTest { + + @Rule + public AllCleanRule clear = new AllCleanRule(); + + private IProject project; + + @Before + public void setUp() throws CoreException { + project = TestUtils.createProject(getClass().getName() + System.currentTimeMillis()); + } + + @Test + public void testSemanticTokensLegendProvider() throws BadLocationException, CoreException, IOException, InterruptedException, ExecutionException { + // Setup Server Capabilities + List tokenTypes = Arrays.asList("keyword","other"); + List tokenModifiers = Arrays.asList("obsolete"); + SemanticTokensUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers); + + // Setup test data + IFile file = TestUtils.createUniqueTestFile(project, "lspt", "test content"); + // start the LS + LanguageServer languageServer = LanguageServiceAccessor.getInitializedLanguageServers(file, c -> Boolean.TRUE).iterator() + .next().get(); + + SemanticTokensLegend semanticTokensLegend = (new SemanticHighlightReconcilerStrategy()).getSemanticTokensLegend(languageServer); + assertNotNull(semanticTokensLegend); + assertEquals(tokenTypes, semanticTokensLegend.getTokenTypes()); + assertEquals(tokenModifiers, semanticTokensLegend.getTokenModifiers()); + } +} diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensUtil.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensUtil.java new file mode 100644 index 000000000..71d210bc4 --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensUtil.java @@ -0,0 +1,122 @@ +package org.eclipse.lsp4e.test.semanticTokens; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.BadLocationException; +import org.eclipse.jface.text.IDocument; +import org.eclipse.jface.text.TextAttribute; +import org.eclipse.jface.text.rules.IToken; +import org.eclipse.lsp4e.LSPEclipseUtils; +import org.eclipse.lsp4e.tests.mock.MockLanguageServer; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; +import org.eclipse.swt.graphics.Color; + +public class SemanticTokensUtil { + public static final String keywordText = + "type foo {\n" + + " \n" + + "}\n" + + "type bar extends foo {\n" + + " \n" + + "}\n"; + + public static List keywordSemanticTokens() { + List> expectedTokens = new ArrayList<>(); + expectedTokens.add(Arrays.asList(0,0,4,1,0)); + expectedTokens.add(Arrays.asList(3,0,4,1,0)); + expectedTokens.add(Arrays.asList(0,9,7,1,0)); + + return expectedTokens.stream().flatMap(List::stream).toList(); + } + + public static final Color GREEN = new Color(133, 153, 0, 255); + public static final Color RED = new Color(255, 0, 0); + + public static final IToken GREEN_TOKEN = new IToken() { + @Override + public boolean isWhitespace() { + return false; + } + + @Override + public boolean isUndefined() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isEOF() { + return false; + } + + @Override + public Object getData() { + return new TextAttribute(GREEN); + } + }; + + public static final IToken RED_TOKEN = new IToken() { + @Override + public boolean isWhitespace() { + return false; + } + + @Override + public boolean isUndefined() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isEOF() { + return false; + } + + @Override + public Object getData() { + return new TextAttribute(RED); + } + }; + + public static Function keywordTokenTypeMapper(final IToken token) { + return t -> { + if ("keyword".equals(t)) { + return token; + } else { + return null; + } + }; + } + + public static @NonNull Function offsetMapper(IDocument document) { + return (p) -> { + try { + return LSPEclipseUtils.toOffset(p, document); + } catch (BadLocationException e) { + throw new RuntimeException(e); + } + }; + } + public static void setSemanticTokensLegend(final List tokenTypes, List tokenModifiers) { + SemanticTokensLegend legend = new SemanticTokensLegend(tokenTypes, tokenModifiers); + SemanticTokensWithRegistrationOptions semanticTokensWithRegistrationOptions = new SemanticTokensWithRegistrationOptions(legend); + semanticTokensWithRegistrationOptions.setFull(true); + semanticTokensWithRegistrationOptions.setRange(false); + + MockLanguageServer.INSTANCE.getInitializeResult().getCapabilities().setSemanticTokensProvider(semanticTokensWithRegistrationOptions); + } +} diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/StyleRangeHolderTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/StyleRangeHolderTest.java new file mode 100644 index 000000000..86f1c6a02 --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/StyleRangeHolderTest.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.util.Arrays; +import java.util.List; + +import org.eclipse.jface.text.DocumentEvent; +import org.eclipse.jface.text.Region; +import org.eclipse.jface.text.TextEvent; +import org.eclipse.lsp4e.operations.semanticTokens.StyleRangeHolder; +import org.eclipse.lsp4e.test.AllCleanRule; +import org.eclipse.swt.custom.StyleRange; +import org.eclipse.swt.graphics.Color; +import org.junit.Rule; +import org.junit.Test; + +public class StyleRangeHolderTest { + @Rule + public AllCleanRule clear = new AllCleanRule(); + + private static final Color RED = new Color(255, 0, 0); + private List originalStyleRanges = Arrays.asList(new StyleRange(0, 4, RED, null), new StyleRange(15, 4, RED, null), new StyleRange(24, 7, RED, null)); + + @Test + public void testAllDocumentRanges() { + StyleRangeHolder holder = new StyleRangeHolder(); + holder.saveStyles(originalStyleRanges); + + StyleRange[] allDocumentRanges = holder.overlappingRanges(new Region(0, 50)); + + assertNotEquals(originalStyleRanges, allDocumentRanges); // styles must be copied + assertEquals(originalStyleRanges.size(), allDocumentRanges.length); + } + + @Test + public void testPartialDocumentRanges() { + StyleRangeHolder holder = new StyleRangeHolder(); + holder.saveStyles(originalStyleRanges); + + StyleRange[] allDocumentRanges = holder.overlappingRanges(new Region(0, 20)); // only two ranges overlap this region + + assertEquals(2, allDocumentRanges.length); + } + + @Test + public void testDocumentChange() { + StyleRangeHolder holder = new StyleRangeHolder(); + holder.saveStyles(originalStyleRanges); + + TextEvent textEvent = new TextEvent(0, 1, " ", null, new DocumentEvent(), false) {}; + + // this will remove the first style and shift the last two + holder.textChanged(textEvent); + + StyleRange[] noOverlappingRanges = holder.overlappingRanges(new Region(0, 10)); // only one range overlap this region + + assertEquals(0, noOverlappingRanges.length); + + StyleRange[] twoShiftedOverlappingRanges = holder.overlappingRanges(new Region(10, 50)); // only one range overlap this region + + assertEquals(2, twoShiftedOverlappingRanges.length); + assertEquals(16, twoShiftedOverlappingRanges[0].start); + assertEquals(4, twoShiftedOverlappingRanges[0].length); + assertEquals(25, twoShiftedOverlappingRanges[1].start); + assertEquals(7, twoShiftedOverlappingRanges[1].length); + } +} diff --git a/org.eclipse.lsp4e.tests.mock/src/org/eclipse/lsp4e/tests/mock/MockTextDocumentService.java b/org.eclipse.lsp4e.tests.mock/src/org/eclipse/lsp4e/tests/mock/MockTextDocumentService.java index ccb57ac61..9073bc830 100644 --- a/org.eclipse.lsp4e.tests.mock/src/org/eclipse/lsp4e/tests/mock/MockTextDocumentService.java +++ b/org.eclipse.lsp4e.tests.mock/src/org/eclipse/lsp4e/tests/mock/MockTextDocumentService.java @@ -64,6 +64,8 @@ import org.eclipse.lsp4j.Range; import org.eclipse.lsp4j.ReferenceParams; import org.eclipse.lsp4j.RenameParams; +import org.eclipse.lsp4j.SemanticTokens; +import org.eclipse.lsp4j.SemanticTokensParams; import org.eclipse.lsp4j.SignatureHelp; import org.eclipse.lsp4j.SignatureHelpParams; import org.eclipse.lsp4j.SymbolInformation; @@ -104,6 +106,7 @@ public class MockTextDocumentService implements TextDocumentService { private WorkspaceEdit mockRenameEdit; private Either3 mockPrepareRenameResult; private List documentSymbols; + private SemanticTokens mockSemanticTokens; public MockTextDocumentService(Function> futureFactory) { this._futureFactory = futureFactory; @@ -387,4 +390,12 @@ public void setWillSaveWaitUntilCallback(List textEdits) { this.mockWillSaveWaitUntilTextEdits = textEdits; } + public void setSemanticTokens(final SemanticTokens semanticTokens) { + this.mockSemanticTokens = semanticTokens; + } + + @Override + public CompletableFuture semanticTokensFull(SemanticTokensParams params) { + return CompletableFuture.completedFuture(this.mockSemanticTokens); + } } diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java index 74eaded19..f7f6777f0 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java @@ -397,7 +397,7 @@ private static Collection getLSWrappers(@NonNull final ID } if (contentType.getBaseType() != null) { - contentTypesToProcess.add(contentType.getBaseType()); + contentTypesToProcess.add(contentType.getBaseType()); } processedContentTypes.add(contentType); } @@ -648,9 +648,12 @@ public static boolean checkCapability(LanguageServer languageServer, Predicate condition.test(wrapper.getServerCapabilities())); } - public static Optional resolveServerDefinition(LanguageServer languageServer) { + public static Optional resolveLanguageServerWrapper(LanguageServer languageServer) { return startedServers.stream() // - .filter(wrapper -> languageServer.equals(wrapper.getServer())).findFirst() - .map(wrapper -> wrapper.serverDefinition); + .filter(wrapper -> languageServer.equals(wrapper.getServer())).findFirst(); + } + + public static Optional resolveServerDefinition(LanguageServer languageServer) { + return resolveLanguageServerWrapper(languageServer).map(w -> w.serverDefinition); } } diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java index 826306a4c..d79b0e718 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java @@ -32,12 +32,14 @@ import org.eclipse.jface.text.reconciler.IReconcilingStrategyExtension; import org.eclipse.lsp4e.LSPEclipseUtils; import org.eclipse.lsp4e.LanguageServerPlugin; +import org.eclipse.lsp4e.LanguageServerWrapper; import org.eclipse.lsp4e.LanguageServiceAccessor; import org.eclipse.lsp4e.LanguageServiceAccessor.LSPDocumentInfo; import org.eclipse.lsp4j.Position; import org.eclipse.lsp4j.SemanticTokens; import org.eclipse.lsp4j.SemanticTokensLegend; import org.eclipse.lsp4j.SemanticTokensParams; +import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; import org.eclipse.lsp4j.ServerCapabilities; import org.eclipse.lsp4j.TextDocumentIdentifier; import org.eclipse.lsp4j.jsonrpc.ResponseErrorException; @@ -84,8 +86,6 @@ public class SemanticHighlightReconcilerStrategy private SemanticTokensDataStreamProcessor semanticTokensDataStreamProcessor; - private SemanticTokensLegendProvider semanticTokensLegendProvider; - /** * Written in {@link this.class#applyTextPresentation(TextPresentation)} * applyTextPresentation and read in the lambda in @@ -112,7 +112,6 @@ public void install(@NonNull final ITextViewer textViewer) { styleRangeHolder = new StyleRangeHolder(); semanticTokensDataStreamProcessor = new SemanticTokensDataStreamProcessor(new TokenTypeMapper(textViewer), offsetMapper()); - semanticTokensLegendProvider = new SemanticTokensLegendProvider(); if (viewer instanceof TextViewer) { ((TextViewer) viewer).addTextPresentationListener(this); @@ -132,7 +131,6 @@ public void uninstall() { viewer.removeTextListener(styleRangeHolder); viewer = null; styleRangeHolder = null; - semanticTokensLegendProvider = null; } private @NonNull Function offsetMapper() { @@ -174,7 +172,6 @@ public void setProgressMonitor(final IProgressMonitor monitor) { @Override public void setDocument(final IDocument document) { this.document = document; - semanticTokensLegendProvider.setDocument(document); } private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilities) { @@ -199,15 +196,33 @@ private boolean isRequestCancelledException(final Throwable throwable) { return false; } + private @Nullable SemanticTokensLegend getSemanticTokensLegend(final LanguageServerWrapper wrapper) { + ServerCapabilities serverCapabilities = wrapper.getServerCapabilities(); + if (serverCapabilities != null) { + SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities + .getSemanticTokensProvider(); + if (semanticTokensProvider != null) { + return semanticTokensProvider.getLegend(); + } + } + return null; + } + + // public for testing + public @Nullable SemanticTokensLegend getSemanticTokensLegend(@NonNull final LanguageServer languageSever) { + return LanguageServiceAccessor.resolveLanguageServerWrapper(languageSever) + .map(this::getSemanticTokensLegend).orElse(null); + } + private CompletableFuture semanticTokensFull(final LanguageServer languageServer, final int version) { SemanticTokensParams semanticTokensParams = getSemanticTokensParams(); return languageServer.getTextDocumentService().semanticTokensFull(semanticTokensParams) .thenAccept(semanticTokens -> { if (getDocumentVersion() == version) { - saveStyle(semanticTokens, semanticTokensLegendProvider.getSemanticTokensLegend(languageServer)); + saveStyle(semanticTokens, getSemanticTokensLegend(languageServer)); StyledText textWidget = viewer.getTextWidget(); textWidget.getDisplay().asyncExec(() -> { - if (!textWidget.isDisposed() && documentVersionAtLastAppliedTextPresentation == version) { + if (!textWidget.isDisposed() && (documentVersionAtLastAppliedTextPresentation == 0 || documentVersionAtLastAppliedTextPresentation == version )) { viewer.invalidateTextPresentation(); } }); diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java deleted file mode 100644 index da662c2b3..000000000 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java +++ /dev/null @@ -1,89 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2022 Avaloq Group AG. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - *******************************************************************************/ -package org.eclipse.lsp4e.operations.semanticTokens; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import org.eclipse.core.resources.IFile; -import org.eclipse.jdt.annotation.NonNull; -import org.eclipse.jface.text.IDocument; -import org.eclipse.lsp4e.LSPEclipseUtils; -import org.eclipse.lsp4e.LanguageServerPlugin; -import org.eclipse.lsp4e.LanguageServerWrapper; -import org.eclipse.lsp4e.LanguageServersRegistry.LanguageServerDefinition; -import org.eclipse.lsp4e.LanguageServiceAccessor; -import org.eclipse.lsp4j.SemanticTokensLegend; -import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; -import org.eclipse.lsp4j.ServerCapabilities; -import org.eclipse.lsp4j.services.LanguageServer; - - -/** - * A provider for {@link SemanticTokensLegend}. - */ -public class SemanticTokensLegendProvider { -private Map semanticTokensLegendMap; -private IDocument document; - -/** - * Tells the provider strategy on which document it will - * work. - * - * @param document - * the document on which this mapper will work - */ -public void setDocument(final IDocument document) { - this.document = document; -} - -private void initSemanticTokensLegendMap() { - IFile file = LSPEclipseUtils.getFile(document); - if (file != null) { - try { - semanticTokensLegendMap = new HashMap<>(); - for (LanguageServerWrapper wrapper: LanguageServiceAccessor.getLSWrappers(file, x -> true)) { - ServerCapabilities serverCapabilities = wrapper.getServerCapabilities(); - if (serverCapabilities != null) { - SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities.getSemanticTokensProvider(); - if (semanticTokensProvider != null) { - semanticTokensLegendMap.put(wrapper.serverDefinition.id, semanticTokensProvider.getLegend()); - } - } - } - } catch (IOException e) { - semanticTokensLegendMap = Collections.emptyMap(); - LanguageServerPlugin.logError(e); - } - } else { - semanticTokensLegendMap = Collections.emptyMap(); - } -} - -/** - * Gets the {@link SemanticTokensLegend} for the given {@link LanguageServer}. - * - * @param languageSever - * @return - */ -public SemanticTokensLegend getSemanticTokensLegend(@NonNull final LanguageServer languageSever) { - Optional serverDefinition = LanguageServiceAccessor.resolveServerDefinition(languageSever); - if (serverDefinition.isPresent()) { - if (semanticTokensLegendMap == null) { - initSemanticTokensLegendMap(); - } - return semanticTokensLegendMap.get(serverDefinition.get().id); - } - return null; -} -} From df9262edb7cc0e560500861cfb1fbd792498e2d8 Mon Sep 17 00:00:00 2001 From: pcr Date: Thu, 15 Dec 2022 16:35:36 +0100 Subject: [PATCH 5/5] Add configuration option to disable the reconciler. --- ...manticHighlightReconcilerStrategyTest.java | 12 +++---- ...SemanticTokensDataStreamProcessorTest.java | 14 ++++---- .../SemanticTokensLegendProviderTest.java | 2 +- ...sUtil.java => SemanticTokensTestUtil.java} | 10 +++++- .../lsp4e/LanguageServiceAccessor.java | 2 +- .../SemanticHighlightReconcilerStrategy.java | 32 ++++++++++++++++++- 6 files changed, 55 insertions(+), 17 deletions(-) rename org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/{SemanticTokensUtil.java => SemanticTokensTestUtil.java} (86%) diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java index b1466fbcb..ef61b6903 100644 --- a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java @@ -45,17 +45,17 @@ public void setUp() throws CoreException { // Setup Server Capabilities List tokenTypes = Arrays.asList("keyword"); List tokenModifiers = Arrays.asList("obsolete"); - SemanticTokensUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers); + SemanticTokensTestUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers); } @Test public void testKeyword() throws InterruptedException, ExecutionException, CoreException { SemanticTokens semanticTokens = new SemanticTokens(); - semanticTokens.setData(SemanticTokensUtil.keywordSemanticTokens()); + semanticTokens.setData(SemanticTokensTestUtil.keywordSemanticTokens()); MockLanguageServer.INSTANCE.getTextDocumentService().setSemanticTokens(semanticTokens); - IFile file = TestUtils.createUniqueTestFile(project, "lspt", SemanticTokensUtil.keywordText); + IFile file = TestUtils.createUniqueTestFile(project, "lspt", SemanticTokensTestUtil.keywordText); ITextViewer textViewer = TestUtils.openTextViewer(file); Display display = shell.getDisplay(); @@ -64,9 +64,9 @@ public void testKeyword() throws InterruptedException, ExecutionException, CoreE StyleRange[] styleRanges = textViewer.getTextWidget().getStyleRanges(); List expectedStyleRanges = Arrays.asList(// - new StyleRange(0, 4, SemanticTokensUtil.GREEN, null), // - new StyleRange(15, 4, SemanticTokensUtil.GREEN, null), // - new StyleRange(24, 7, SemanticTokensUtil.GREEN, null)// + new StyleRange(0, 4, SemanticTokensTestUtil.GREEN, null), // + new StyleRange(15, 4, SemanticTokensTestUtil.GREEN, null), // + new StyleRange(24, 7, SemanticTokensTestUtil.GREEN, null)// ); assertArrayEquals(expectedStyleRanges.toArray(), styleRanges); } diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java index 176940aba..e40a915c3 100644 --- a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java @@ -28,16 +28,16 @@ public class SemanticTokensDataStreamProcessorTest { @Test public void testKeyword() throws InterruptedException, ExecutionException { - Document document = new Document(SemanticTokensUtil.keywordText); + Document document = new Document(SemanticTokensTestUtil.keywordText); - SemanticTokensDataStreamProcessor processor = new SemanticTokensDataStreamProcessor(SemanticTokensUtil - .keywordTokenTypeMapper(SemanticTokensUtil.RED_TOKEN), SemanticTokensUtil.offsetMapper(document)); + SemanticTokensDataStreamProcessor processor = new SemanticTokensDataStreamProcessor(SemanticTokensTestUtil + .keywordTokenTypeMapper(SemanticTokensTestUtil.RED_TOKEN), SemanticTokensTestUtil.offsetMapper(document)); - List expectedStream = SemanticTokensUtil.keywordSemanticTokens(); + List expectedStream = SemanticTokensTestUtil.keywordSemanticTokens(); List expectedStyleRanges = Arrays.asList(// - new StyleRange(0, 4, SemanticTokensUtil.RED, null), // - new StyleRange(15, 4, SemanticTokensUtil.RED, null), // - new StyleRange(24, 7, SemanticTokensUtil.RED, null)// + new StyleRange(0, 4, SemanticTokensTestUtil.RED, null), // + new StyleRange(15, 4, SemanticTokensTestUtil.RED, null), // + new StyleRange(24, 7, SemanticTokensTestUtil.RED, null)// ); List styleRanges = processor.getStyleRanges(expectedStream, getSemanticTokensLegend()); diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java index db17202c7..ae6ffc97d 100644 --- a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java @@ -47,7 +47,7 @@ public void testSemanticTokensLegendProvider() throws BadLocationException, Core // Setup Server Capabilities List tokenTypes = Arrays.asList("keyword","other"); List tokenModifiers = Arrays.asList("obsolete"); - SemanticTokensUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers); + SemanticTokensTestUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers); // Setup test data IFile file = TestUtils.createUniqueTestFile(project, "lspt", "test content"); diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensUtil.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensTestUtil.java similarity index 86% rename from org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensUtil.java rename to org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensTestUtil.java index 71d210bc4..5d603a72b 100644 --- a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensUtil.java +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensTestUtil.java @@ -1,3 +1,11 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ package org.eclipse.lsp4e.test.semanticTokens; import java.util.ArrayList; @@ -17,7 +25,7 @@ import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; import org.eclipse.swt.graphics.Color; -public class SemanticTokensUtil { +public class SemanticTokensTestUtil { public static final String keywordText = "type foo {\n" + " \n" + diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java index f7f6777f0..a892eed56 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java @@ -397,7 +397,7 @@ private static Collection getLSWrappers(@NonNull final ID } if (contentType.getBaseType() != null) { - contentTypesToProcess.add(contentType.getBaseType()); + contentTypesToProcess.add(contentType.getBaseType()); } processedContentTypes.add(contentType); } diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java index d79b0e718..3bd985c21 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java @@ -19,6 +19,7 @@ import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.jface.preference.IPreferenceStore; import org.eclipse.jface.text.BadLocationException; import org.eclipse.jface.text.IDocument; import org.eclipse.jface.text.IRegion; @@ -74,10 +75,15 @@ *

* For simplicity, out-dated responses are discarded, as we know we shall get * newer ones. + *

+ * In case the reconciler produces bogus results, it can be disabled with the key + * {@literal semanticHighlightReconciler.disabled} until fix is provided. */ public class SemanticHighlightReconcilerStrategy implements IReconcilingStrategy, IReconcilingStrategyExtension, ITextPresentationListener { + private final boolean disabled; + private ITextViewer viewer; private IDocument document; @@ -99,6 +105,11 @@ public class SemanticHighlightReconcilerStrategy private CompletableFuture semanticTokensFullFuture; + public SemanticHighlightReconcilerStrategy() { + IPreferenceStore store = LanguageServerPlugin.getDefault().getPreferenceStore(); + disabled = store.getBoolean("semanticHighlightReconciler.disabled"); //$NON-NLS-1$ + } + /** * Installs the reconciler on the given text viewer. After this method has been * finished, the reconciler is operational, i.e., it works without requesting @@ -108,6 +119,9 @@ public class SemanticHighlightReconcilerStrategy * the viewer on which the reconciler is installed */ public void install(@NonNull final ITextViewer textViewer) { + if (disabled) { + return; + } viewer = textViewer; styleRangeHolder = new StyleRangeHolder(); semanticTokensDataStreamProcessor = new SemanticTokensDataStreamProcessor(new TokenTypeMapper(textViewer), @@ -124,6 +138,9 @@ public void install(@NonNull final ITextViewer textViewer) { * on. */ public void uninstall() { + if (disabled) { + return; + } semanticTokensDataStreamProcessor = null; if (viewer instanceof TextViewer) { ((TextViewer) viewer).removeTextPresentationListener(this); @@ -214,6 +231,16 @@ private boolean isRequestCancelledException(final Throwable throwable) { .map(this::getSemanticTokensLegend).orElse(null); } + /** The presentation is invalidated if applyTextPresentation has never been called (e.g. there is + * no syntactic reconciler as in our unit tests) or the syntactic reconciler has already been applied + * for the given document. Otherwise the style rages will be applied when applyTextPresentation is + * called as part of the syntactic reconciliation. + */ + private boolean invalidateTextPresentation(final int version) { + return documentVersionAtLastAppliedTextPresentation == 0 + || documentVersionAtLastAppliedTextPresentation == version; + } + private CompletableFuture semanticTokensFull(final LanguageServer languageServer, final int version) { SemanticTokensParams semanticTokensParams = getSemanticTokensParams(); return languageServer.getTextDocumentService().semanticTokensFull(semanticTokensParams) @@ -222,7 +249,7 @@ private CompletableFuture semanticTokensFull(final LanguageServer language saveStyle(semanticTokens, getSemanticTokensLegend(languageServer)); StyledText textWidget = viewer.getTextWidget(); textWidget.getDisplay().asyncExec(() -> { - if (!textWidget.isDisposed() && (documentVersionAtLastAppliedTextPresentation == 0 || documentVersionAtLastAppliedTextPresentation == version )) { + if (!textWidget.isDisposed() && invalidateTextPresentation(version)) { viewer.invalidateTextPresentation(); } }); @@ -257,6 +284,9 @@ private void cancelSemanticTokensFull() { } private void fullReconcile() { + if (disabled) { + return; + } IDocument theDocument = document; cancelSemanticTokensFull(); if (theDocument != null) {