diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java
index 076344ec4..0a542d6c8 100644
--- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java
+++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * Copyright (c) 2022 Avaloq Evolution AG.
+ * Copyright (c) 2022 Avaloq Group AG.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
@@ -12,25 +12,27 @@
import org.eclipse.jface.text.ITextViewer;
import org.eclipse.jface.text.reconciler.MonoReconciler;
-
public class SemanticHighlightReconciler extends MonoReconciler {
- public SemanticHighlightReconciler() {
- super(new SemanticHighlightReconcilerStrategy(), false);
- }
+ public SemanticHighlightReconciler() {
+ super(new SemanticHighlightReconcilerStrategy(), false);
+ }
- @Override
- public void install(final ITextViewer textViewer) {
- super.install(textViewer);
- // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted
- ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).install(textViewer);
- }
+ @Override
+ public void install(final ITextViewer textViewer) {
+ super.install(textViewer);
+ // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is
+ // accepted
+ ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE))
+ .install(textViewer);
+ }
- @Override
- public void uninstall() {
- super.uninstall();
- // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted
- ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).uninstall();
- }
+ @Override
+ public void uninstall() {
+ super.uninstall();
+ // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is
+ // accepted
+ ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).uninstall();
+ }
}
\ No newline at end of file
diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java
index fd1d4e730..46331ae96 100644
--- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java
+++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java
@@ -8,69 +8,95 @@
*******************************************************************************/
package org.eclipse.lsp4e.operations.semanticTokens;
-import java.io.IOException;
import java.net.URI;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
-import java.util.Map;
-import java.util.Optional;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
-import java.util.stream.Collectors;
+import java.util.function.Function;
-import org.eclipse.core.resources.IFile;
import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.jdt.annotation.Nullable;
+import org.eclipse.jdt.annotation.NonNull;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.IRegion;
+import org.eclipse.jface.text.ITextListener;
import org.eclipse.jface.text.ITextPresentationListener;
import org.eclipse.jface.text.ITextViewer;
-import org.eclipse.jface.text.TextAttribute;
import org.eclipse.jface.text.TextPresentation;
import org.eclipse.jface.text.TextViewer;
import org.eclipse.jface.text.reconciler.DirtyRegion;
import org.eclipse.jface.text.reconciler.IReconcilingStrategy;
import org.eclipse.jface.text.reconciler.IReconcilingStrategyExtension;
-import org.eclipse.jface.text.rules.IToken;
import org.eclipse.lsp4e.LSPEclipseUtils;
import org.eclipse.lsp4e.LanguageServerPlugin;
-import org.eclipse.lsp4e.LanguageServerWrapper;
-import org.eclipse.lsp4e.LanguageServersRegistry.LanguageServerDefinition;
import org.eclipse.lsp4e.LanguageServiceAccessor;
+import org.eclipse.lsp4e.LanguageServiceAccessor.LSPDocumentInfo;
import org.eclipse.lsp4j.Position;
-import org.eclipse.lsp4j.SemanticTokenModifiers;
import org.eclipse.lsp4j.SemanticTokens;
import org.eclipse.lsp4j.SemanticTokensLegend;
import org.eclipse.lsp4j.SemanticTokensParams;
-import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions;
import org.eclipse.lsp4j.ServerCapabilities;
import org.eclipse.lsp4j.TextDocumentIdentifier;
+import org.eclipse.lsp4j.jsonrpc.ResponseErrorException;
+import org.eclipse.lsp4j.jsonrpc.messages.ResponseError;
+import org.eclipse.lsp4j.jsonrpc.messages.ResponseErrorCode;
import org.eclipse.lsp4j.services.LanguageServer;
-import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.StyleRange;
-import org.eclipse.tm4e.ui.TMUIPlugin;
-import org.eclipse.tm4e.ui.themes.ITheme;
+import org.eclipse.swt.custom.StyledText;
/**
- * A reconciler strategy using semantic highlighting as defined by LSP.
+ * A semantic reconciler strategy using LSP.
+ *
+ * This strategy applies semantic highlight on top of the syntactic highlight
+ * provided by TM4E by implementing {@link ITextPresentationListener}. Because
+ * semantic highlight involves remote calls, it is expected to be slower than
+ * syntactic highlight. Thus we execute semantic highlight processing in a
+ * separate reconciler thread that will never block TM4E.
+ *
+ * This strategy records the version of the document when the semantic highlight
+ * is sent to the LS and when TM4E highlight is applied. If the response for a
+ * particular document version comes after the TM4E highlight has been applied,
+ * the text presentation is invalidated so that highlight is extended with the
+ * results.
+ *
+ * To avoid flickering, {@link StyleRangeHolder} implement {@link ITextListener}
+ * to adapt recorded semantic highlights and apply those instead of nothing
+ * where needed.
+ *
+ * If the response comes before, the data is saved and applied later on top of
+ * the presentation provided by TM4E. The results from our reconciler are
+ * recorded
+ *
+ * For simplicity, out-dated responses are discarded, as we know we shall get
+ * newer ones.
*/
public class SemanticHighlightReconcilerStrategy
implements IReconcilingStrategy, IReconcilingStrategyExtension, ITextPresentationListener {
- private @Nullable ITextViewer viewer;
-
- private @Nullable ITheme theme;
+ private ITextViewer viewer;
private IDocument document;
- private Map semanticTokensLegendMap;
+ private StyleRangeHolder styleRangeHolder;
+
+ private SemanticTokensDataStreamProcessor semanticTokensDataStreamProcessor;
+
+ private SemanticTokensLegendProvider semanticTokensLegendProvider;
+
+ /**
+ * Written in {@link this.class#applyTextPresentation(TextPresentation)}
+ * applyTextPresentation and read in the lambda in
+ * {@link this.class#semanticTokensFull(LanguageServer, int)}, the lambda and
+ * the former method are executed in the display thread, thus serializing access
+ * using volatile without using explicit synchronized blocks is enough to avoid
+ * that org.eclipse.jface.text.ITextViewer.invalidateTextPresentation() is
+ * called by use while the presentation is being updated.
+ */
+ private volatile int documentVersionAtLastAppliedTextPresentation;
- private List previousRanges;
+ private CompletableFuture semanticTokensFullFuture;
/**
* Installs the reconciler on the given text viewer. After this method has been
@@ -82,11 +108,15 @@ public class SemanticHighlightReconcilerStrategy
*/
public void install(final ITextViewer textViewer) {
viewer = textViewer;
- theme = TMUIPlugin.getThemeManager().getDefaultTheme();
- if (textViewer instanceof TextViewer viewer) {
- viewer.addTextPresentationListener(this);
+ styleRangeHolder = new StyleRangeHolder();
+ semanticTokensDataStreamProcessor = new SemanticTokensDataStreamProcessor(new TokenTypeMapper(viewer),
+ offsetMapper());
+ semanticTokensLegendProvider = new SemanticTokensLegendProvider();
+
+ if (viewer instanceof TextViewer) {
+ ((TextViewer) viewer).addTextPresentationListener(this);
}
- previousRanges = new ArrayList<>();
+ viewer.addTextListener(styleRangeHolder);
}
/**
@@ -94,35 +124,24 @@ public void install(final ITextViewer textViewer) {
* on.
*/
public void uninstall() {
- theme = null;
- ITextViewer textViewer = viewer;
- if (textViewer instanceof TextViewer viewer) {
- viewer.removeTextPresentationListener(this);
+ semanticTokensDataStreamProcessor = null;
+ if (viewer instanceof TextViewer) {
+ ((TextViewer) viewer).removeTextPresentationListener(this);
}
+ viewer.removeTextListener(styleRangeHolder);
viewer = null;
- previousRanges = null;
- semanticTokensLegendMap = null;
-
+ styleRangeHolder = null;
+ semanticTokensLegendProvider = null;
}
- private void initSemanticTokensLegendMap() {
- IFile file = LSPEclipseUtils.getFile(document);
- if (file != null) {
+ private Function offsetMapper() {
+ return (p) -> {
try {
- semanticTokensLegendMap = new HashMap<>();
- for (LanguageServerWrapper wrapper: LanguageServiceAccessor.getLSWrappers(file, x -> true)) {
- ServerCapabilities serverCapabilities = wrapper.getServerCapabilities();
- if (serverCapabilities != null) {
- SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities.getSemanticTokensProvider();
- if (semanticTokensProvider != null) {
- semanticTokensLegendMap.put(wrapper.serverDefinition.id, semanticTokensProvider.getLegend());
- }
- }
- }
- } catch (IOException e) {
- LanguageServerPlugin.logError(e);
+ return LSPEclipseUtils.toOffset(p, document);
+ } catch (BadLocationException e) {
+ throw new RuntimeException(e);
}
- }
+ };
}
private SemanticTokensParams getSemanticTokensParams() {
@@ -141,159 +160,12 @@ private void saveStyle(final SemanticTokens semanticTokens, final SemanticTokens
}
List dataStream = semanticTokens.getData();
if (!dataStream.isEmpty()) {
- try {
- List styleRanges = getStyleRanges(dataStream, semanticTokensLegend);
- saveStyles(styleRanges);
- } catch (BadLocationException e) {
- LanguageServerPlugin.logError(e);
- }
+ List styleRanges = semanticTokensDataStreamProcessor.getStyleRanges(dataStream,
+ semanticTokensLegend);
+ styleRangeHolder.saveStyles(styleRanges);
}
}
- private StyleRange clone(final StyleRange styleRange) {
- StyleRange clonedStyleRange = new StyleRange(styleRange.start, styleRange.length, styleRange.foreground,
- styleRange.background, styleRange.fontStyle);
- clonedStyleRange.strikeout = styleRange.strikeout;
- return clonedStyleRange;
- }
-
- private void mergeStyles(final TextPresentation textPresentation, final List styleRanges) {
- StyleRange[] array = new StyleRange[styleRanges.size()];
- array = styleRanges.toArray(array);
- textPresentation.replaceStyleRanges(array);
- }
-
- private boolean overlaps(final StyleRange range, final IRegion region) {
- return isContained(range.start, region) || isContained(range.start + range.length, region)
- || isContained(region.getOffset(), range);
- }
-
- private boolean isContained(final int offset, final StyleRange range) {
- return offset >= range.start && offset < (range.start + range.length);
- }
-
- private boolean isContained(final int offset, final IRegion region) {
- return offset >= region.getOffset() && offset < (region.getOffset() + region.getLength());
- }
-
- private void saveStyles(final List styleRanges) {
- synchronized (previousRanges) {
- previousRanges.clear();
- previousRanges.addAll(styleRanges);
- previousRanges.sort(Comparator.comparing(s -> s.start));
- }
- }
-
- private List getStyleRanges(final List dataStream,
- final SemanticTokensLegend semanticTokensLegend) throws BadLocationException {
- List styleRanges = new ArrayList<>(dataStream.size() / 5);
-
- int idx = 0;
- int prevLine = 0;
- int line = 0;
- int offset = 0;
- int length = 0;
- String tokenType = null;
- for (Integer data : dataStream) {
- switch (idx % 5) {
- case 0: // line
- line += data;
- break;
- case 1: // offset
- if (line == prevLine) {
- offset += data;
- } else {
- offset = LSPEclipseUtils.toOffset(new Position(line, data), document);
- }
- break;
- case 2: // length
- length = data;
- break;
- case 3: // token type
- tokenType = tokenType(data, semanticTokensLegend.getTokenTypes());
- break;
- case 4: // token modifier
- prevLine = line;
- List tokenModifiers = tokenModifiers(data, semanticTokensLegend.getTokenModifiers());
- StyleRange styleRange = getStyleRange(offset, length, textAttribute(tokenType));
- if (tokenModifiers.stream().anyMatch(x -> x.equals(SemanticTokenModifiers.Deprecated))) {
- styleRange.strikeout = true;
- }
- styleRanges.add(styleRange);
- break;
- }
- idx++;
- }
- return styleRanges;
- }
-
- private String tokenType(final Integer data, final List legend) {
- try {
- return legend.get(data - 1);
- } catch (IndexOutOfBoundsException e) {
- return null; // no match
- }
- }
-
- private List tokenModifiers(final Integer data, final List legend) {
- if (data.intValue() == 0) {
- return Collections.emptyList();
- }
- BitSet bitSet = BitSet.valueOf(new long[] { data });
- List tokenModifiers = new ArrayList<>();
- for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- try {
- tokenModifiers.add(legend.get(i));
- } catch (IndexOutOfBoundsException e) {
- // no match
- }
- }
-
- return tokenModifiers;
- }
-
- private TextAttribute textAttribute(final String tokenType) {
- ITheme localTheme = theme;
- if (localTheme != null && tokenType != null) {
- IToken token = localTheme.getToken(tokenType);
- if (token != null) {
- Object data = token.getData();
- if (data instanceof TextAttribute textAttribute) {
- return textAttribute;
- }
- }
- }
- return null;
- }
-
- /**
- * Gets a style range for the given inputs.
- *
- * @param offset
- * the offset of the range to be styled
- * @param length
- * the length of the range to be styled
- * @param attr
- * the attribute describing the style of the range to be styled
- */
- private StyleRange getStyleRange(final int offset, final int length, final TextAttribute attr) {
- final StyleRange styleRange;
- if (attr != null) {
- final int style = attr.getStyle();
- final int fontStyle = style & (SWT.ITALIC | SWT.BOLD | SWT.NORMAL);
- styleRange = new StyleRange(offset, length, attr.getForeground(), attr.getBackground(), fontStyle);
- styleRange.strikeout = (style & TextAttribute.STRIKETHROUGH) != 0;
- styleRange.underline = (style & TextAttribute.UNDERLINE) != 0;
- styleRange.font = attr.getFont();
- return styleRange;
- } else {
- styleRange = new StyleRange();
- styleRange.start = offset;
- styleRange.length = length;
- }
- return styleRange;
- }
-
@Override
public void setProgressMonitor(final IProgressMonitor monitor) {
}
@@ -301,16 +173,7 @@ public void setProgressMonitor(final IProgressMonitor monitor) {
@Override
public void setDocument(final IDocument document) {
this.document = document;
- initSemanticTokensLegendMap();
- }
-
- private SemanticTokensLegend getSemanticTokensLegend(final LanguageServer languageSever) {
- Optional serverDefinition = LanguageServiceAccessor
- .resolveServerDefinition(languageSever);
- if (serverDefinition.isPresent()) {
- return semanticTokensLegendMap.get(serverDefinition.get().id);
- }
- return null;
+ semanticTokensLegendProvider.setDocument(document);
}
private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilities) {
@@ -318,26 +181,66 @@ private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilitie
&& serverCapabilities.getSemanticTokensProvider().getFull().getLeft();
}
- private CompletableFuture semanticTokensFull(final List languageServers) {
- return CompletableFuture
- .allOf(languageServers.stream().map(this::semanticTokensFull).toArray(CompletableFuture[]::new));
+ private CompletableFuture semanticTokensFull(final List languageServers, final int version) {
+ return CompletableFuture.allOf(
+ languageServers.stream().map(ls -> semanticTokensFull(ls, version)).toArray(CompletableFuture[]::new));
}
- private CompletableFuture semanticTokensFull(final LanguageServer languageServer) {
+ private boolean isRequestCancelledException(final Throwable throwable) {
+ if (throwable instanceof CompletionException) {
+ Throwable cause = ((CompletionException) throwable).getCause();
+ if (cause instanceof ResponseErrorException) {
+ ResponseError responseError = ((ResponseErrorException) cause).getResponseError();
+ return responseError != null
+ && responseError.getCode() == ResponseErrorCode.RequestCancelled.getValue();
+ }
+ }
+ return false;
+ }
+
+ private CompletableFuture semanticTokensFull(final LanguageServer languageServer, final int version) {
SemanticTokensParams semanticTokensParams = getSemanticTokensParams();
return languageServer.getTextDocumentService().semanticTokensFull(semanticTokensParams)
.thenAccept(semanticTokens -> {
- saveStyle(semanticTokens, getSemanticTokensLegend(languageServer));
+ if (getDocumentVersion() == version) {
+ saveStyle(semanticTokens, semanticTokensLegendProvider.getSemanticTokensLegend(languageServer));
+ StyledText textWidget = viewer.getTextWidget();
+ textWidget.getDisplay().asyncExec(() -> {
+ if (!textWidget.isDisposed() && documentVersionAtLastAppliedTextPresentation == version) {
+ viewer.invalidateTextPresentation();
+ }
+ });
+ }
}).exceptionally(e -> {
- LanguageServerPlugin.logError(e);
+ if (!isRequestCancelledException(e)) {
+ LanguageServerPlugin.logError(e);
+ }
return null;
});
}
+ private int getDocumentVersion() {
+ Iterator<@NonNull LSPDocumentInfo> iterator = LanguageServiceAccessor
+ .getLSPDocumentInfosFor(document, this::hasSemanticTokensFull).iterator();
+ if (iterator.hasNext()) {
+ return iterator.next().getVersion();
+ }
+ return -1;
+ }
+
+ private void cancelSemanticTokensFull() {
+ if (semanticTokensFullFuture != null) {
+ semanticTokensFullFuture.cancel(true);
+ }
+ }
+
private void fullReconcile() {
+ cancelSemanticTokensFull();
try {
- LanguageServiceAccessor.getLanguageServers(document, this::hasSemanticTokensFull)//
- .thenAccept(this::semanticTokensFull).get();
+ int version = getDocumentVersion();
+ semanticTokensFullFuture = LanguageServiceAccessor.getLanguageServers(document, this::hasSemanticTokensFull)//
+ .thenAccept(ls -> semanticTokensFull(ls, version));
+ semanticTokensFullFuture.get();
} catch (InterruptedException | ExecutionException e) {
LanguageServerPlugin.logError(e);
}
@@ -358,19 +261,9 @@ public void reconcile(final IRegion partition) {
fullReconcile();
}
- private List appliedRanges(final TextPresentation textPresentation) {
- synchronized (previousRanges) {
- // we need to create new styles because the text presentation might change a
- // style when applied to the presentation
- // and we want the ones saved from the reconciling as immutable
- return previousRanges.stream()//
- .filter(r -> overlaps(r, textPresentation.getExtent()))//
- .map(this::clone).collect(Collectors.toList());
- }
- }
-
@Override
public void applyTextPresentation(final TextPresentation textPresentation) {
- mergeStyles(textPresentation, appliedRanges(textPresentation));
+ documentVersionAtLastAppliedTextPresentation = getDocumentVersion();
+ textPresentation.replaceStyleRanges(styleRangeHolder.overlappingRanges(textPresentation.getExtent()));
}
-}
\ No newline at end of file
+}
diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java
new file mode 100644
index 000000000..efad1f0cc
--- /dev/null
+++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java
@@ -0,0 +1,163 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Avaloq Group AG.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ *******************************************************************************/
+package org.eclipse.lsp4e.operations.semanticTokens;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+
+import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.jface.text.TextAttribute;
+import org.eclipse.jface.text.rules.IToken;
+import org.eclipse.lsp4j.Position;
+import org.eclipse.lsp4j.SemanticTokenModifiers;
+import org.eclipse.lsp4j.SemanticTokensLegend;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.StyleRange;
+
+/**
+ * The Class SemanticTokensDataStreamProcessor translates a stream of integers
+ * as defined by the LSP SemanticTokenRequests into a list of StyleRanges.
+ */
+public class SemanticTokensDataStreamProcessor {
+
+ private final Function offsetMapper;
+ private final Function tokenTypeMapper;
+
+ /**
+ * Creates a new instance of {@link SemanticTokensDataStreamProcessor}.
+ *
+ * @param tokenTypeMapper
+ * @param offsetMapper
+ */
+ public SemanticTokensDataStreamProcessor(@NonNull final Function tokenTypeMapper,
+ @NonNull final Function offsetMapper) {
+ this.tokenTypeMapper = tokenTypeMapper;
+ this.offsetMapper = offsetMapper;
+ }
+
+ /**
+ * Get the StyleRanges for the given data stream and tokens legend.
+ *
+ * @param dataStream
+ * @param semanticTokensLegend
+ * @return
+ */
+ public @NonNull List getStyleRanges(@NonNull final List dataStream,
+ @NonNull final SemanticTokensLegend semanticTokensLegend) {
+ List styleRanges = new ArrayList<>(dataStream.size() / 5);
+
+ int idx = 0;
+ int prevLine = 0;
+ int line = 0;
+ int offset = 0;
+ int length = 0;
+ String tokenType = null;
+ for (Integer data : dataStream) {
+ switch (idx % 5) {
+ case 0: // line
+ line += data;
+ break;
+ case 1: // offset
+ if (line == prevLine) {
+ offset += data;
+ } else {
+ offset = offsetMapper.apply(new Position(line, data));
+ }
+ break;
+ case 2: // length
+ length = data;
+ break;
+ case 3: // token type
+ tokenType = tokenType(data, semanticTokensLegend.getTokenTypes());
+ break;
+ case 4: // token modifier
+ prevLine = line;
+ List tokenModifiers = tokenModifiers(data, semanticTokensLegend.getTokenModifiers());
+ StyleRange styleRange = getStyleRange(offset, length, textAttribute(tokenType));
+ if (tokenModifiers.stream().anyMatch(x -> x.equals(SemanticTokenModifiers.Deprecated))) {
+ styleRange.strikeout = true;
+ }
+ styleRanges.add(styleRange);
+ break;
+ }
+ idx++;
+ }
+ return styleRanges;
+ }
+
+ private String tokenType(final Integer data, final List legend) {
+ try {
+ return legend.get(data - 1);
+ } catch (IndexOutOfBoundsException e) {
+ return null; // no match
+ }
+ }
+
+ private List tokenModifiers(final Integer data, final List legend) {
+ if (data.intValue() == 0) {
+ return Collections.emptyList();
+ }
+ BitSet bitSet = BitSet.valueOf(new long[] { data });
+ List tokenModifiers = new ArrayList<>();
+ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
+ try {
+ tokenModifiers.add(legend.get(i));
+ } catch (IndexOutOfBoundsException e) {
+ // no match
+ }
+ }
+
+ return tokenModifiers;
+ }
+
+ private TextAttribute textAttribute(final String tokenType) {
+ if (tokenType != null) {
+ IToken token = tokenTypeMapper.apply(tokenType);
+ if (token != null) {
+ Object data = token.getData();
+ if (data instanceof TextAttribute) {
+ return (TextAttribute) data;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Gets a style range for the given inputs.
+ *
+ * @param offset
+ * the offset of the range to be styled
+ * @param length
+ * the length of the range to be styled
+ * @param attr
+ * the attribute describing the style of the range to be styled
+ */
+ private StyleRange getStyleRange(final int offset, final int length, final TextAttribute attr) {
+ final StyleRange styleRange;
+ if (attr != null) {
+ final int style = attr.getStyle();
+ final int fontStyle = style & (SWT.ITALIC | SWT.BOLD | SWT.NORMAL);
+ styleRange = new StyleRange(offset, length, attr.getForeground(), attr.getBackground(), fontStyle);
+ styleRange.strikeout = (style & TextAttribute.STRIKETHROUGH) != 0;
+ styleRange.underline = (style & TextAttribute.UNDERLINE) != 0;
+ styleRange.font = attr.getFont();
+ return styleRange;
+ } else {
+ styleRange = new StyleRange();
+ styleRange.start = offset;
+ styleRange.length = length;
+ }
+ return styleRange;
+ }
+}
\ No newline at end of file
diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java
new file mode 100644
index 000000000..da662c2b3
--- /dev/null
+++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensLegendProvider.java
@@ -0,0 +1,89 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Avaloq Group AG.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ *******************************************************************************/
+package org.eclipse.lsp4e.operations.semanticTokens;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.lsp4e.LSPEclipseUtils;
+import org.eclipse.lsp4e.LanguageServerPlugin;
+import org.eclipse.lsp4e.LanguageServerWrapper;
+import org.eclipse.lsp4e.LanguageServersRegistry.LanguageServerDefinition;
+import org.eclipse.lsp4e.LanguageServiceAccessor;
+import org.eclipse.lsp4j.SemanticTokensLegend;
+import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions;
+import org.eclipse.lsp4j.ServerCapabilities;
+import org.eclipse.lsp4j.services.LanguageServer;
+
+
+/**
+ * A provider for {@link SemanticTokensLegend}.
+ */
+public class SemanticTokensLegendProvider {
+private Map semanticTokensLegendMap;
+private IDocument document;
+
+/**
+ * Tells the provider strategy on which document it will
+ * work.
+ *
+ * @param document
+ * the document on which this mapper will work
+ */
+public void setDocument(final IDocument document) {
+ this.document = document;
+}
+
+private void initSemanticTokensLegendMap() {
+ IFile file = LSPEclipseUtils.getFile(document);
+ if (file != null) {
+ try {
+ semanticTokensLegendMap = new HashMap<>();
+ for (LanguageServerWrapper wrapper: LanguageServiceAccessor.getLSWrappers(file, x -> true)) {
+ ServerCapabilities serverCapabilities = wrapper.getServerCapabilities();
+ if (serverCapabilities != null) {
+ SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities.getSemanticTokensProvider();
+ if (semanticTokensProvider != null) {
+ semanticTokensLegendMap.put(wrapper.serverDefinition.id, semanticTokensProvider.getLegend());
+ }
+ }
+ }
+ } catch (IOException e) {
+ semanticTokensLegendMap = Collections.emptyMap();
+ LanguageServerPlugin.logError(e);
+ }
+ } else {
+ semanticTokensLegendMap = Collections.emptyMap();
+ }
+}
+
+/**
+ * Gets the {@link SemanticTokensLegend} for the given {@link LanguageServer}.
+ *
+ * @param languageSever
+ * @return
+ */
+public SemanticTokensLegend getSemanticTokensLegend(@NonNull final LanguageServer languageSever) {
+ Optional serverDefinition = LanguageServiceAccessor.resolveServerDefinition(languageSever);
+ if (serverDefinition.isPresent()) {
+ if (semanticTokensLegendMap == null) {
+ initSemanticTokensLegendMap();
+ }
+ return semanticTokensLegendMap.get(serverDefinition.get().id);
+ }
+ return null;
+}
+}
diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java
new file mode 100644
index 000000000..d03b4a04d
--- /dev/null
+++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java
@@ -0,0 +1,103 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Avaloq Group AG.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ *******************************************************************************/
+package org.eclipse.lsp4e.operations.semanticTokens;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.jface.text.IRegion;
+import org.eclipse.jface.text.ITextListener;
+import org.eclipse.jface.text.Region;
+import org.eclipse.jface.text.TextEvent;
+import org.eclipse.swt.custom.StyleRange;
+
+/**
+ * The Class SemanticTokensDataStreamProcessor holds a list of StyleRanges.
+ *
+ * To avoid flickering, we also implement {@link ITextListener} to adapt (the
+ * only adaptation currently supported shifting ranges) recorded semantic
+ * highlights When the user writes a single or multi-line comments shifting is
+ * not enough. That could be improved if we can access
+ * org.eclipse.tm4e.languageconfiguration.ILanguageConfiguration.getComments()
+ * (still unclear on how to do that).
+ */
+public class StyleRangeHolder implements ITextListener {
+ private final List previousRanges;
+
+ public StyleRangeHolder() {
+ previousRanges = new ArrayList<>();
+ }
+
+ /**
+ * save the styles.
+ *
+ * @param styleRanges
+ */
+ public void saveStyles(@NonNull final List styleRanges) {
+ synchronized (previousRanges) {
+ previousRanges.clear();
+ previousRanges.addAll(styleRanges);
+ previousRanges.sort(Comparator.comparing(s -> s.start));
+ }
+ }
+
+ /**
+ * return a copy of the saved styles that overlap the given region.
+ *
+ * @param region
+ * @return
+ */
+ public StyleRange[] overlappingRanges(@NonNull final IRegion region) {
+ synchronized (previousRanges) {
+ // we need to create new styles because the text presentation might change a
+ // style when applied to the presentation
+ // and we want the ones saved from the reconciling as immutable
+ return previousRanges.stream()//
+ .filter(r -> overlaps(r, region))//
+ .map(this::clone).toArray(StyleRange[]::new);
+ }
+ }
+
+ private StyleRange clone(final StyleRange styleRange) {
+ StyleRange clonedStyleRange = new StyleRange(styleRange.start, styleRange.length, styleRange.foreground,
+ styleRange.background, styleRange.fontStyle);
+ clonedStyleRange.strikeout = styleRange.strikeout;
+ return clonedStyleRange;
+ }
+
+ private boolean isContained(final int offset, final StyleRange range) {
+ return offset >= range.start && offset < (range.start + range.length);
+ }
+
+ private boolean isContained(final int offset, final IRegion region) {
+ return offset >= region.getOffset() && offset < (region.getOffset() + region.getLength());
+ }
+
+ private boolean overlaps(final StyleRange range, final IRegion region) {
+ return isContained(range.start, region) || isContained(range.start + range.length, region)
+ || isContained(region.getOffset(), range);
+ }
+
+ @Override
+ public void textChanged(final TextEvent event) {
+ if (event.getDocumentEvent() != null) { // if null, it is an internal event, not a changed text
+ String replacedText = event.getReplacedText();
+ String text = event.getText();
+ int delta = (text != null ? text.length() : 0) - (replacedText != null ? replacedText.length() : 0);
+ synchronized (previousRanges) {
+ previousRanges.removeIf(r -> isContained(event.getOffset(), new Region(r.start, r.length)));
+ previousRanges.stream().filter(r -> r.start >= event.getOffset()).forEach(r -> r.start += delta);
+ }
+ }
+ }
+
+}
diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java
new file mode 100644
index 000000000..bf7b2900c
--- /dev/null
+++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java
@@ -0,0 +1,44 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Avaloq Group AG.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ *******************************************************************************/
+package org.eclipse.lsp4e.operations.semanticTokens;
+
+import java.util.function.Function;
+
+import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.jface.text.ITextViewer;
+import org.eclipse.jface.text.rules.IToken;
+import org.eclipse.tm4e.ui.TMUIPlugin;
+import org.eclipse.tm4e.ui.text.TMPresentationReconciler;
+import org.eclipse.tm4e.ui.themes.ITokenProvider;
+
+/**
+ * A Class that maps TokenTypes to {@link IToken}.
+ */
+public class TokenTypeMapper implements Function {
+ private @NonNull final ITextViewer viewer;
+
+ public TokenTypeMapper(final ITextViewer viewer) {
+ this.viewer = viewer;
+ }
+
+ @Override
+ public IToken apply(final String tokenType) {
+ TMPresentationReconciler tmPresentationReconciler = TMPresentationReconciler
+ .getTMPresentationReconciler(viewer);
+
+ if (tmPresentationReconciler != null) {
+ ITokenProvider tokenProvider = tmPresentationReconciler.getTokenProvider();
+ if (tokenProvider != null) {
+ tokenProvider.getToken(tokenType);
+ }
+ }
+ return TMUIPlugin.getThemeManager().getDefaultTheme().getToken(tokenType);
+ }
+}