diff --git a/org.eclipse.lsp4e/META-INF/MANIFEST.MF b/org.eclipse.lsp4e/META-INF/MANIFEST.MF
index 3ca31d949..6c912da20 100644
--- a/org.eclipse.lsp4e/META-INF/MANIFEST.MF
+++ b/org.eclipse.lsp4e/META-INF/MANIFEST.MF
@@ -25,6 +25,7 @@ Require-Bundle: org.eclipse.core.runtime;bundle-version="3.12.0",
org.eclipse.debug.ui;bundle-version="3.11.200",
org.eclipse.swt,
org.eclipse.jdt.annotation;bundle-version="2.1.0";resolution:=optional,
+ org.eclipse.tm4e.ui,
org.eclipse.ui.editors,
org.eclipse.ui.navigator;bundle-version="3.6.100",
org.eclipse.lsp4j;bundle-version="[0.15.0,0.16.0)",
diff --git a/org.eclipse.lsp4e/plugin.xml b/org.eclipse.lsp4e/plugin.xml
index 2a5100923..d1c0e2392 100644
--- a/org.eclipse.lsp4e/plugin.xml
+++ b/org.eclipse.lsp4e/plugin.xml
@@ -663,4 +663,21 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java
new file mode 100644
index 000000000..3f9a1fb6c
--- /dev/null
+++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java
@@ -0,0 +1,28 @@
+package org.eclipse.lsp4e.operations.semanticTokens;
+
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.ITextViewer;
+import org.eclipse.jface.text.reconciler.MonoReconciler;
+
+
+public class SemanticHighlightReconciler extends MonoReconciler {
+
+ public SemanticHighlightReconciler() {
+ super(new SemanticHighlightReconcilerStrategy(), false);
+ }
+
+ @Override
+ public void install(final ITextViewer textViewer) {
+ super.install(textViewer);
+ // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted
+ ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).install(textViewer);
+ }
+
+ @Override
+ public void uninstall() {
+ super.uninstall();
+ // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted
+ ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).uninstall();
+ }
+
+}
\ No newline at end of file
diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java
new file mode 100644
index 000000000..3238ef375
--- /dev/null
+++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java
@@ -0,0 +1,357 @@
+package org.eclipse.lsp4e.operations.semanticTokens;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.jdt.annotation.Nullable;
+import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.IRegion;
+import org.eclipse.jface.text.ITextPresentationListener;
+import org.eclipse.jface.text.ITextViewer;
+import org.eclipse.jface.text.TextAttribute;
+import org.eclipse.jface.text.TextPresentation;
+import org.eclipse.jface.text.TextViewer;
+import org.eclipse.jface.text.reconciler.DirtyRegion;
+import org.eclipse.jface.text.reconciler.IReconcilingStrategy;
+import org.eclipse.jface.text.reconciler.IReconcilingStrategyExtension;
+import org.eclipse.jface.text.rules.IToken;
+import org.eclipse.lsp4e.LSPEclipseUtils;
+import org.eclipse.lsp4e.LanguageServerPlugin;
+import org.eclipse.lsp4e.LanguageServersRegistry.LanguageServerDefinition;
+import org.eclipse.lsp4e.LanguageServiceAccessor;
+import org.eclipse.lsp4j.Position;
+import org.eclipse.lsp4j.SemanticTokenModifiers;
+import org.eclipse.lsp4j.SemanticTokens;
+import org.eclipse.lsp4j.SemanticTokensLegend;
+import org.eclipse.lsp4j.SemanticTokensParams;
+import org.eclipse.lsp4j.ServerCapabilities;
+import org.eclipse.lsp4j.TextDocumentIdentifier;
+import org.eclipse.lsp4j.services.LanguageServer;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.StyleRange;
+import org.eclipse.tm4e.ui.internal.themes.ThemeManager;
+
+/**
+ * A reconciler strategy using semantic highlighting as defined by LSP.
+ */
+@SuppressWarnings("restriction")
+public class SemanticHighlightReconcilerStrategy
+ implements IReconcilingStrategy, IReconcilingStrategyExtension, ITextPresentationListener {
+
+ private @Nullable ITextViewer viewer;
+
+ private @Nullable ITheme theme;
+
+ private IDocument document;
+
+ private Map semanticTokensLegendMap;
+
+ private List previousRanges;
+
+ /**
+ * Installs the reconciler on the given text viewer. After this method has been
+ * finished, the reconciler is operational, i.e., it works without requesting
+ * further client actions until uninstall
is called.
+ *
+ * @param textViewer
+ * the viewer on which the reconciler is installed
+ */
+ public void install(final ITextViewer textViewer) {
+ viewer = textViewer;
+ theme = ThemeManager.getInstance().getDefaultTheme();
+ if (viewer instanceof TextViewer) {
+ ((TextViewer) viewer).addTextPresentationListener(this);
+ }
+ previousRanges = new ArrayList<>();
+ }
+
+ /**
+ * Removes the reconciler from the text viewer it has previously been installed
+ * on.
+ */
+ public void uninstall() {
+ theme = null;
+ if (viewer instanceof TextViewer) {
+ ((TextViewer) viewer).removeTextPresentationListener(this);
+ }
+ viewer = null;
+ previousRanges = null;
+ semanticTokensLegendMap = null;
+
+ }
+
+ private void initSemanticTokensLegendMap() {
+ IFile file = LSPEclipseUtils.getFile(document);
+ if (file != null) {
+ try {
+ semanticTokensLegendMap = LanguageServiceAccessor.getLSWrappers(file, x -> true).stream()//
+ .filter(wrapper -> wrapper.getServerCapabilities() != null)//
+ .collect(Collectors.toMap(wrapper -> wrapper.serverDefinition.id,
+ wrapper -> wrapper.getServerCapabilities().getSemanticTokensProvider().getLegend()));
+ } catch (IOException e) {
+ LanguageServerPlugin.logError(e);
+ }
+ }
+ }
+
+ private SemanticTokensParams getSemanticTokensParams() {
+ URI uri = LSPEclipseUtils.toUri(document);
+ if (uri != null) {
+ SemanticTokensParams semanticTokensParams = new SemanticTokensParams();
+ semanticTokensParams.setTextDocument(new TextDocumentIdentifier(uri.toString()));
+ return semanticTokensParams;
+ }
+ return null;
+ }
+
+ private void saveStyle(final SemanticTokens semanticTokens, final SemanticTokensLegend semanticTokensLegend) {
+ if (semanticTokens == null || semanticTokensLegend == null) {
+ return;
+ }
+ List dataStream = semanticTokens.getData();
+ if (!dataStream.isEmpty()) {
+ try {
+ List styleRanges = getStyleRanges(dataStream, semanticTokensLegend);
+ saveStyles(styleRanges);
+ } catch (BadLocationException e) {
+ LanguageServerPlugin.logError(e);
+ }
+ }
+ }
+
+ private StyleRange clone(final StyleRange styleRange) {
+ StyleRange clonedStyleRange = new StyleRange(styleRange.start, styleRange.length, styleRange.foreground,
+ styleRange.background, styleRange.fontStyle);
+ clonedStyleRange.strikeout = styleRange.strikeout;
+ return clonedStyleRange;
+ }
+
+ private void mergeStyles(final TextPresentation textPresentation, final List styleRanges) {
+ StyleRange[] array = new StyleRange[styleRanges.size()];
+ array = styleRanges.toArray(array);
+ textPresentation.replaceStyleRanges(array);
+ }
+
+ private boolean overlaps(final StyleRange range, final IRegion region) {
+ return isContained(range.start, region) || isContained(range.start + range.length, region)
+ || isContained(region.getOffset(), range);
+ }
+
+ private boolean isContained(final int offset, final StyleRange range) {
+ return offset >= range.start && offset < (range.start + range.length);
+ }
+
+ private boolean isContained(final int offset, final IRegion region) {
+ return offset >= region.getOffset() && offset < (region.getOffset() + region.getLength());
+ }
+
+ private void saveStyles(final List styleRanges) {
+ synchronized (previousRanges) {
+ previousRanges.clear();
+ previousRanges.addAll(styleRanges);
+ previousRanges.sort(Comparator.comparing(s -> s.start));
+ }
+ }
+
+ private List getStyleRanges(final List dataStream,
+ final SemanticTokensLegend semanticTokensLegend) throws BadLocationException {
+ List styleRanges = new ArrayList<>(dataStream.size() / 5);
+
+ int idx = 0;
+ int prevLine = 0;
+ int line = 0;
+ int offset = 0;
+ int length = 0;
+ String tokenType = null;
+ for (Integer data : dataStream) {
+ switch (idx % 5) {
+ case 0: // line
+ line += data;
+ break;
+ case 1: // offset
+ if (line == prevLine) {
+ offset += data;
+ } else {
+ offset = LSPEclipseUtils.toOffset(new Position(line, data), document);
+ }
+ break;
+ case 2: // length
+ length = data;
+ break;
+ case 3: // token type
+ tokenType = tokenType(data, semanticTokensLegend.getTokenTypes());
+ break;
+ case 4: // token modifier
+ prevLine = line;
+ List tokenModifiers = tokenModifiers(data, semanticTokensLegend.getTokenModifiers());
+ StyleRange styleRange = getStyleRange(offset, length, textAttribute(tokenType));
+ if (tokenModifiers.stream().anyMatch(x -> x.equals(SemanticTokenModifiers.Deprecated))) {
+ styleRange.strikeout = true;
+ }
+ styleRanges.add(styleRange);
+ break;
+ }
+ idx++;
+ }
+ return styleRanges;
+ }
+
+ private String tokenType(final Integer data, final List legend) {
+ try {
+ return legend.get(data - 1);
+ } catch (IndexOutOfBoundsException e) {
+ return null; // no match
+ }
+ }
+
+ private List tokenModifiers(final Integer data, final List legend) {
+ if (data.intValue() == 0) {
+ return Collections.emptyList();
+ }
+ BitSet bitSet = BitSet.valueOf(new long[] { data });
+ List tokenModifiers = new ArrayList<>();
+ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
+ try {
+ tokenModifiers.add(legend.get(i));
+ } catch (IndexOutOfBoundsException e) {
+ // no match
+ }
+ }
+
+ return tokenModifiers;
+ }
+
+ private TextAttribute textAttribute(final String tokenType) {
+ if (tokenType != null) {
+ IToken token = theme.getToken(tokenType);
+ if (token != null) {
+ Object data = token.getData();
+ if (data instanceof TextAttribute) {
+ return (TextAttribute) data;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Gets a style range for the given inputs.
+ *
+ * @param offset
+ * the offset of the range to be styled
+ * @param length
+ * the length of the range to be styled
+ * @param attr
+ * the attribute describing the style of the range to be styled
+ */
+ private StyleRange getStyleRange(final int offset, final int length, final TextAttribute attr) {
+ final StyleRange styleRange;
+ if (attr != null) {
+ final int style = attr.getStyle();
+ final int fontStyle = style & (SWT.ITALIC | SWT.BOLD | SWT.NORMAL);
+ styleRange = new StyleRange(offset, length, attr.getForeground(), attr.getBackground(), fontStyle);
+ styleRange.strikeout = (style & TextAttribute.STRIKETHROUGH) != 0;
+ styleRange.underline = (style & TextAttribute.UNDERLINE) != 0;
+ styleRange.font = attr.getFont();
+ return styleRange;
+ } else {
+ styleRange = new StyleRange();
+ styleRange.start = offset;
+ styleRange.length = length;
+ }
+ return styleRange;
+ }
+
+ @Override
+ public void setProgressMonitor(final IProgressMonitor monitor) {
+ }
+
+ @Override
+ public void setDocument(final IDocument document) {
+ this.document = document;
+ initSemanticTokensLegendMap();
+ }
+
+ private SemanticTokensLegend getSemanticTokensLegend(final LanguageServer languageSever) {
+ Optional serverDefinition = LanguageServiceAccessor
+ .resolveServerDefinition(languageSever);
+ if (serverDefinition.isPresent()) {
+ return semanticTokensLegendMap.get(serverDefinition.get().id);
+ }
+ return null;
+ }
+
+ private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilities) {
+ return serverCapabilities.getSemanticTokensProvider() != null
+ && serverCapabilities.getSemanticTokensProvider().getFull().getLeft();
+ }
+
+ private CompletableFuture semanticTokensFull(final List languageServers) {
+ return CompletableFuture
+ .allOf(languageServers.stream().map(this::semanticTokensFull).toArray(CompletableFuture[]::new));
+ }
+
+ private CompletableFuture semanticTokensFull(final LanguageServer languageServer) {
+ SemanticTokensParams semanticTokensParams = getSemanticTokensParams();
+ return languageServer.getTextDocumentService().semanticTokensFull(semanticTokensParams)
+ .thenAccept(semanticTokens -> {
+ saveStyle(semanticTokens, getSemanticTokensLegend(languageServer));
+ }).exceptionally(e -> {
+ LanguageServerPlugin.logError(e);
+ return null;
+ });
+ }
+
+ private void fullReconcile() {
+ try {
+ LanguageServiceAccessor.getLanguageServers(document, this::hasSemanticTokensFull)//
+ .thenAccept(this::semanticTokensFull).get();
+ } catch (InterruptedException | ExecutionException e) {
+ LanguageServerPlugin.logError(e);
+ }
+ }
+
+ @Override
+ public void initialReconcile() {
+ fullReconcile();
+ }
+
+ @Override
+ public void reconcile(final DirtyRegion dirtyRegion, final IRegion subRegion) {
+ fullReconcile();
+ }
+
+ @Override
+ public void reconcile(final IRegion partition) {
+ fullReconcile();
+ }
+
+ private List appliedRanges(final TextPresentation textPresentation) {
+ synchronized (previousRanges) {
+ // we need to create new styles because the text presentation might change a
+ // style when applied to the presentation
+ // and we want the ones saved from the reconciling as immutable
+ return previousRanges.stream()//
+ .filter(r -> overlaps(r, textPresentation.getExtent()))//
+ .map(this::clone).collect(Collectors.toList());
+ }
+ }
+
+ @Override
+ public void applyTextPresentation(final TextPresentation textPresentation) {
+ mergeStyles(textPresentation, appliedRanges(textPresentation));
+ }
+}
\ No newline at end of file
diff --git a/target-platforms/target-platform-latest/target-platform-latest.target b/target-platforms/target-platform-latest/target-platform-latest.target
index 1ad1d7123..5a53e3255 100644
--- a/target-platforms/target-platform-latest/target-platform-latest.target
+++ b/target-platforms/target-platform-latest/target-platform-latest.target
@@ -30,6 +30,10 @@
+
+
+
+
\ No newline at end of file