From 6816201f607314f3bf3d3d04e864c733b17b6678 Mon Sep 17 00:00:00 2001 From: pcr Date: Wed, 5 Oct 2022 10:55:26 +0200 Subject: [PATCH] Initial implementation of a semantic tokens reconciler Initial implementation of a semantic tokens reconciler which supports only SemanticTokensFull. The reconciler uses the theme defined for TM4E so that the same styles are used for the same token types. In addition to the tokens defined for TM4E, also the deprecated modifier is supported by marking striking out the regions. --- org.eclipse.lsp4e/META-INF/MANIFEST.MF | 1 + org.eclipse.lsp4e/plugin.xml | 17 + .../SemanticHighlightReconciler.java | 36 ++ .../SemanticHighlightReconcilerStrategy.java | 376 ++++++++++++++++++ .../target-platform-latest.target | 4 + 5 files changed, 434 insertions(+) create mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java create mode 100644 org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java diff --git a/org.eclipse.lsp4e/META-INF/MANIFEST.MF b/org.eclipse.lsp4e/META-INF/MANIFEST.MF index 0ec8bee99..1dd54dfa8 100644 --- a/org.eclipse.lsp4e/META-INF/MANIFEST.MF +++ b/org.eclipse.lsp4e/META-INF/MANIFEST.MF @@ -25,6 +25,7 @@ Require-Bundle: org.eclipse.core.runtime;bundle-version="3.12.0", org.eclipse.debug.ui;bundle-version="3.11.200", org.eclipse.swt, org.eclipse.jdt.annotation;bundle-version="2.1.0";resolution:=optional, + org.eclipse.tm4e.ui, org.eclipse.ui.editors, org.eclipse.ui.navigator;bundle-version="3.6.100", org.eclipse.lsp4j;bundle-version="[0.18.0,0.19.0)", diff --git a/org.eclipse.lsp4e/plugin.xml b/org.eclipse.lsp4e/plugin.xml index fad248dce..67fdadd1c 100644 --- a/org.eclipse.lsp4e/plugin.xml +++ b/org.eclipse.lsp4e/plugin.xml @@ -657,4 +657,21 @@ + + + + + + + + + + + + + + diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java new file mode 100644 index 000000000..076344ec4 --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Evolution AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import org.eclipse.jface.text.IDocument; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.jface.text.reconciler.MonoReconciler; + + +public class SemanticHighlightReconciler extends MonoReconciler { + + public SemanticHighlightReconciler() { + super(new SemanticHighlightReconcilerStrategy(), false); + } + + @Override + public void install(final ITextViewer textViewer) { + super.install(textViewer); + // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted + ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).install(textViewer); + } + + @Override + public void uninstall() { + super.uninstall(); + // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is accepted + ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).uninstall(); + } + +} \ No newline at end of file diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java new file mode 100644 index 000000000..fd1d4e730 --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java @@ -0,0 +1,376 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Evolution AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import org.eclipse.core.resources.IFile; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.jface.text.BadLocationException; +import org.eclipse.jface.text.IDocument; +import org.eclipse.jface.text.IRegion; +import org.eclipse.jface.text.ITextPresentationListener; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.jface.text.TextAttribute; +import org.eclipse.jface.text.TextPresentation; +import org.eclipse.jface.text.TextViewer; +import org.eclipse.jface.text.reconciler.DirtyRegion; +import org.eclipse.jface.text.reconciler.IReconcilingStrategy; +import org.eclipse.jface.text.reconciler.IReconcilingStrategyExtension; +import org.eclipse.jface.text.rules.IToken; +import org.eclipse.lsp4e.LSPEclipseUtils; +import org.eclipse.lsp4e.LanguageServerPlugin; +import org.eclipse.lsp4e.LanguageServerWrapper; +import org.eclipse.lsp4e.LanguageServersRegistry.LanguageServerDefinition; +import org.eclipse.lsp4e.LanguageServiceAccessor; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.SemanticTokenModifiers; +import org.eclipse.lsp4j.SemanticTokens; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.lsp4j.SemanticTokensParams; +import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; +import org.eclipse.lsp4j.ServerCapabilities; +import org.eclipse.lsp4j.TextDocumentIdentifier; +import org.eclipse.lsp4j.services.LanguageServer; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.StyleRange; +import org.eclipse.tm4e.ui.TMUIPlugin; +import org.eclipse.tm4e.ui.themes.ITheme; + +/** + * A reconciler strategy using semantic highlighting as defined by LSP. + */ +public class SemanticHighlightReconcilerStrategy + implements IReconcilingStrategy, IReconcilingStrategyExtension, ITextPresentationListener { + + private @Nullable ITextViewer viewer; + + private @Nullable ITheme theme; + + private IDocument document; + + private Map semanticTokensLegendMap; + + private List previousRanges; + + /** + * Installs the reconciler on the given text viewer. After this method has been + * finished, the reconciler is operational, i.e., it works without requesting + * further client actions until uninstall is called. + * + * @param textViewer + * the viewer on which the reconciler is installed + */ + public void install(final ITextViewer textViewer) { + viewer = textViewer; + theme = TMUIPlugin.getThemeManager().getDefaultTheme(); + if (textViewer instanceof TextViewer viewer) { + viewer.addTextPresentationListener(this); + } + previousRanges = new ArrayList<>(); + } + + /** + * Removes the reconciler from the text viewer it has previously been installed + * on. + */ + public void uninstall() { + theme = null; + ITextViewer textViewer = viewer; + if (textViewer instanceof TextViewer viewer) { + viewer.removeTextPresentationListener(this); + } + viewer = null; + previousRanges = null; + semanticTokensLegendMap = null; + + } + + private void initSemanticTokensLegendMap() { + IFile file = LSPEclipseUtils.getFile(document); + if (file != null) { + try { + semanticTokensLegendMap = new HashMap<>(); + for (LanguageServerWrapper wrapper: LanguageServiceAccessor.getLSWrappers(file, x -> true)) { + ServerCapabilities serverCapabilities = wrapper.getServerCapabilities(); + if (serverCapabilities != null) { + SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities.getSemanticTokensProvider(); + if (semanticTokensProvider != null) { + semanticTokensLegendMap.put(wrapper.serverDefinition.id, semanticTokensProvider.getLegend()); + } + } + } + } catch (IOException e) { + LanguageServerPlugin.logError(e); + } + } + } + + private SemanticTokensParams getSemanticTokensParams() { + URI uri = LSPEclipseUtils.toUri(document); + if (uri != null) { + SemanticTokensParams semanticTokensParams = new SemanticTokensParams(); + semanticTokensParams.setTextDocument(new TextDocumentIdentifier(uri.toString())); + return semanticTokensParams; + } + return null; + } + + private void saveStyle(final SemanticTokens semanticTokens, final SemanticTokensLegend semanticTokensLegend) { + if (semanticTokens == null || semanticTokensLegend == null) { + return; + } + List dataStream = semanticTokens.getData(); + if (!dataStream.isEmpty()) { + try { + List styleRanges = getStyleRanges(dataStream, semanticTokensLegend); + saveStyles(styleRanges); + } catch (BadLocationException e) { + LanguageServerPlugin.logError(e); + } + } + } + + private StyleRange clone(final StyleRange styleRange) { + StyleRange clonedStyleRange = new StyleRange(styleRange.start, styleRange.length, styleRange.foreground, + styleRange.background, styleRange.fontStyle); + clonedStyleRange.strikeout = styleRange.strikeout; + return clonedStyleRange; + } + + private void mergeStyles(final TextPresentation textPresentation, final List styleRanges) { + StyleRange[] array = new StyleRange[styleRanges.size()]; + array = styleRanges.toArray(array); + textPresentation.replaceStyleRanges(array); + } + + private boolean overlaps(final StyleRange range, final IRegion region) { + return isContained(range.start, region) || isContained(range.start + range.length, region) + || isContained(region.getOffset(), range); + } + + private boolean isContained(final int offset, final StyleRange range) { + return offset >= range.start && offset < (range.start + range.length); + } + + private boolean isContained(final int offset, final IRegion region) { + return offset >= region.getOffset() && offset < (region.getOffset() + region.getLength()); + } + + private void saveStyles(final List styleRanges) { + synchronized (previousRanges) { + previousRanges.clear(); + previousRanges.addAll(styleRanges); + previousRanges.sort(Comparator.comparing(s -> s.start)); + } + } + + private List getStyleRanges(final List dataStream, + final SemanticTokensLegend semanticTokensLegend) throws BadLocationException { + List styleRanges = new ArrayList<>(dataStream.size() / 5); + + int idx = 0; + int prevLine = 0; + int line = 0; + int offset = 0; + int length = 0; + String tokenType = null; + for (Integer data : dataStream) { + switch (idx % 5) { + case 0: // line + line += data; + break; + case 1: // offset + if (line == prevLine) { + offset += data; + } else { + offset = LSPEclipseUtils.toOffset(new Position(line, data), document); + } + break; + case 2: // length + length = data; + break; + case 3: // token type + tokenType = tokenType(data, semanticTokensLegend.getTokenTypes()); + break; + case 4: // token modifier + prevLine = line; + List tokenModifiers = tokenModifiers(data, semanticTokensLegend.getTokenModifiers()); + StyleRange styleRange = getStyleRange(offset, length, textAttribute(tokenType)); + if (tokenModifiers.stream().anyMatch(x -> x.equals(SemanticTokenModifiers.Deprecated))) { + styleRange.strikeout = true; + } + styleRanges.add(styleRange); + break; + } + idx++; + } + return styleRanges; + } + + private String tokenType(final Integer data, final List legend) { + try { + return legend.get(data - 1); + } catch (IndexOutOfBoundsException e) { + return null; // no match + } + } + + private List tokenModifiers(final Integer data, final List legend) { + if (data.intValue() == 0) { + return Collections.emptyList(); + } + BitSet bitSet = BitSet.valueOf(new long[] { data }); + List tokenModifiers = new ArrayList<>(); + for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { + try { + tokenModifiers.add(legend.get(i)); + } catch (IndexOutOfBoundsException e) { + // no match + } + } + + return tokenModifiers; + } + + private TextAttribute textAttribute(final String tokenType) { + ITheme localTheme = theme; + if (localTheme != null && tokenType != null) { + IToken token = localTheme.getToken(tokenType); + if (token != null) { + Object data = token.getData(); + if (data instanceof TextAttribute textAttribute) { + return textAttribute; + } + } + } + return null; + } + + /** + * Gets a style range for the given inputs. + * + * @param offset + * the offset of the range to be styled + * @param length + * the length of the range to be styled + * @param attr + * the attribute describing the style of the range to be styled + */ + private StyleRange getStyleRange(final int offset, final int length, final TextAttribute attr) { + final StyleRange styleRange; + if (attr != null) { + final int style = attr.getStyle(); + final int fontStyle = style & (SWT.ITALIC | SWT.BOLD | SWT.NORMAL); + styleRange = new StyleRange(offset, length, attr.getForeground(), attr.getBackground(), fontStyle); + styleRange.strikeout = (style & TextAttribute.STRIKETHROUGH) != 0; + styleRange.underline = (style & TextAttribute.UNDERLINE) != 0; + styleRange.font = attr.getFont(); + return styleRange; + } else { + styleRange = new StyleRange(); + styleRange.start = offset; + styleRange.length = length; + } + return styleRange; + } + + @Override + public void setProgressMonitor(final IProgressMonitor monitor) { + } + + @Override + public void setDocument(final IDocument document) { + this.document = document; + initSemanticTokensLegendMap(); + } + + private SemanticTokensLegend getSemanticTokensLegend(final LanguageServer languageSever) { + Optional serverDefinition = LanguageServiceAccessor + .resolveServerDefinition(languageSever); + if (serverDefinition.isPresent()) { + return semanticTokensLegendMap.get(serverDefinition.get().id); + } + return null; + } + + private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilities) { + return serverCapabilities.getSemanticTokensProvider() != null + && serverCapabilities.getSemanticTokensProvider().getFull().getLeft(); + } + + private CompletableFuture semanticTokensFull(final List languageServers) { + return CompletableFuture + .allOf(languageServers.stream().map(this::semanticTokensFull).toArray(CompletableFuture[]::new)); + } + + private CompletableFuture semanticTokensFull(final LanguageServer languageServer) { + SemanticTokensParams semanticTokensParams = getSemanticTokensParams(); + return languageServer.getTextDocumentService().semanticTokensFull(semanticTokensParams) + .thenAccept(semanticTokens -> { + saveStyle(semanticTokens, getSemanticTokensLegend(languageServer)); + }).exceptionally(e -> { + LanguageServerPlugin.logError(e); + return null; + }); + } + + private void fullReconcile() { + try { + LanguageServiceAccessor.getLanguageServers(document, this::hasSemanticTokensFull)// + .thenAccept(this::semanticTokensFull).get(); + } catch (InterruptedException | ExecutionException e) { + LanguageServerPlugin.logError(e); + } + } + + @Override + public void initialReconcile() { + fullReconcile(); + } + + @Override + public void reconcile(final DirtyRegion dirtyRegion, final IRegion subRegion) { + fullReconcile(); + } + + @Override + public void reconcile(final IRegion partition) { + fullReconcile(); + } + + private List appliedRanges(final TextPresentation textPresentation) { + synchronized (previousRanges) { + // we need to create new styles because the text presentation might change a + // style when applied to the presentation + // and we want the ones saved from the reconciling as immutable + return previousRanges.stream()// + .filter(r -> overlaps(r, textPresentation.getExtent()))// + .map(this::clone).collect(Collectors.toList()); + } + } + + @Override + public void applyTextPresentation(final TextPresentation textPresentation) { + mergeStyles(textPresentation, appliedRanges(textPresentation)); + } +} \ No newline at end of file diff --git a/target-platforms/target-platform-latest/target-platform-latest.target b/target-platforms/target-platform-latest/target-platform-latest.target index f0ce2b558..c2b4d4ea3 100644 --- a/target-platforms/target-platform-latest/target-platform-latest.target +++ b/target-platforms/target-platform-latest/target-platform-latest.target @@ -30,6 +30,10 @@ + + + + \ No newline at end of file