diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java index b6a731965..bef29aafa 100644 --- a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java @@ -44,6 +44,10 @@ import org.eclipse.lsp4e.test.references.FindReferencesTest; import org.eclipse.lsp4e.test.rename.LSPTextChangeTest; import org.eclipse.lsp4e.test.rename.RenameTest; +import org.eclipse.lsp4e.test.semanticTokens.SemanticHighlightReconcilerStrategyTest; +import org.eclipse.lsp4e.test.semanticTokens.SemanticTokensDataStreamProcessorTest; +import org.eclipse.lsp4e.test.semanticTokens.SemanticTokensLegendProviderTest; +import org.eclipse.lsp4e.test.semanticTokens.StyleRangeHolderTest; import org.eclipse.lsp4e.test.symbols.SymbolsModelTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -88,7 +92,11 @@ LSPCodeMiningTest.class, ShowMessageTest.class, WorkspaceFoldersTest.class, - DebugTest.class + DebugTest.class, + SemanticHighlightReconcilerStrategyTest.class, + SemanticTokensDataStreamProcessorTest.class, + SemanticTokensLegendProviderTest.class, + StyleRangeHolderTest.class }) public class AllTests { diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java new file mode 100644 index 000000000..ef61b6903 --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import static org.junit.Assert.assertArrayEquals; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import org.eclipse.core.resources.IFile; +import org.eclipse.core.resources.IProject; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.lsp4e.test.AllCleanRule; +import org.eclipse.lsp4e.test.TestUtils; +import org.eclipse.lsp4e.tests.mock.MockLanguageServer; +import org.eclipse.lsp4j.SemanticTokens; +import org.eclipse.swt.custom.StyleRange; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.ui.tests.harness.util.DisplayHelper; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +public class SemanticHighlightReconcilerStrategyTest { + @Rule + public AllCleanRule clear = new AllCleanRule(); + + private IProject project; + private Shell shell; + + @Before + public void setUp() throws CoreException { + project = TestUtils.createProject(getClass().getName() + System.currentTimeMillis()); + shell = new Shell(); + + // Setup Server Capabilities + List tokenTypes = Arrays.asList("keyword"); + List tokenModifiers = Arrays.asList("obsolete"); + SemanticTokensTestUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers); + } + + @Test + public void testKeyword() throws InterruptedException, ExecutionException, CoreException { + SemanticTokens semanticTokens = new SemanticTokens(); + semanticTokens.setData(SemanticTokensTestUtil.keywordSemanticTokens()); + + MockLanguageServer.INSTANCE.getTextDocumentService().setSemanticTokens(semanticTokens); + + IFile file = TestUtils.createUniqueTestFile(project, "lspt", SemanticTokensTestUtil.keywordText); + ITextViewer textViewer = TestUtils.openTextViewer(file); + + Display display = shell.getDisplay(); + DisplayHelper.sleep(display, 2_000); // Give some time to the editor to update + + StyleRange[] styleRanges = textViewer.getTextWidget().getStyleRanges(); + + List expectedStyleRanges = Arrays.asList(// + new StyleRange(0, 4, SemanticTokensTestUtil.GREEN, null), // + new StyleRange(15, 4, SemanticTokensTestUtil.GREEN, null), // + new StyleRange(24, 7, SemanticTokensTestUtil.GREEN, null)// + ); + assertArrayEquals(expectedStyleRanges.toArray(), styleRanges); + } +} diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java new file mode 100644 index 000000000..e40a915c3 --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import org.eclipse.jface.text.Document; +import org.eclipse.lsp4e.operations.semanticTokens.SemanticTokensDataStreamProcessor; +import org.eclipse.lsp4e.test.AllCleanRule; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.swt.custom.StyleRange; +import org.junit.Rule; +import org.junit.Test; + +public class SemanticTokensDataStreamProcessorTest { + @Rule + public AllCleanRule clear = new AllCleanRule(); + + @Test + public void testKeyword() throws InterruptedException, ExecutionException { + Document document = new Document(SemanticTokensTestUtil.keywordText); + + SemanticTokensDataStreamProcessor processor = new SemanticTokensDataStreamProcessor(SemanticTokensTestUtil + .keywordTokenTypeMapper(SemanticTokensTestUtil.RED_TOKEN), SemanticTokensTestUtil.offsetMapper(document)); + + List expectedStream = SemanticTokensTestUtil.keywordSemanticTokens(); + List expectedStyleRanges = Arrays.asList(// + new StyleRange(0, 4, SemanticTokensTestUtil.RED, null), // + new StyleRange(15, 4, SemanticTokensTestUtil.RED, null), // + new StyleRange(24, 7, SemanticTokensTestUtil.RED, null)// + ); + + List styleRanges = processor.getStyleRanges(expectedStream, getSemanticTokensLegend()); + + assertEquals(expectedStyleRanges, styleRanges); + } + + private SemanticTokensLegend getSemanticTokensLegend() { + SemanticTokensLegend semanticTokensLegend = new SemanticTokensLegend(); + semanticTokensLegend.setTokenTypes(Arrays.asList("keyword","other")); + semanticTokensLegend.setTokenModifiers(Arrays.asList("obsolete")); + return semanticTokensLegend; + } +} diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java new file mode 100644 index 000000000..ae6ffc97d --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java @@ -0,0 +1,63 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import org.eclipse.core.resources.IFile; +import org.eclipse.core.resources.IProject; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.jface.text.BadLocationException; +import org.eclipse.lsp4e.LanguageServiceAccessor; +import org.eclipse.lsp4e.operations.semanticTokens.SemanticHighlightReconcilerStrategy; +import org.eclipse.lsp4e.test.AllCleanRule; +import org.eclipse.lsp4e.test.TestUtils; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.lsp4j.services.LanguageServer; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +public class SemanticTokensLegendProviderTest { + + @Rule + public AllCleanRule clear = new AllCleanRule(); + + private IProject project; + + @Before + public void setUp() throws CoreException { + project = TestUtils.createProject(getClass().getName() + System.currentTimeMillis()); + } + + @Test + public void testSemanticTokensLegendProvider() throws BadLocationException, CoreException, IOException, InterruptedException, ExecutionException { + // Setup Server Capabilities + List tokenTypes = Arrays.asList("keyword","other"); + List tokenModifiers = Arrays.asList("obsolete"); + SemanticTokensTestUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers); + + // Setup test data + IFile file = TestUtils.createUniqueTestFile(project, "lspt", "test content"); + // start the LS + LanguageServer languageServer = LanguageServiceAccessor.getInitializedLanguageServers(file, c -> Boolean.TRUE).iterator() + .next().get(); + + SemanticTokensLegend semanticTokensLegend = (new SemanticHighlightReconcilerStrategy()).getSemanticTokensLegend(languageServer); + assertNotNull(semanticTokensLegend); + assertEquals(tokenTypes, semanticTokensLegend.getTokenTypes()); + assertEquals(tokenModifiers, semanticTokensLegend.getTokenModifiers()); + } +} diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensTestUtil.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensTestUtil.java new file mode 100644 index 000000000..5d603a72b --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensTestUtil.java @@ -0,0 +1,130 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.BadLocationException; +import org.eclipse.jface.text.IDocument; +import org.eclipse.jface.text.TextAttribute; +import org.eclipse.jface.text.rules.IToken; +import org.eclipse.lsp4e.LSPEclipseUtils; +import org.eclipse.lsp4e.tests.mock.MockLanguageServer; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; +import org.eclipse.swt.graphics.Color; + +public class SemanticTokensTestUtil { + public static final String keywordText = + "type foo {\n" + + " \n" + + "}\n" + + "type bar extends foo {\n" + + " \n" + + "}\n"; + + public static List keywordSemanticTokens() { + List> expectedTokens = new ArrayList<>(); + expectedTokens.add(Arrays.asList(0,0,4,1,0)); + expectedTokens.add(Arrays.asList(3,0,4,1,0)); + expectedTokens.add(Arrays.asList(0,9,7,1,0)); + + return expectedTokens.stream().flatMap(List::stream).toList(); + } + + public static final Color GREEN = new Color(133, 153, 0, 255); + public static final Color RED = new Color(255, 0, 0); + + public static final IToken GREEN_TOKEN = new IToken() { + @Override + public boolean isWhitespace() { + return false; + } + + @Override + public boolean isUndefined() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isEOF() { + return false; + } + + @Override + public Object getData() { + return new TextAttribute(GREEN); + } + }; + + public static final IToken RED_TOKEN = new IToken() { + @Override + public boolean isWhitespace() { + return false; + } + + @Override + public boolean isUndefined() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isEOF() { + return false; + } + + @Override + public Object getData() { + return new TextAttribute(RED); + } + }; + + public static Function keywordTokenTypeMapper(final IToken token) { + return t -> { + if ("keyword".equals(t)) { + return token; + } else { + return null; + } + }; + } + + public static @NonNull Function offsetMapper(IDocument document) { + return (p) -> { + try { + return LSPEclipseUtils.toOffset(p, document); + } catch (BadLocationException e) { + throw new RuntimeException(e); + } + }; + } + public static void setSemanticTokensLegend(final List tokenTypes, List tokenModifiers) { + SemanticTokensLegend legend = new SemanticTokensLegend(tokenTypes, tokenModifiers); + SemanticTokensWithRegistrationOptions semanticTokensWithRegistrationOptions = new SemanticTokensWithRegistrationOptions(legend); + semanticTokensWithRegistrationOptions.setFull(true); + semanticTokensWithRegistrationOptions.setRange(false); + + MockLanguageServer.INSTANCE.getInitializeResult().getCapabilities().setSemanticTokensProvider(semanticTokensWithRegistrationOptions); + } +} diff --git a/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/StyleRangeHolderTest.java b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/StyleRangeHolderTest.java new file mode 100644 index 000000000..86f1c6a02 --- /dev/null +++ b/org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/StyleRangeHolderTest.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.test.semanticTokens; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.util.Arrays; +import java.util.List; + +import org.eclipse.jface.text.DocumentEvent; +import org.eclipse.jface.text.Region; +import org.eclipse.jface.text.TextEvent; +import org.eclipse.lsp4e.operations.semanticTokens.StyleRangeHolder; +import org.eclipse.lsp4e.test.AllCleanRule; +import org.eclipse.swt.custom.StyleRange; +import org.eclipse.swt.graphics.Color; +import org.junit.Rule; +import org.junit.Test; + +public class StyleRangeHolderTest { + @Rule + public AllCleanRule clear = new AllCleanRule(); + + private static final Color RED = new Color(255, 0, 0); + private List originalStyleRanges = Arrays.asList(new StyleRange(0, 4, RED, null), new StyleRange(15, 4, RED, null), new StyleRange(24, 7, RED, null)); + + @Test + public void testAllDocumentRanges() { + StyleRangeHolder holder = new StyleRangeHolder(); + holder.saveStyles(originalStyleRanges); + + StyleRange[] allDocumentRanges = holder.overlappingRanges(new Region(0, 50)); + + assertNotEquals(originalStyleRanges, allDocumentRanges); // styles must be copied + assertEquals(originalStyleRanges.size(), allDocumentRanges.length); + } + + @Test + public void testPartialDocumentRanges() { + StyleRangeHolder holder = new StyleRangeHolder(); + holder.saveStyles(originalStyleRanges); + + StyleRange[] allDocumentRanges = holder.overlappingRanges(new Region(0, 20)); // only two ranges overlap this region + + assertEquals(2, allDocumentRanges.length); + } + + @Test + public void testDocumentChange() { + StyleRangeHolder holder = new StyleRangeHolder(); + holder.saveStyles(originalStyleRanges); + + TextEvent textEvent = new TextEvent(0, 1, " ", null, new DocumentEvent(), false) {}; + + // this will remove the first style and shift the last two + holder.textChanged(textEvent); + + StyleRange[] noOverlappingRanges = holder.overlappingRanges(new Region(0, 10)); // only one range overlap this region + + assertEquals(0, noOverlappingRanges.length); + + StyleRange[] twoShiftedOverlappingRanges = holder.overlappingRanges(new Region(10, 50)); // only one range overlap this region + + assertEquals(2, twoShiftedOverlappingRanges.length); + assertEquals(16, twoShiftedOverlappingRanges[0].start); + assertEquals(4, twoShiftedOverlappingRanges[0].length); + assertEquals(25, twoShiftedOverlappingRanges[1].start); + assertEquals(7, twoShiftedOverlappingRanges[1].length); + } +} diff --git a/org.eclipse.lsp4e.tests.mock/src/org/eclipse/lsp4e/tests/mock/MockTextDocumentService.java b/org.eclipse.lsp4e.tests.mock/src/org/eclipse/lsp4e/tests/mock/MockTextDocumentService.java index ccb57ac61..9073bc830 100644 --- a/org.eclipse.lsp4e.tests.mock/src/org/eclipse/lsp4e/tests/mock/MockTextDocumentService.java +++ b/org.eclipse.lsp4e.tests.mock/src/org/eclipse/lsp4e/tests/mock/MockTextDocumentService.java @@ -64,6 +64,8 @@ import org.eclipse.lsp4j.Range; import org.eclipse.lsp4j.ReferenceParams; import org.eclipse.lsp4j.RenameParams; +import org.eclipse.lsp4j.SemanticTokens; +import org.eclipse.lsp4j.SemanticTokensParams; import org.eclipse.lsp4j.SignatureHelp; import org.eclipse.lsp4j.SignatureHelpParams; import org.eclipse.lsp4j.SymbolInformation; @@ -104,6 +106,7 @@ public class MockTextDocumentService implements TextDocumentService { private WorkspaceEdit mockRenameEdit; private Either3 mockPrepareRenameResult; private List documentSymbols; + private SemanticTokens mockSemanticTokens; public MockTextDocumentService(Function> futureFactory) { this._futureFactory = futureFactory; @@ -387,4 +390,12 @@ public void setWillSaveWaitUntilCallback(List textEdits) { this.mockWillSaveWaitUntilTextEdits = textEdits; } + public void setSemanticTokens(final SemanticTokens semanticTokens) { + this.mockSemanticTokens = semanticTokens; + } + + @Override + public CompletableFuture semanticTokensFull(SemanticTokensParams params) { + return CompletableFuture.completedFuture(this.mockSemanticTokens); + } } diff --git a/org.eclipse.lsp4e/META-INF/MANIFEST.MF b/org.eclipse.lsp4e/META-INF/MANIFEST.MF index 158dd9d53..32f3638bf 100644 --- a/org.eclipse.lsp4e/META-INF/MANIFEST.MF +++ b/org.eclipse.lsp4e/META-INF/MANIFEST.MF @@ -25,6 +25,7 @@ Require-Bundle: org.eclipse.core.runtime;bundle-version="3.12.0", org.eclipse.debug.ui;bundle-version="3.11.200", org.eclipse.swt, org.eclipse.jdt.annotation;bundle-version="2.1.0";resolution:=optional, + org.eclipse.tm4e.ui, org.eclipse.ui.editors, org.eclipse.ui.navigator;bundle-version="3.6.100", org.eclipse.lsp4j;bundle-version="[0.19.0,0.20.0)", diff --git a/org.eclipse.lsp4e/plugin.xml b/org.eclipse.lsp4e/plugin.xml index fad248dce..67fdadd1c 100644 --- a/org.eclipse.lsp4e/plugin.xml +++ b/org.eclipse.lsp4e/plugin.xml @@ -657,4 +657,21 @@ + + + + + + + + + + + + + + diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java index 74eaded19..a892eed56 100644 --- a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/LanguageServiceAccessor.java @@ -397,7 +397,7 @@ private static Collection getLSWrappers(@NonNull final ID } if (contentType.getBaseType() != null) { - contentTypesToProcess.add(contentType.getBaseType()); + contentTypesToProcess.add(contentType.getBaseType()); } processedContentTypes.add(contentType); } @@ -648,9 +648,12 @@ public static boolean checkCapability(LanguageServer languageServer, Predicate condition.test(wrapper.getServerCapabilities())); } - public static Optional resolveServerDefinition(LanguageServer languageServer) { + public static Optional resolveLanguageServerWrapper(LanguageServer languageServer) { return startedServers.stream() // - .filter(wrapper -> languageServer.equals(wrapper.getServer())).findFirst() - .map(wrapper -> wrapper.serverDefinition); + .filter(wrapper -> languageServer.equals(wrapper.getServer())).findFirst(); + } + + public static Optional resolveServerDefinition(LanguageServer languageServer) { + return resolveLanguageServerWrapper(languageServer).map(w -> w.serverDefinition); } } diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java new file mode 100644 index 000000000..78771850b --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconciler.java @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.util.Objects; + +import org.eclipse.jface.text.IDocument; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.jface.text.reconciler.MonoReconciler; + +public class SemanticHighlightReconciler extends MonoReconciler { + + public SemanticHighlightReconciler() { + super(new SemanticHighlightReconcilerStrategy(), false); + } + + @Override + public void install(final ITextViewer textViewer) { + super.install(textViewer); + // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is + // accepted + Objects.requireNonNull(textViewer); + ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)) + .install(textViewer); + } + + @Override + public void uninstall() { + super.uninstall(); + // no need to do that if https://bugs.eclipse.org/bugs/show_bug.cgi?id=521326 is + // accepted + ((SemanticHighlightReconcilerStrategy) getReconcilingStrategy(IDocument.DEFAULT_CONTENT_TYPE)).uninstall(); + } + +} \ No newline at end of file diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java new file mode 100644 index 000000000..3bd985c21 --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticHighlightReconcilerStrategy.java @@ -0,0 +1,327 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Evolution AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.net.URI; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; + +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.jface.preference.IPreferenceStore; +import org.eclipse.jface.text.BadLocationException; +import org.eclipse.jface.text.IDocument; +import org.eclipse.jface.text.IRegion; +import org.eclipse.jface.text.ITextListener; +import org.eclipse.jface.text.ITextPresentationListener; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.jface.text.TextPresentation; +import org.eclipse.jface.text.TextViewer; +import org.eclipse.jface.text.reconciler.DirtyRegion; +import org.eclipse.jface.text.reconciler.IReconcilingStrategy; +import org.eclipse.jface.text.reconciler.IReconcilingStrategyExtension; +import org.eclipse.lsp4e.LSPEclipseUtils; +import org.eclipse.lsp4e.LanguageServerPlugin; +import org.eclipse.lsp4e.LanguageServerWrapper; +import org.eclipse.lsp4e.LanguageServiceAccessor; +import org.eclipse.lsp4e.LanguageServiceAccessor.LSPDocumentInfo; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.SemanticTokens; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.lsp4j.SemanticTokensParams; +import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions; +import org.eclipse.lsp4j.ServerCapabilities; +import org.eclipse.lsp4j.TextDocumentIdentifier; +import org.eclipse.lsp4j.jsonrpc.ResponseErrorException; +import org.eclipse.lsp4j.jsonrpc.messages.ResponseError; +import org.eclipse.lsp4j.jsonrpc.messages.ResponseErrorCode; +import org.eclipse.lsp4j.services.LanguageServer; +import org.eclipse.swt.custom.StyleRange; +import org.eclipse.swt.custom.StyledText; + +/** + * A semantic reconciler strategy using LSP. + *

+ * This strategy applies semantic highlight on top of the syntactic highlight + * provided by TM4E by implementing {@link ITextPresentationListener}. Because + * semantic highlight involves remote calls, it is expected to be slower than + * syntactic highlight. Thus we execute semantic highlight processing in a + * separate reconciler thread that will never block TM4E. + *

+ * This strategy records the version of the document when the semantic highlight + * is sent to the LS and when TM4E highlight is applied. If the response for a + * particular document version comes after the TM4E highlight has been applied, + * the text presentation is invalidated so that highlight is extended with the + * results. + *

+ * To avoid flickering, {@link StyleRangeHolder} implement {@link ITextListener} + * to adapt recorded semantic highlights and apply those instead of nothing + * where needed. + *

+ * If the response comes before, the data is saved and applied later on top of + * the presentation provided by TM4E. The results from our reconciler are + * recorded + *

+ * For simplicity, out-dated responses are discarded, as we know we shall get + * newer ones. + *

+ * In case the reconciler produces bogus results, it can be disabled with the key + * {@literal semanticHighlightReconciler.disabled} until fix is provided. + */ +public class SemanticHighlightReconcilerStrategy + implements IReconcilingStrategy, IReconcilingStrategyExtension, ITextPresentationListener { + + private final boolean disabled; + + private ITextViewer viewer; + + private IDocument document; + + private StyleRangeHolder styleRangeHolder; + + private SemanticTokensDataStreamProcessor semanticTokensDataStreamProcessor; + + /** + * Written in {@link this.class#applyTextPresentation(TextPresentation)} + * applyTextPresentation and read in the lambda in + * {@link this.class#semanticTokensFull(LanguageServer, int)}, the lambda and + * the former method are executed in the display thread, thus serializing access + * using volatile without using explicit synchronized blocks is enough to avoid + * that org.eclipse.jface.text.ITextViewer.invalidateTextPresentation() is + * called by use while the presentation is being updated. + */ + private volatile int documentVersionAtLastAppliedTextPresentation; + + private CompletableFuture semanticTokensFullFuture; + + public SemanticHighlightReconcilerStrategy() { + IPreferenceStore store = LanguageServerPlugin.getDefault().getPreferenceStore(); + disabled = store.getBoolean("semanticHighlightReconciler.disabled"); //$NON-NLS-1$ + } + + /** + * Installs the reconciler on the given text viewer. After this method has been + * finished, the reconciler is operational, i.e., it works without requesting + * further client actions until uninstall is called. + * + * @param textViewer + * the viewer on which the reconciler is installed + */ + public void install(@NonNull final ITextViewer textViewer) { + if (disabled) { + return; + } + viewer = textViewer; + styleRangeHolder = new StyleRangeHolder(); + semanticTokensDataStreamProcessor = new SemanticTokensDataStreamProcessor(new TokenTypeMapper(textViewer), + offsetMapper()); + + if (viewer instanceof TextViewer) { + ((TextViewer) viewer).addTextPresentationListener(this); + } + viewer.addTextListener(styleRangeHolder); + } + + /** + * Removes the reconciler from the text viewer it has previously been installed + * on. + */ + public void uninstall() { + if (disabled) { + return; + } + semanticTokensDataStreamProcessor = null; + if (viewer instanceof TextViewer) { + ((TextViewer) viewer).removeTextPresentationListener(this); + } + viewer.removeTextListener(styleRangeHolder); + viewer = null; + styleRangeHolder = null; + } + + private @NonNull Function offsetMapper() { + return (p) -> { + try { + return LSPEclipseUtils.toOffset(p, document); + } catch (BadLocationException e) { + throw new RuntimeException(e); + } + }; + } + + private SemanticTokensParams getSemanticTokensParams() { + URI uri = LSPEclipseUtils.toUri(document); + if (uri != null) { + SemanticTokensParams semanticTokensParams = new SemanticTokensParams(); + semanticTokensParams.setTextDocument(new TextDocumentIdentifier(uri.toString())); + return semanticTokensParams; + } + return null; + } + + private void saveStyle(final SemanticTokens semanticTokens, final SemanticTokensLegend semanticTokensLegend) { + if (semanticTokens == null || semanticTokensLegend == null) { + return; + } + List dataStream = semanticTokens.getData(); + if (!dataStream.isEmpty()) { + List styleRanges = semanticTokensDataStreamProcessor.getStyleRanges(dataStream, + semanticTokensLegend); + styleRangeHolder.saveStyles(styleRanges); + } + } + + @Override + public void setProgressMonitor(final IProgressMonitor monitor) { + } + + @Override + public void setDocument(final IDocument document) { + this.document = document; + } + + private boolean hasSemanticTokensFull(final ServerCapabilities serverCapabilities) { + return serverCapabilities.getSemanticTokensProvider() != null + && serverCapabilities.getSemanticTokensProvider().getFull().getLeft(); + } + + private CompletableFuture semanticTokensFull(final List languageServers, final int version) { + return CompletableFuture.allOf( + languageServers.stream().map(ls -> semanticTokensFull(ls, version)).toArray(CompletableFuture[]::new)); + } + + private boolean isRequestCancelledException(final Throwable throwable) { + if (throwable instanceof CompletionException) { + Throwable cause = ((CompletionException) throwable).getCause(); + if (cause instanceof ResponseErrorException) { + ResponseError responseError = ((ResponseErrorException) cause).getResponseError(); + return responseError != null + && responseError.getCode() == ResponseErrorCode.RequestCancelled.getValue(); + } + } + return false; + } + + private @Nullable SemanticTokensLegend getSemanticTokensLegend(final LanguageServerWrapper wrapper) { + ServerCapabilities serverCapabilities = wrapper.getServerCapabilities(); + if (serverCapabilities != null) { + SemanticTokensWithRegistrationOptions semanticTokensProvider = serverCapabilities + .getSemanticTokensProvider(); + if (semanticTokensProvider != null) { + return semanticTokensProvider.getLegend(); + } + } + return null; + } + + // public for testing + public @Nullable SemanticTokensLegend getSemanticTokensLegend(@NonNull final LanguageServer languageSever) { + return LanguageServiceAccessor.resolveLanguageServerWrapper(languageSever) + .map(this::getSemanticTokensLegend).orElse(null); + } + + /** The presentation is invalidated if applyTextPresentation has never been called (e.g. there is + * no syntactic reconciler as in our unit tests) or the syntactic reconciler has already been applied + * for the given document. Otherwise the style rages will be applied when applyTextPresentation is + * called as part of the syntactic reconciliation. + */ + private boolean invalidateTextPresentation(final int version) { + return documentVersionAtLastAppliedTextPresentation == 0 + || documentVersionAtLastAppliedTextPresentation == version; + } + + private CompletableFuture semanticTokensFull(final LanguageServer languageServer, final int version) { + SemanticTokensParams semanticTokensParams = getSemanticTokensParams(); + return languageServer.getTextDocumentService().semanticTokensFull(semanticTokensParams) + .thenAccept(semanticTokens -> { + if (getDocumentVersion() == version) { + saveStyle(semanticTokens, getSemanticTokensLegend(languageServer)); + StyledText textWidget = viewer.getTextWidget(); + textWidget.getDisplay().asyncExec(() -> { + if (!textWidget.isDisposed() && invalidateTextPresentation(version)) { + viewer.invalidateTextPresentation(); + } + }); + } + }).exceptionally(e -> { + if (!isRequestCancelledException(e)) { + LanguageServerPlugin.logError(e); + } + return null; + }); + } + + private int getDocumentVersion() { + IDocument theDocument = document; + if (theDocument != null) { + Iterator<@NonNull LSPDocumentInfo> iterator = LanguageServiceAccessor + .getLSPDocumentInfosFor(theDocument, this::hasSemanticTokensFull).iterator(); + if (iterator.hasNext()) { + @Nullable LSPDocumentInfo documentInfo = iterator.next(); + if (documentInfo != null) { + return documentInfo.getVersion(); + } + } + } + return -1; + } + + private void cancelSemanticTokensFull() { + if (semanticTokensFullFuture != null) { + semanticTokensFullFuture.cancel(true); + } + } + + private void fullReconcile() { + if (disabled) { + return; + } + IDocument theDocument = document; + cancelSemanticTokensFull(); + if (theDocument != null) { + try { + int version = getDocumentVersion(); + semanticTokensFullFuture = LanguageServiceAccessor.getLanguageServers(theDocument, this::hasSemanticTokensFull)// + .thenAccept(ls -> semanticTokensFull(ls, version)); + semanticTokensFullFuture.get(); + } catch (InterruptedException | ExecutionException e) { + LanguageServerPlugin.logError(e); + } + } + } + + @Override + public void initialReconcile() { + fullReconcile(); + } + + @Override + public void reconcile(final DirtyRegion dirtyRegion, final IRegion subRegion) { + fullReconcile(); + } + + @Override + public void reconcile(final IRegion partition) { + fullReconcile(); + } + + @Override + public void applyTextPresentation(final TextPresentation textPresentation) { + documentVersionAtLastAppliedTextPresentation = getDocumentVersion(); + IRegion extent = textPresentation.getExtent(); + if (extent != null) { + textPresentation.replaceStyleRanges(styleRangeHolder.overlappingRanges(extent)); + } + } +} diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java new file mode 100644 index 000000000..efad1f0cc --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/SemanticTokensDataStreamProcessor.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.TextAttribute; +import org.eclipse.jface.text.rules.IToken; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.SemanticTokenModifiers; +import org.eclipse.lsp4j.SemanticTokensLegend; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.StyleRange; + +/** + * The Class SemanticTokensDataStreamProcessor translates a stream of integers + * as defined by the LSP SemanticTokenRequests into a list of StyleRanges. + */ +public class SemanticTokensDataStreamProcessor { + + private final Function offsetMapper; + private final Function tokenTypeMapper; + + /** + * Creates a new instance of {@link SemanticTokensDataStreamProcessor}. + * + * @param tokenTypeMapper + * @param offsetMapper + */ + public SemanticTokensDataStreamProcessor(@NonNull final Function tokenTypeMapper, + @NonNull final Function offsetMapper) { + this.tokenTypeMapper = tokenTypeMapper; + this.offsetMapper = offsetMapper; + } + + /** + * Get the StyleRanges for the given data stream and tokens legend. + * + * @param dataStream + * @param semanticTokensLegend + * @return + */ + public @NonNull List getStyleRanges(@NonNull final List dataStream, + @NonNull final SemanticTokensLegend semanticTokensLegend) { + List styleRanges = new ArrayList<>(dataStream.size() / 5); + + int idx = 0; + int prevLine = 0; + int line = 0; + int offset = 0; + int length = 0; + String tokenType = null; + for (Integer data : dataStream) { + switch (idx % 5) { + case 0: // line + line += data; + break; + case 1: // offset + if (line == prevLine) { + offset += data; + } else { + offset = offsetMapper.apply(new Position(line, data)); + } + break; + case 2: // length + length = data; + break; + case 3: // token type + tokenType = tokenType(data, semanticTokensLegend.getTokenTypes()); + break; + case 4: // token modifier + prevLine = line; + List tokenModifiers = tokenModifiers(data, semanticTokensLegend.getTokenModifiers()); + StyleRange styleRange = getStyleRange(offset, length, textAttribute(tokenType)); + if (tokenModifiers.stream().anyMatch(x -> x.equals(SemanticTokenModifiers.Deprecated))) { + styleRange.strikeout = true; + } + styleRanges.add(styleRange); + break; + } + idx++; + } + return styleRanges; + } + + private String tokenType(final Integer data, final List legend) { + try { + return legend.get(data - 1); + } catch (IndexOutOfBoundsException e) { + return null; // no match + } + } + + private List tokenModifiers(final Integer data, final List legend) { + if (data.intValue() == 0) { + return Collections.emptyList(); + } + BitSet bitSet = BitSet.valueOf(new long[] { data }); + List tokenModifiers = new ArrayList<>(); + for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { + try { + tokenModifiers.add(legend.get(i)); + } catch (IndexOutOfBoundsException e) { + // no match + } + } + + return tokenModifiers; + } + + private TextAttribute textAttribute(final String tokenType) { + if (tokenType != null) { + IToken token = tokenTypeMapper.apply(tokenType); + if (token != null) { + Object data = token.getData(); + if (data instanceof TextAttribute) { + return (TextAttribute) data; + } + } + } + return null; + } + + /** + * Gets a style range for the given inputs. + * + * @param offset + * the offset of the range to be styled + * @param length + * the length of the range to be styled + * @param attr + * the attribute describing the style of the range to be styled + */ + private StyleRange getStyleRange(final int offset, final int length, final TextAttribute attr) { + final StyleRange styleRange; + if (attr != null) { + final int style = attr.getStyle(); + final int fontStyle = style & (SWT.ITALIC | SWT.BOLD | SWT.NORMAL); + styleRange = new StyleRange(offset, length, attr.getForeground(), attr.getBackground(), fontStyle); + styleRange.strikeout = (style & TextAttribute.STRIKETHROUGH) != 0; + styleRange.underline = (style & TextAttribute.UNDERLINE) != 0; + styleRange.font = attr.getFont(); + return styleRange; + } else { + styleRange = new StyleRange(); + styleRange.start = offset; + styleRange.length = length; + } + return styleRange; + } +} \ No newline at end of file diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java new file mode 100644 index 000000000..d03b4a04d --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/StyleRangeHolder.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.IRegion; +import org.eclipse.jface.text.ITextListener; +import org.eclipse.jface.text.Region; +import org.eclipse.jface.text.TextEvent; +import org.eclipse.swt.custom.StyleRange; + +/** + * The Class SemanticTokensDataStreamProcessor holds a list of StyleRanges. + *

+ * To avoid flickering, we also implement {@link ITextListener} to adapt (the + * only adaptation currently supported shifting ranges) recorded semantic + * highlights When the user writes a single or multi-line comments shifting is + * not enough. That could be improved if we can access + * org.eclipse.tm4e.languageconfiguration.ILanguageConfiguration.getComments() + * (still unclear on how to do that). + */ +public class StyleRangeHolder implements ITextListener { + private final List previousRanges; + + public StyleRangeHolder() { + previousRanges = new ArrayList<>(); + } + + /** + * save the styles. + * + * @param styleRanges + */ + public void saveStyles(@NonNull final List styleRanges) { + synchronized (previousRanges) { + previousRanges.clear(); + previousRanges.addAll(styleRanges); + previousRanges.sort(Comparator.comparing(s -> s.start)); + } + } + + /** + * return a copy of the saved styles that overlap the given region. + * + * @param region + * @return + */ + public StyleRange[] overlappingRanges(@NonNull final IRegion region) { + synchronized (previousRanges) { + // we need to create new styles because the text presentation might change a + // style when applied to the presentation + // and we want the ones saved from the reconciling as immutable + return previousRanges.stream()// + .filter(r -> overlaps(r, region))// + .map(this::clone).toArray(StyleRange[]::new); + } + } + + private StyleRange clone(final StyleRange styleRange) { + StyleRange clonedStyleRange = new StyleRange(styleRange.start, styleRange.length, styleRange.foreground, + styleRange.background, styleRange.fontStyle); + clonedStyleRange.strikeout = styleRange.strikeout; + return clonedStyleRange; + } + + private boolean isContained(final int offset, final StyleRange range) { + return offset >= range.start && offset < (range.start + range.length); + } + + private boolean isContained(final int offset, final IRegion region) { + return offset >= region.getOffset() && offset < (region.getOffset() + region.getLength()); + } + + private boolean overlaps(final StyleRange range, final IRegion region) { + return isContained(range.start, region) || isContained(range.start + range.length, region) + || isContained(region.getOffset(), range); + } + + @Override + public void textChanged(final TextEvent event) { + if (event.getDocumentEvent() != null) { // if null, it is an internal event, not a changed text + String replacedText = event.getReplacedText(); + String text = event.getText(); + int delta = (text != null ? text.length() : 0) - (replacedText != null ? replacedText.length() : 0); + synchronized (previousRanges) { + previousRanges.removeIf(r -> isContained(event.getOffset(), new Region(r.start, r.length))); + previousRanges.stream().filter(r -> r.start >= event.getOffset()).forEach(r -> r.start += delta); + } + } + } + +} diff --git a/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java new file mode 100644 index 000000000..00fdda7bf --- /dev/null +++ b/org.eclipse.lsp4e/src/org/eclipse/lsp4e/operations/semanticTokens/TokenTypeMapper.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Copyright (c) 2022 Avaloq Group AG. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + *******************************************************************************/ +package org.eclipse.lsp4e.operations.semanticTokens; + +import java.util.function.Function; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jface.text.ITextViewer; +import org.eclipse.jface.text.rules.IToken; +import org.eclipse.tm4e.ui.TMUIPlugin; +import org.eclipse.tm4e.ui.text.TMPresentationReconciler; +import org.eclipse.tm4e.ui.themes.ITokenProvider; + +/** + * A Class that maps TokenTypes to {@link IToken}. + */ +public class TokenTypeMapper implements Function { + private @NonNull final ITextViewer viewer; + + public TokenTypeMapper(@NonNull final ITextViewer viewer) { + this.viewer = viewer; + } + + @Override + public IToken apply(final String tokenType) { + if (tokenType == null) { + return null; + } + TMPresentationReconciler tmPresentationReconciler = TMPresentationReconciler + .getTMPresentationReconciler(viewer); + + if (tmPresentationReconciler != null) { + ITokenProvider tokenProvider = tmPresentationReconciler.getTokenProvider(); + if (tokenProvider != null) { + tokenProvider.getToken(tokenType); + } + } + return TMUIPlugin.getThemeManager().getDefaultTheme().getToken(tokenType); + } +} diff --git a/target-platforms/target-platform-latest/target-platform-latest.target b/target-platforms/target-platform-latest/target-platform-latest.target index 1cae84f65..703b30c58 100644 --- a/target-platforms/target-platform-latest/target-platform-latest.target +++ b/target-platforms/target-platform-latest/target-platform-latest.target @@ -26,6 +26,10 @@ + + + + \ No newline at end of file