Implementation of a semantic tokens reconciler

org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/AllTests.java

@@ -44,6 +44,10 @@
 import org.eclipse.lsp4e.test.references.FindReferencesTest;
 import org.eclipse.lsp4e.test.rename.LSPTextChangeTest;
 import org.eclipse.lsp4e.test.rename.RenameTest;
+import org.eclipse.lsp4e.test.semanticTokens.SemanticHighlightReconcilerStrategyTest;
+import org.eclipse.lsp4e.test.semanticTokens.SemanticTokensDataStreamProcessorTest;
+import org.eclipse.lsp4e.test.semanticTokens.SemanticTokensLegendProviderTest;
+import org.eclipse.lsp4e.test.semanticTokens.StyleRangeHolderTest;
 import org.eclipse.lsp4e.test.symbols.SymbolsModelTest;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
@@ -88,7 +92,11 @@
 	LSPCodeMiningTest.class,
 	ShowMessageTest.class,
 	WorkspaceFoldersTest.class,
-	DebugTest.class
+	DebugTest.class,
+	SemanticHighlightReconcilerStrategyTest.class,
+	SemanticTokensDataStreamProcessorTest.class,
+	SemanticTokensLegendProviderTest.class,
+	StyleRangeHolderTest.class
 })
 public class AllTests {
 

org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticHighlightReconcilerStrategyTest.java

@@ -0,0 +1,73 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Avaloq Group AG.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.lsp4e.test.semanticTokens;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.jface.text.ITextViewer;
+import org.eclipse.lsp4e.test.AllCleanRule;
+import org.eclipse.lsp4e.test.TestUtils;
+import org.eclipse.lsp4e.tests.mock.MockLanguageServer;
+import org.eclipse.lsp4j.SemanticTokens;
+import org.eclipse.swt.custom.StyleRange;
+import org.eclipse.swt.widgets.Display;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.ui.tests.harness.util.DisplayHelper;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class SemanticHighlightReconcilerStrategyTest {
+	@Rule
+	public AllCleanRule clear = new AllCleanRule();
+
+	private IProject project;
+	private Shell shell;
+
+	@Before
+	public void setUp() throws CoreException {
+		project = TestUtils.createProject(getClass().getName() + System.currentTimeMillis());
+		shell = new Shell();
+
+		// Setup Server Capabilities
+		List<String> tokenTypes = Arrays.asList("keyword");
+		List<String> tokenModifiers = Arrays.asList("obsolete");
+		SemanticTokensTestUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers);
+	}
+
+	@Test
+	public void testKeyword() throws InterruptedException, ExecutionException, CoreException {
+		SemanticTokens semanticTokens = new SemanticTokens();
+		semanticTokens.setData(SemanticTokensTestUtil.keywordSemanticTokens());
+
+		MockLanguageServer.INSTANCE.getTextDocumentService().setSemanticTokens(semanticTokens);
+
+		IFile file = TestUtils.createUniqueTestFile(project, "lspt", SemanticTokensTestUtil.keywordText);
+		ITextViewer textViewer = TestUtils.openTextViewer(file);
+
+		Display display = shell.getDisplay();
+		DisplayHelper.sleep(display, 2_000); // Give some time to the editor to update
+
+		StyleRange[] styleRanges = textViewer.getTextWidget().getStyleRanges();
+
+		List<StyleRange> expectedStyleRanges = Arrays.asList(//
+				new StyleRange(0, 4, SemanticTokensTestUtil.GREEN, null), //
+				new StyleRange(15, 4, SemanticTokensTestUtil.GREEN, null), //
+				new StyleRange(24, 7, SemanticTokensTestUtil.GREEN, null)//
+		);
+		assertArrayEquals(expectedStyleRanges.toArray(), styleRanges);
+	}
+}

org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensDataStreamProcessorTest.java

@@ -0,0 +1,54 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Avaloq Group AG.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.lsp4e.test.semanticTokens;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import org.eclipse.jface.text.Document;
+import org.eclipse.lsp4e.operations.semanticTokens.SemanticTokensDataStreamProcessor;
+import org.eclipse.lsp4e.test.AllCleanRule;
+import org.eclipse.lsp4j.SemanticTokensLegend;
+import org.eclipse.swt.custom.StyleRange;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class SemanticTokensDataStreamProcessorTest {
+	@Rule
+	public AllCleanRule clear = new AllCleanRule();
+
+	@Test
+	public void testKeyword() throws InterruptedException, ExecutionException {
+		Document document = new Document(SemanticTokensTestUtil.keywordText);
+
+		SemanticTokensDataStreamProcessor processor = new SemanticTokensDataStreamProcessor(SemanticTokensTestUtil
+				.keywordTokenTypeMapper(SemanticTokensTestUtil.RED_TOKEN), SemanticTokensTestUtil.offsetMapper(document));
+
+		List<Integer> expectedStream = SemanticTokensTestUtil.keywordSemanticTokens();
+		List<StyleRange> expectedStyleRanges = Arrays.asList(//
+				new StyleRange(0, 4, SemanticTokensTestUtil.RED, null), //
+				new StyleRange(15, 4, SemanticTokensTestUtil.RED, null), //
+				new StyleRange(24, 7, SemanticTokensTestUtil.RED, null)//
+				);
+
+		List<StyleRange> styleRanges = processor.getStyleRanges(expectedStream, getSemanticTokensLegend());
+
+		assertEquals(expectedStyleRanges, styleRanges);
+	}
+
+	private SemanticTokensLegend getSemanticTokensLegend() {
+		SemanticTokensLegend semanticTokensLegend = new SemanticTokensLegend();
+		semanticTokensLegend.setTokenTypes(Arrays.asList("keyword","other"));
+		semanticTokensLegend.setTokenModifiers(Arrays.asList("obsolete"));
+		return semanticTokensLegend;
+	}
+}

org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensLegendProviderTest.java

@@ -0,0 +1,63 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Avaloq Group AG.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.lsp4e.test.semanticTokens;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.lsp4e.LanguageServiceAccessor;
+import org.eclipse.lsp4e.operations.semanticTokens.SemanticHighlightReconcilerStrategy;
+import org.eclipse.lsp4e.test.AllCleanRule;
+import org.eclipse.lsp4e.test.TestUtils;
+import org.eclipse.lsp4j.SemanticTokensLegend;
+import org.eclipse.lsp4j.services.LanguageServer;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class SemanticTokensLegendProviderTest {
+
+	@Rule
+	public AllCleanRule clear = new AllCleanRule();
+
+	private IProject project;
+
+	@Before
+	public void setUp() throws CoreException {
+		project = TestUtils.createProject(getClass().getName() + System.currentTimeMillis());
+	}
+
+	@Test
+	public void testSemanticTokensLegendProvider() throws BadLocationException, CoreException, IOException, InterruptedException, ExecutionException {
+		// Setup Server Capabilities
+		List<String> tokenTypes = Arrays.asList("keyword","other");
+		List<String> tokenModifiers = Arrays.asList("obsolete");
+		SemanticTokensTestUtil.setSemanticTokensLegend(tokenTypes, tokenModifiers);
+
+		// Setup test data
+		IFile file = TestUtils.createUniqueTestFile(project, "lspt", "test content");
+		// start the LS
+		LanguageServer languageServer = LanguageServiceAccessor.getInitializedLanguageServers(file, c -> Boolean.TRUE).iterator()
+		.next().get();
+
+		SemanticTokensLegend semanticTokensLegend = (new SemanticHighlightReconcilerStrategy()).getSemanticTokensLegend(languageServer);
+		assertNotNull(semanticTokensLegend);
+		assertEquals(tokenTypes, semanticTokensLegend.getTokenTypes());
+		assertEquals(tokenModifiers, semanticTokensLegend.getTokenModifiers());
+	}
+}

org.eclipse.lsp4e.test/src/org/eclipse/lsp4e/test/semanticTokens/SemanticTokensTestUtil.java

@@ -0,0 +1,130 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Avaloq Group AG.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.lsp4e.test.semanticTokens;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+
+import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.TextAttribute;
+import org.eclipse.jface.text.rules.IToken;
+import org.eclipse.lsp4e.LSPEclipseUtils;
+import org.eclipse.lsp4e.tests.mock.MockLanguageServer;
+import org.eclipse.lsp4j.Position;
+import org.eclipse.lsp4j.SemanticTokensLegend;
+import org.eclipse.lsp4j.SemanticTokensWithRegistrationOptions;
+import org.eclipse.swt.graphics.Color;
+
+public class SemanticTokensTestUtil {
+	public static final String keywordText =
+			"type foo {\n" +
+			"	\n" +
+			"}\n" +
+			"type bar extends foo {\n" +
+			"	\n" +
+			"}\n";
+
+	public static List<Integer> keywordSemanticTokens() {
+		List<List<Integer>> expectedTokens = new ArrayList<>();
+		expectedTokens.add(Arrays.asList(0,0,4,1,0));
+		expectedTokens.add(Arrays.asList(3,0,4,1,0));
+		expectedTokens.add(Arrays.asList(0,9,7,1,0));
+
+		return expectedTokens.stream().flatMap(List::stream).toList();
+	}
+
+	public static final Color GREEN = new Color(133, 153, 0, 255);
+	public static final Color RED = new Color(255, 0, 0);
+
+	public static final IToken GREEN_TOKEN = new IToken() {
+		@Override
+		public boolean isWhitespace() {
+			return false;
+		}
+
+		@Override
+		public boolean isUndefined() {
+			return false;
+		}
+
+		@Override
+		public boolean isOther() {
+			return false;
+		}
+
+		@Override
+		public boolean isEOF() {
+			return false;
+		}
+
+		@Override
+		public Object getData() {
+			return new TextAttribute(GREEN);
+		}
+	};
+
+	public static final IToken RED_TOKEN = new IToken() {
+		@Override
+		public boolean isWhitespace() {
+			return false;
+		}
+
+		@Override
+		public boolean isUndefined() {
+			return false;
+		}
+
+		@Override
+		public boolean isOther() {
+			return false;
+		}
+
+		@Override
+		public boolean isEOF() {
+			return false;
+		}
+
+		@Override
+		public Object getData() {
+			return new TextAttribute(RED);
+		}
+	};
+
+	public static Function<String, IToken> keywordTokenTypeMapper(final IToken token) {
+		return t -> {
+			if ("keyword".equals(t)) {
+				return token;
+			} else {
+				return null;
+			}
+		};
+	}
+
+	public static @NonNull Function<Position, Integer> offsetMapper(IDocument document) {
+		return (p) -> {
+			try {
+				return LSPEclipseUtils.toOffset(p, document);
+			} catch (BadLocationException e) {
+				throw new RuntimeException(e);
+			}
+		};
+	}
+	public static void setSemanticTokensLegend(final List<String> tokenTypes, List<String> tokenModifiers) {
+		SemanticTokensLegend legend = new SemanticTokensLegend(tokenTypes, tokenModifiers);
+		SemanticTokensWithRegistrationOptions semanticTokensWithRegistrationOptions = new SemanticTokensWithRegistrationOptions(legend);
+		semanticTokensWithRegistrationOptions.setFull(true);
+		semanticTokensWithRegistrationOptions.setRange(false);
+
+		MockLanguageServer.INSTANCE.getInitializeResult().getCapabilities().setSemanticTokensProvider(semanticTokensWithRegistrationOptions);
+	}
+}