Update textmate4eclipse, jcodings, joni

apache · Nov 17, 2024 · 17a6a99 · 17a6a99
1 parent 4c79ffb
commit 17a6a99
Show file tree

Hide file tree

Showing 17 changed files with 1,251 additions and 116 deletions.
diff --git a/ide/libs.jcodings/external/binaries-list b/ide/libs.jcodings/external/binaries-list
@@ -14,4 +14,4 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-E2C76A19F00128BB1806207E2989139BFB45F49D org.jruby.jcodings:jcodings:1.0.18
+DCE27159DC0382E5F7518D4F3E499FC8396357ED org.jruby.jcodings:jcodings:1.0.58
diff --git a/...ings/external/jcodings-1.0.18-license.txt → ...ings/external/jcodings-1.0.58-license.txt b/...ings/external/jcodings-1.0.18-license.txt → ...ings/external/jcodings-1.0.58-license.txt
@@ -1,5 +1,5 @@
 Name: jcodings
-Version: 1.0.18
+Version: 1.0.58
 Description: Java-based codings helper classes for Joni and JRuby
 License: MIT-nocopyright
 Origin: https://github.com/jruby/jcodings

diff --git a/ide/libs.jcodings/manifest.mf b/ide/libs.jcodings/manifest.mf
@@ -1,4 +1,4 @@
 Manifest-Version: 1.0
-OpenIDE-Module: org.netbeans.libs.jcodings/1
+OpenIDE-Module: org.netbeans.libs.jcodings/2
 OpenIDE-Module-Localizing-Bundle: org/netbeans/libs/jcodings/Bundle.properties
-OpenIDE-Module-Specification-Version: 0.16
+OpenIDE-Module-Specification-Version: 1.0.58
diff --git a/ide/libs.jcodings/nbproject/org-netbeans-libs-jcodings.sig b/ide/libs.jcodings/nbproject/org-netbeans-libs-jcodings.sig
diff --git a/ide/libs.jcodings/nbproject/project.properties b/ide/libs.jcodings/nbproject/project.properties
@@ -16,4 +16,4 @@
 # under the License.
 
 is.autoload=true
-release.external/jcodings-1.0.18.jar=modules/ext/jcodings-1.0.18.jar
+release.external/jcodings-1.0.58.jar=modules/ext/jcodings-1.0.58.jar
diff --git a/ide/libs.jcodings/nbproject/project.xml b/ide/libs.jcodings/nbproject/project.xml
@@ -29,8 +29,8 @@
                 <subpackages>org.jcodings</subpackages>
             </public-packages>
             <class-path-extension>
-                <runtime-relative-path>ext/jcodings-1.0.18.jar</runtime-relative-path>
-                <binary-origin>external/jcodings-1.0.18.jar</binary-origin>
+                <runtime-relative-path>ext/jcodings-1.0.58.jar</runtime-relative-path>
+                <binary-origin>external/jcodings-1.0.58.jar</binary-origin>
             </class-path-extension>
         </data>
     </configuration>

diff --git a/ide/lsp.client/src/org/netbeans/modules/lsp/client/options/LanguageStorage.java b/ide/lsp.client/src/org/netbeans/modules/lsp/client/options/LanguageStorage.java
@@ -27,15 +27,14 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import javax.swing.event.ChangeEvent;
-import org.eclipse.tm4e.core.registry.IRegistryOptions;
-import org.eclipse.tm4e.core.registry.Registry;
+import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammarReader;
+import org.eclipse.tm4e.core.registry.IGrammarSource;
 import org.netbeans.modules.textmate.lexer.TextmateTokenId;
 import org.netbeans.spi.navigator.NavigatorPanel;
 import org.openide.filesystems.FileObject;
@@ -214,23 +213,9 @@ static void store(List<LanguageDescription> languages) {
     }
 
     private static String findScope(File grammar) throws Exception {
-        IRegistryOptions opts = new IRegistryOptions() {
-            @Override
-            public String getFilePath(String scopeName) {
-                return null;
-            }
-            @Override
-            public InputStream getInputStream(String scopeName) throws IOException {
-                return null;
-            }
-            @Override
-            public Collection<String> getInjections(String scopeName) {
-                return null;
-            }
-        };
-        return new Registry(opts).loadGrammarFromPathSync(grammar).getScopeName();
+        return RawGrammarReader.readGrammar(IGrammarSource.fromFile(grammar.toPath())).getScopeName();
     }
-    
+
     public static class LanguageDescription {
 
         public String id;

diff --git a/ide/textmate.lexer/external/binaries-list b/ide/textmate.lexer/external/binaries-list
@@ -14,5 +14,5 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-655CC3ABA1BC9DBDD653F28937BEC16F3E9C4CEC org.jruby.joni:joni:2.1.11
-109763C0803F3EAC2372B4E4A9A0D7BED9CAB5BC eu.doppel-helix.netbeans.lib.tm4e:org.eclipse.tm4e.core:0.4.1-pack1
+23D2F2EFF7FA0CDA465D86EC9D8BAB53E496D9E6 org.jruby.joni:joni:2.2.1
+9234380A35AD5C0A707E850D24031AAA158D283A https://download.eclipse.org/tm4e/releases/0.14.0/plugins/org.eclipse.tm4e.core_0.14.0.202410282056.jar org.eclipse.tm4e.core-0.14.0.jar
diff --git a/...te.lexer/external/joni-2.1.11-license.txt → ...ate.lexer/external/joni-2.2.1-license.txt b/...te.lexer/external/joni-2.1.11-license.txt → ...ate.lexer/external/joni-2.2.1-license.txt
@@ -1,5 +1,5 @@
 Name: jcodings
-Version: 2.1.11
+Version: 2.2.1
 License: MIT-jruby
 Description: Needed by TextMate support for Eclipse
 Origin: https://github.com/jruby/joni

diff --git a/...eclipse.tm4e.core-0.4.1-pack1-license.txt → .../org.eclipse.tm4e.core-0.14.0-license.txt b/...eclipse.tm4e.core-0.4.1-pack1-license.txt → .../org.eclipse.tm4e.core-0.14.0-license.txt
@@ -1,5 +1,5 @@
 Name: TextMate support for Eclipse
-Version: 0.4.1-pack1
+Version: 0.14.0
 License: EPL-v20
 Description: Used to interpret TextMate grammars
 Origin: https://github.com/eclipse/tm4e

diff --git a/ide/textmate.lexer/nbproject/project.properties b/ide/textmate.lexer/nbproject/project.properties
@@ -15,10 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 is.autoload=true
-javac.source=1.8
+javac.release=17
 javac.compilerargs=-Xlint -Xlint:-serial
 javadoc.arch=${basedir}/arch.xml
 javadoc.apichanges=${basedir}/apichanges.xml
-release.external/joni-2.1.11.jar=modules/ext/joni-2.1.11.jar
-release.external/org.eclipse.tm4e.core-0.4.1-pack1.jar=modules/ext/org.eclipse.tm4e.core-0.4.1-pack1.jar
-spec.version.base=1.27.0
+release.external/joni-2.2.1.jar=modules/ext/joni-2.2.1.jar
+release.external/org.eclipse.tm4e.core-0.14.0.jar=modules/ext/org.eclipse.tm4e.core-0.14.0.jar
+spec.version.base=1.28.0
diff --git a/ide/textmate.lexer/nbproject/project.xml b/ide/textmate.lexer/nbproject/project.xml
@@ -42,8 +42,8 @@
                 <dependency>
                     <code-name-base>org.netbeans.libs.jcodings</code-name-base>
                     <run-dependency>
-                        <release-version>1</release-version>
-                        <specification-version>0.1</specification-version>
+                        <release-version>2</release-version>
+                        <specification-version>1.0.58</specification-version>
                     </run-dependency>
                 </dependency>
                 <dependency>
@@ -200,12 +200,12 @@
                 <package>org.netbeans.modules.textmate.lexer.api</package>
             </public-packages>
             <class-path-extension>
-                <runtime-relative-path>ext/joni-2.1.11.jar</runtime-relative-path>
-                <binary-origin>external/joni-2.1.11.jar</binary-origin>
+                <runtime-relative-path>ext/joni-2.2.1.jar</runtime-relative-path>
+                <binary-origin>external/joni-2.2.1.jar</binary-origin>
             </class-path-extension>
             <class-path-extension>
-                <runtime-relative-path>ext/org.eclipse.tm4e.core-0.4.1-pack1.jar</runtime-relative-path>
-                <binary-origin>external/org.eclipse.tm4e.core-0.4.1-pack1.jar</binary-origin>
+                <runtime-relative-path>ext/org.eclipse.tm4e.core-0.14.0.jar</runtime-relative-path>
+                <binary-origin>external/org.eclipse.tm4e.core-0.14.0.jar</binary-origin>
             </class-path-extension>
         </data>
     </configuration>

diff --git a/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/CreateRegistrationProcessor.java b/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/CreateRegistrationProcessor.java
@@ -21,7 +21,9 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Collection;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -37,8 +39,8 @@
 import javax.lang.model.element.ExecutableElement;
 import javax.lang.model.element.Name;
 import javax.lang.model.element.TypeElement;
-import org.eclipse.tm4e.core.registry.IRegistryOptions;
-import org.eclipse.tm4e.core.registry.Registry;
+import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammarReader;
+import org.eclipse.tm4e.core.registry.IGrammarSource;
 import org.openide.filesystems.annotations.LayerBuilder;
 import org.openide.filesystems.annotations.LayerGeneratingProcessor;
 import org.openide.filesystems.annotations.LayerGenerationException;
@@ -141,22 +143,21 @@ private void process(Element toRegister, AnnotationMirror mimeRegistration) thro
 
             LayerBuilder layer = layer(toRegister);
             javax.tools.FileObject file = layer.validateResource(grammar, toRegister, null, null, false);
-            try (InputStream in = file.openInputStream()) {
-                IRegistryOptions opts = new IRegistryOptions() {
+            try (InputStream in = file.openInputStream();
+                    InputStreamReader isr = new InputStreamReader(in, StandardCharsets.UTF_8)) {
+                String finalGrammar = grammar;
+                IGrammarSource referencedGrammar = new IGrammarSource() {
                     @Override
-                    public String getFilePath(String scopeName) {
-                        return null;
-                    }
-                    @Override
-                    public InputStream getInputStream(String scopeName) throws IOException {
-                        return null;
+                    public String getFilePath() {
+                        return finalGrammar;
                     }
+
                     @Override
-                    public Collection<String> getInjections(String scopeName) {
-                        return null;
+                    public Reader getReader() throws IOException {
+                        return isr;
                     }
                 };
-                String scopeName = new Registry(opts).loadGrammarFromPathSync(grammar, in).getScopeName();
+                String scopeName = RawGrammarReader.readGrammar(referencedGrammar).getScopeName();
                 String simpleName = grammar.lastIndexOf('/') != (-1) ? grammar.substring(grammar.lastIndexOf('/') + 1) : grammar;
                 layer.file("Editors" + mimeType + "/" + simpleName)
                      .url("nbresloc:/" + grammar)
@@ -191,22 +192,21 @@ private void processInjection(Element toRegister, AnnotationMirror injectionRegi
         if (injectTo != null && grammar != null) {
             LayerBuilder layer = layer(toRegister);
             javax.tools.FileObject file = layer.validateResource(grammar, toRegister, null, null, false);
-            try (InputStream in = file.openInputStream()) {
-                IRegistryOptions opts = new IRegistryOptions() {
+            try (InputStream in = file.openInputStream();
+                    InputStreamReader isr = new InputStreamReader(in, StandardCharsets.UTF_8)) {
+                String finalGrammar = grammar;
+                IGrammarSource referencedGrammar = new IGrammarSource() {
                     @Override
-                    public String getFilePath(String scopeName) {
-                        return null;
-                    }
-                    @Override
-                    public InputStream getInputStream(String scopeName) throws IOException {
-                        return null;
+                    public String getFilePath() {
+                        return finalGrammar;
                     }
+
                     @Override
-                    public Collection<String> getInjections(String scopeName) {
-                        return null;
+                    public Reader getReader() throws IOException {
+                        return isr;
                     }
                 };
-                String scopeName = new Registry(opts).loadGrammarFromPathSync(grammar, in).getScopeName();
+                String scopeName = RawGrammarReader.readGrammar(referencedGrammar).getScopeName();
                 String simpleName = grammar.lastIndexOf('/') != (-1) ? grammar.substring(grammar.lastIndexOf('/') + 1) : grammar;
                 layer.file("Editors" + "/" + simpleName)
                      .url("nbresloc:/" + grammar)

diff --git a/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/FileObjectGrammarSource.java b/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/FileObjectGrammarSource.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.netbeans.modules.textmate.lexer;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import org.eclipse.tm4e.core.registry.IGrammarSource;
+import org.openide.filesystems.FileObject;
+
+public class FileObjectGrammarSource implements IGrammarSource {
+
+    private final FileObject fileObject;
+
+    public FileObjectGrammarSource(FileObject fileObject) {
+        this.fileObject = fileObject;
+    }
+
+    @Override
+    public String getFilePath() {
+        return fileObject.getPath();
+    }
+
+    @Override
+    public Reader getReader() throws IOException {
+        return new InputStreamReader(fileObject.getInputStream(), StandardCharsets.UTF_8);
+    }
+
+}
diff --git a/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java b/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java
@@ -18,14 +18,15 @@
  */
 package org.netbeans.modules.textmate.lexer;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import org.eclipse.tm4e.core.grammar.IGrammar;
+import org.eclipse.tm4e.core.grammar.IStateStack;
 import org.eclipse.tm4e.core.grammar.IToken;
 import org.eclipse.tm4e.core.grammar.ITokenizeLineResult;
-import org.eclipse.tm4e.core.grammar.StackElement;
 import org.netbeans.api.lexer.Token;
 import org.netbeans.spi.lexer.Lexer;
 import org.netbeans.spi.lexer.LexerInput;
@@ -42,7 +43,7 @@ public class TextmateLexer implements Lexer<TextmateTokenId>{
     private int currentOffset;
     private List<IToken> lineTokens;
     private int currentIdx;
-    private StackElement state;
+    private IStateStack state;
     private boolean forceReadLine;
 
     public TextmateLexer(LexerInput li, Object state, TokenFactory<TextmateTokenId> factory, IGrammar grammar) {
@@ -58,7 +59,7 @@ public TextmateLexer(LexerInput li, Object state, TokenFactory<TextmateTokenId>
             this.state = istate.state;
             this.forceReadLine = true;
         } else {
-            this.state = (StackElement) state;
+            this.state = (IStateStack) state;
         }
     }
 
@@ -72,7 +73,7 @@ public Token<TextmateTokenId> nextToken() {
                 if (li.readLength() != 0) {
                     lineLen = li.readText().length();
                     currentOffset = 0;
-                    ITokenizeLineResult tokenized = grammar.tokenizeLine(li.readText().toString(), state);
+                    ITokenizeLineResult<IToken[]> tokenized = grammar.tokenizeLine(li.readText().toString(), state, Duration.ofMinutes(1));
                     lineTokens = new ArrayList<>(Arrays.asList(tokenized.getTokens()));
                     currentIdx = 0;
                     state = tokenized.getRuleStack();
@@ -124,9 +125,9 @@ private static final class IntralineState {
         private int currentOffset;
         private List<IToken> lineTokens;
         private int currentIdx;
-        private StackElement state;
+        private IStateStack state;
 
-        public IntralineState(int lineLen, int currentOffset, List<IToken> lineTokens, int currentIdx, StackElement state) {
+        public IntralineState(int lineLen, int currentOffset, List<IToken> lineTokens, int currentIdx, IStateStack state) {
             this.lineLen = lineLen;
             this.currentOffset = currentOffset;
             this.lineTokens = lineTokens;

diff --git a/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateTokenId.java b/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateTokenId.java
@@ -18,10 +18,9 @@
  */
 package org.netbeans.modules.textmate.lexer;
 
-import java.io.IOException;
-import java.io.InputStream;
 import java.lang.ref.Reference;
 import java.lang.ref.WeakReference;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -33,6 +32,7 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.eclipse.tm4e.core.grammar.IGrammar;
+import org.eclipse.tm4e.core.registry.IGrammarSource;
 import org.eclipse.tm4e.core.registry.IRegistryOptions;
 import org.eclipse.tm4e.core.registry.Registry;
 import org.netbeans.api.editor.mimelookup.MimePath;
@@ -160,19 +160,13 @@ public LanguageHierarchyImpl(String mimeType, String scope) throws Exception {
             this.mimeType = mimeType;
             IRegistryOptions opts = new IRegistryOptions() {
                 @Override
-                public String getFilePath(String scopeName) {
+                public IGrammarSource getGrammarSource(String scopeName) {
                     synchronized (LanguageHierarchyImpl.class) {
                         FileObject file = scope2File.get(scopeName);
-                        return file != null ? file.getNameExt() : null;
-                    }
-                }
-                @Override
-                public InputStream getInputStream(String scopeName) throws IOException {
-                    synchronized (LanguageHierarchyImpl.class) {
-                        FileObject file = scope2File.get(scopeName);
-                        return file != null ? file.getInputStream(): null;
+                        return file != null ? new FileObjectGrammarSource(file) : null;
                     }
                 }
+
                 @Override
                 public Collection<String> getInjections(String scopeName) {
                     synchronized (LanguageHierarchyImpl.class) {

diff --git a/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java b/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java
@@ -203,7 +203,7 @@ public void testUTF8() throws Exception {
         assertTokenProperties(ts, "test", "ident.test");
         LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "  ");
         assertTokenProperties(ts, "test");
-        LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "\n");
+        LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.UNTOKENIZED, "\n");
         assertFalse(ts.moveNext());
     }