Error recovery for Rascal

package.sh

@@ -2,5 +2,19 @@
 
 set -euxo pipefail
 
-(cd rascal-vscode-extension && npx vsce package )
+cd rascal-vscode-extension
 
+TEMP1=$(mktemp)
+TEMP2=$(mktemp)
+cp package.json $TEMP1
+cp package-lock.json $TEMP2
+
+VERSION=$(node -p "require('./package.json').version")
+PREFIX=$(echo $VERSION | cut -d "-" -f 1)
+SUFFIX=$(git log --pretty=format:'%h' -n 1)
+npx vsce package $PREFIX-$SUFFIX
+
+cp $TEMP1 package.json
+cp $TEMP2 package-lock.json
+
+cd -

rascal-lsp/pom.xml

@@ -65,7 +65,7 @@
     <dependency>
       <groupId>org.rascalmpl</groupId>
       <artifactId>rascal</artifactId>
-      <version>0.40.17</version>
+      <version>0.40.17-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.rascalmpl</groupId>

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/TextDocumentState.java

@@ -26,14 +26,37 @@
  */
 package org.rascalmpl.vscode.lsp;
 
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.TimeUnit;
 import java.util.function.BiFunction;
+import java.util.function.Function;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.eclipse.lsp4j.Diagnostic;
+import org.eclipse.lsp4j.DiagnosticSeverity;
+import org.eclipse.lsp4j.Position;
+import org.eclipse.lsp4j.Range;
+import org.rascalmpl.library.util.ErrorRecovery;
+import org.rascalmpl.parser.gtd.exception.ParseError;
+import org.rascalmpl.values.RascalValueFactory;
+import org.rascalmpl.values.ValueFactoryFactory;
 import org.rascalmpl.values.parsetrees.ITree;
+import org.rascalmpl.vscode.lsp.util.Diagnostics;
 import org.rascalmpl.vscode.lsp.util.Versioned;
+import org.rascalmpl.vscode.lsp.util.concurrent.DebouncedSupplier;
+import org.rascalmpl.vscode.lsp.util.locations.ColumnMaps;
 
+import io.usethesource.vallang.IList;
 import io.usethesource.vallang.ISourceLocation;
+import io.usethesource.vallang.IValue;
 
 /**
  * TextDocumentState encapsulates the current contents of every open file editor,
@@ -45,64 +68,179 @@
  * and ParametricTextDocumentService.
  */
 public class TextDocumentState {
+    private static final Logger logger = LogManager.getLogger(TextDocumentState.class);
+
     private final BiFunction<ISourceLocation, String, CompletableFuture<ITree>> parser;
+    private final ISourceLocation location;
+    private final ColumnMaps columns;
+
+    @SuppressWarnings("java:S3077") // Visibility of writes is enough
+    private volatile Update current;
+    private final DebouncedSupplier<Update> parseAndGetCurrentDebouncer;
+
+    private final AtomicReference<@MonotonicNonNull Versioned<ITree>> lastWithoutErrors;
+    private final AtomicReference<@MonotonicNonNull Versioned<ITree>> last;
 
-    private final ISourceLocation file;
-    @SuppressWarnings("java:S3077") // we are use volatile correctly
-    private volatile Versioned<String> currentContent;
-    @SuppressWarnings("java:S3077") // we are use volatile correctly
-    private volatile @MonotonicNonNull Versioned<ITree> lastFullTree;
-    @SuppressWarnings("java:S3077") // we are use volatile correctly
-    private volatile CompletableFuture<Versioned<ITree>> currentTree;
+    public TextDocumentState(
+            BiFunction<ISourceLocation, String, CompletableFuture<ITree>> parser,
+            ISourceLocation location, ColumnMaps columns,
+            int initialVersion, String initialContent) {
 
-    public TextDocumentState(BiFunction<ISourceLocation, String, CompletableFuture<ITree>> parser, ISourceLocation file, int initialVersion, String initialContent) {
         this.parser = parser;
-        this.file = file;
-        this.currentContent = new Versioned<>(initialVersion, initialContent);
-        this.currentTree = newTreeAsync(initialVersion, initialContent);
+        this.location = location;
+        this.columns = columns;
+
+        this.current = new Update(initialVersion, initialContent);
+        this.parseAndGetCurrentDebouncer = new DebouncedSupplier<>(this::parseAndGetCurrent);
+        this.lastWithoutErrors = new AtomicReference<>();
+        this.last = new AtomicReference<>();
     }
 
-    /**
-     * The current call of this method guarantees that, until the next call,
-     * each intermediate call of `getCurrentTreeAsync` returns (a future for) a
-     * *correct* versioned tree. This means that:
-     *   - the version of the tree is parameter `version`;
-     *   - the tree is produced by parsing parameter `content`.
-     *
-     * Thus, callers of `getCurrentTreeAsync` are guaranteed to obtain a
-     * consistent <version, tree> pair.
-     */
-    public CompletableFuture<Versioned<ITree>> update(int version, String content) {
-        currentContent = new Versioned<>(version, content);
-        var newTree = newTreeAsync(version, content);
-        currentTree = newTree;
-        return newTree;
+    public ISourceLocation getLocation() {
+        return location;
     }
 
-    @SuppressWarnings("java:S1181") // we want to catch all Java exceptions from the parser
-    private CompletableFuture<Versioned<ITree>> newTreeAsync(int version, String content) {
-        return parser.apply(file, content)
-            .thenApply(t -> new Versioned<ITree>(version, t))
-            .whenComplete((r, t) -> {
-                if (r != null) {
-                    lastFullTree = r;
-                }
-            });
+    public void update(int version, String content) {
+        current = new Update(version, content);
+        // The creation of the `Update` object doesn't trigger the parser yet.
+        // This happens only when the tree or diagnostics are requested.
+    }
+
+    public Versioned<String> getCurrentContent() {
+        return current.getContent();
     }
 
     public CompletableFuture<Versioned<ITree>> getCurrentTreeAsync() {
-        return currentTree;
+        return getCurrentTreeAsync(Duration.ZERO);
     }
 
-    public @MonotonicNonNull Versioned<ITree> getMostRecentTree() {
-        return lastFullTree;
+    public CompletableFuture<Versioned<ITree>> getCurrentTreeAsync(Duration delay) {
+        return parseAndGetCurrent(delay)
+            .thenApply(Update::getTreeAsync)
+            .thenCompose(Function.identity());
     }
 
-    public ISourceLocation getLocation() {
-        return file;
+    public CompletableFuture<Versioned<List<Diagnostic>>> getCurrentDiagnosticsAsync() {
+        return getCurrentDiagnosticsAsync(Duration.ZERO);
     }
 
-    public Versioned<String> getCurrentContent() {
-        return currentContent;
+    public CompletableFuture<Versioned<List<Diagnostic>>> getCurrentDiagnosticsAsync(Duration delay) {
+        return parseAndGetCurrent(delay)
+            .thenApply(Update::getDiagnosticsAsync)
+            .thenCompose(Function.identity());
+    }
+
+    public @MonotonicNonNull Versioned<ITree> getLastTree() {
+        return last.get();
+    }
+
+    public @MonotonicNonNull Versioned<ITree> getLastTreeWithoutErrors() {
+        return lastWithoutErrors.get();
+    }
+
+    private CompletableFuture<Update> parseAndGetCurrent() {
+        var update = current;
+        update.parseIfNotParsing();
+        return CompletableFuture.completedFuture(update);
+    }
+
+    private CompletableFuture<Update> parseAndGetCurrent(Duration delay) {
+        var update = current;
+        if (update.isParsing()) {
+            return CompletableFuture.completedFuture(update);
+        } else {
+            return parseAndGetCurrentDebouncer.get(delay);
+        }
+    }
+
+    private class Update {
+        private final int version;
+        private final String content;
+        private final CompletableFuture<Versioned<ITree>> treeAsync;
+        private final CompletableFuture<Versioned<List<Diagnostic>>> diagnosticsAsync;
+        private final AtomicBoolean parsing;
+
+        public Update(int version, String content) {
+            this.version = version;
+            this.content = content;
+            this.treeAsync = new CompletableFuture<>();
+            this.diagnosticsAsync = new CompletableFuture<>();
+            this.parsing = new AtomicBoolean(false);
+        }
+
+        public Versioned<String> getContent() {
+            return new Versioned<>(version, content);
+        }
+
+        public CompletableFuture<Versioned<ITree>> getTreeAsync() {
+            parseIfNotParsing();
+            return treeAsync;
+        }
+
+        public CompletableFuture<Versioned<List<Diagnostic>>> getDiagnosticsAsync() {
+            parseIfNotParsing();
+            return diagnosticsAsync;
+        }
+
+        public boolean isParsing() {
+            return parsing.get();
+        }
+
+        private void parseIfNotParsing() {
+            if (parsing.compareAndSet(false, true)) {
+                parser
+                    .apply(location, content)
+                    .whenComplete((t, e) -> {
+
+                        // Prepare result values for futures
+                        var tree = new Versioned<>(version, t);
+                        var diagnostics = new Versioned<>(version, toDiagnostics(t, e));
+
+                        // Complete future to get the tree
+                        if (t == null) {
+                            treeAsync.completeExceptionally(e);
+                        } else {
+                            treeAsync.complete(tree);
+                            Versioned.replaceIfNewer(last, tree);
+                            if (diagnostics.get().isEmpty()) {
+                                Versioned.replaceIfNewer(lastWithoutErrors, tree);
+                            }
+                        }
+
+                        // Complete future to get diagnostics
+                        diagnosticsAsync.complete(diagnostics);
+                    });
+            }
+        }
+
+        private List<Diagnostic> toDiagnostics(ITree tree, Throwable excp) {
+            List<Diagnostic> parseErrors = new ArrayList<>();
+
+            if (excp instanceof CompletionException) {
+                excp = excp.getCause();
+            }
+
+            if (excp instanceof ParseError) {
+                parseErrors.add(Diagnostics.translateDiagnostic((ParseError)excp, columns));
+            } else if (excp != null) {
+                logger.error("Parsing crashed", excp);
+                parseErrors.add(new Diagnostic(
+                    new Range(new Position(0,0), new Position(0,1)),
+                    "Parsing failed: " + excp.getMessage(),
+                    DiagnosticSeverity.Error,
+                    "Rascal Parser"));
+            }
+
+            if (tree != null) {
+                RascalValueFactory valueFactory = (RascalValueFactory) ValueFactoryFactory.getValueFactory();
+                IList errors = new ErrorRecovery(valueFactory).findAllErrors(tree);
+                for (IValue error : errors) {
+                    ITree errorTree = (ITree) error;
+                    parseErrors.add(Diagnostics.translateErrorRecoveryDiagnostic(errorTree, columns));
+                }
+            }
+
+            return parseErrors;
+        }
     }
 }

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java

@@ -278,7 +278,8 @@ private void triggerBuilder(TextDocumentIdentifier doc) {
     private TextDocumentState updateContents(VersionedTextDocumentIdentifier doc, String newContents) {
         TextDocumentState file = getFile(doc);
         logger.trace("New contents for {}", doc);
-        handleParsingErrors(file, file.update(doc.getVersion(), newContents));
+        file.update(doc.getVersion(), newContents);
+        handleParsingErrors(file, file.getCurrentTreeAsync()); // Warning: Might be a later version (when a concurrent update happened)
         return file;
     }
 
@@ -334,7 +335,7 @@ public CompletableFuture<List<InlayHint>> inlayHint(InlayHintParams params) {
         final TextDocumentState file = getFile(params.getTextDocument());
         final ILanguageContributions contrib = contributions(params.getTextDocument());
         return recoverExceptions(
-                recoverExceptions(file.getCurrentTreeAsync(), file::getMostRecentTree)
+                recoverExceptions(file.getCurrentTreeAsync(), file::getLastTreeWithoutErrors)
                 .thenApply(Versioned::get)
                 .thenApply(contrib::inlayHint)
                 .thenCompose(InterruptibleFuture::get)
@@ -441,7 +442,7 @@ private ParametricFileFacts facts(String doc) {
 
     private TextDocumentState open(TextDocumentItem doc) {
         return files.computeIfAbsent(Locations.toLoc(doc),
-            l -> new TextDocumentState(contributions(doc)::parsing, l, doc.getVersion(), doc.getText())
+            l -> new TextDocumentState(contributions(doc)::parsing, l, columns, doc.getVersion(), doc.getText())
         );
     }
 

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParserOnlyContribution.java

@@ -96,7 +96,7 @@ private static Either<IFunction, Exception> loadParser(ParserSpecification spec)
         try {
             logger.debug("Loading parser {} at {}", reifiedType, spec.getParserLocation());
             // this hides all the loading and instantiation details of Rascal-generated parsers
-            var parser = vf.loadParser(reifiedType, spec.getParserLocation(), VF.bool(spec.getAllowAmbiguity()), VF.bool(false), VF.bool(false), vf.set());
+            var parser = vf.loadParser(reifiedType, spec.getParserLocation(), VF.bool(spec.getAllowAmbiguity()), VF.bool(false), VF.bool(false), VF.bool(false), vf.set());
             logger.debug("Got parser: {}", parser);
             return Either.forLeft(parser);
         }

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java

@@ -52,6 +52,7 @@
 import org.eclipse.lsp4j.jsonrpc.messages.ResponseErrorCode;
 import org.rascalmpl.exceptions.Throw;
 import org.rascalmpl.interpreter.Evaluator;
+import org.rascalmpl.library.util.ErrorRecovery;
 import org.rascalmpl.interpreter.env.ModuleEnvironment;
 import org.rascalmpl.library.util.PathConfig;
 import org.rascalmpl.values.IRascalValueFactory;
@@ -83,6 +84,7 @@
 
 public class RascalLanguageServices {
     private static final IValueFactory VF = IRascalValueFactory.getInstance();
+    private static final ErrorRecovery RECOVERY = new ErrorRecovery(IRascalValueFactory.getInstance());
 
     private static final Logger logger = LogManager.getLogger(RascalLanguageServices.class);
 
@@ -247,8 +249,13 @@ public List<CodeLensSuggestion> locateCodeLenses(ITree tree) {
         List<CodeLensSuggestion> result = new ArrayList<>(2);
         result.add(new CodeLensSuggestion(module, "Import in new Rascal terminal", "rascalmpl.importModule", moduleName));
 
-        for (IValue topLevel : TreeAdapter
-            .getListASTArgs(TreeAdapter.getArg(TreeAdapter.getArg(tree, "body"), "toplevels"))) {
+        ITree body = TreeAdapter.getArg(tree, "body");
+        ITree toplevels = TreeAdapter.getArg(body, "toplevels");
+        for (IValue topLevel : TreeAdapter.getListASTArgs(toplevels)) {
+            if (RECOVERY.hasErrors((ITree) topLevel)) {
+                continue;
+            }
+
             ITree decl = TreeAdapter.getArg((ITree) topLevel, "declaration");
             if ("function".equals(TreeAdapter.getConstructorName(decl))) {
                 ITree signature = TreeAdapter.getArg(TreeAdapter.getArg(decl, "functionDeclaration"), "signature");