SLE-900: Speed up indexing by excluding unrelated files

org.sonarlint.eclipse.core/META-INF/MANIFEST.MF

@@ -14,6 +14,7 @@ Export-Package: org.sonarlint.eclipse.core,
  org.sonarlint.eclipse.core.internal;x-friends:="org.sonarlint.eclipse.core.tests,org.sonarlint.eclipse.ui",
  org.sonarlint.eclipse.core.internal.adapter;x-friends:="org.sonarlint.eclipse.ui",
  org.sonarlint.eclipse.core.internal.backend;x-friends:="org.sonarlint.eclipse.ui,org.sonarlint.eclipse.core.tests",
+ org.sonarlint.eclipse.core.internal.cache;x-friends:="org.sonarlint.eclipse.ui",
  org.sonarlint.eclipse.core.internal.engine;x-friends:="org.sonarlint.eclipse.ui,org.sonarlint.eclipse.core.tests",
  org.sonarlint.eclipse.core.internal.engine.connected;x-friends:="org.sonarlint.eclipse.ui,org.sonarlint.eclipse.core.tests",
  org.sonarlint.eclipse.core.internal.event;x-friends:="org.sonarlint.eclipse.ui",

org.sonarlint.eclipse.core/plugin.xml

@@ -11,6 +11,9 @@
    <extension-point id="projectHierarchyProvider"
    					name="SonarLint Project Hierarchy Provider"
    					schema="schema/projectHierarchyProvider.exsd" />
+   <extension-point id="projectScopeProvider"
+   					name="SonarLint Project Scope Provider"
+   					schema="schema/projectScopeProvider.exsd" />
 
 
    <extension

org.sonarlint.eclipse.core/schema/projectScopeProvider.exsd

@@ -0,0 +1,86 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!-- Schema file written by PDE -->
+<schema targetNamespace="org.sonarlint.eclipse.core" xmlns="http://www.w3.org/2001/XMLSchema">
+<annotation>
+      <appInfo>
+         <meta.schema plugin="org.sonarlint.eclipse.core" id="projectScopeProvider" name="SonarLint Project Scope Provider"/>
+      </appInfo>
+      <documentation>
+         Used for indexing projects and narrowing down the focus for SonarLint to improve performance and lower the memory footprint by removing irrelevant resources from the analysis.
+      </documentation>
+   </annotation>
+
+   <element name="extension">
+      <annotation>
+         <appInfo>
+            <meta.element />
+         </appInfo>
+      </annotation>
+      <complexType>
+         <sequence minOccurs="0" maxOccurs="unbounded">
+            <element ref="participant"/>
+         </sequence>
+         <attribute name="point" type="string" use="required">
+            <annotation>
+               <documentation>
+
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="id" type="string">
+            <annotation>
+               <documentation>
+
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="name" type="string">
+            <annotation>
+               <documentation>
+
+               </documentation>
+               <appInfo>
+                  <meta.attribute translatable="true"/>
+               </appInfo>
+            </annotation>
+         </attribute>
+      </complexType>
+   </element>
+
+   <element name="participant">
+      <complexType>
+         <attribute name="class" type="string" use="required">
+            <annotation>
+               <documentation>
+
+               </documentation>
+               <appInfo>
+                  <meta.attribute kind="java" basedOn=":org.sonarlint.eclipse.core.resource.IProjectScopeProvider"/>
+               </appInfo>
+            </annotation>
+         </attribute>
+      </complexType>
+   </element>
+
+   <annotation>
+      <appInfo>
+         <meta.section type="since"/>
+      </appInfo>
+      <documentation>
+         10.6
+      </documentation>
+   </annotation>
+
+   <annotation>
+      <appInfo>
+         <meta.section type="examples"/>
+      </appInfo>
+      <documentation>
+         Please take a look at the &quot;SonarLint for Eclipse JDT&quot; bundle for a reference implementation.
+      </documentation>
+   </annotation>
+
+
+
+
+</schema>

org.sonarlint.eclipse.core/src/org/sonarlint/eclipse/core/SonarLintLogger.java

@@ -105,4 +105,13 @@ public void traceIdeMessage(String msg) {
       listener.traceIdeMessage(msg);
     }
   }
+
+  public void traceIdeMessage(String msg, Throwable t) {
+    for (var listener : logListeners) {
+      listener.traceIdeMessage(msg);
+      var stack = new StringWriter();
+      t.printStackTrace(new PrintWriter(stack));
+      listener.traceIdeMessage(stack.toString());
+    }
+  }
 }

org.sonarlint.eclipse.core/src/org/sonarlint/eclipse/core/internal/backend/FileSystemSynchronizer.java

@@ -27,19 +27,23 @@
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 import org.eclipse.core.filesystem.EFS;
+import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.IResourceChangeEvent;
 import org.eclipse.core.resources.IResourceChangeListener;
 import org.eclipse.core.resources.IResourceDelta;
 import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.jdt.annotation.Nullable;
 import org.sonarlint.eclipse.core.SonarLintLogger;
 import org.sonarlint.eclipse.core.analysis.SonarLintLanguage;
+import org.sonarlint.eclipse.core.internal.cache.IProjectScopeProviderCache;
 import org.sonarlint.eclipse.core.internal.extension.SonarLintExtensionTracker;
 import org.sonarlint.eclipse.core.internal.jobs.TestFileClassifier;
 import org.sonarlint.eclipse.core.internal.utils.SonarLintUtils;
@@ -76,6 +80,11 @@ public void resourceChanged(IResourceChangeEvent event) {
       SonarLintLogger.get().error(e.getMessage(), e);
     }
 
+    // When there was no valuable resource changed, then we don't have to do anything else!
+    if (changedOrAddedFiles.isEmpty() && removedFiles.isEmpty()) {
+      return;
+    }
+
     var job = new Job("SonarLint - Propagate FileSystem changes") {
       @Override
       protected IStatus run(IProgressMonitor monitor) {
@@ -124,52 +133,73 @@ protected IStatus run(IProgressMonitor monitor) {
 
   private static boolean visitDeltaPostChange(IResourceDelta delta, List<ISonarLintFile> changedOrAddedFiles, List<URI> removedFiles) {
     var res = delta.getResource();
-    switch (delta.getKind()) {
-      case IResourceDelta.ADDED:
-        var slFile = SonarLintUtils.adapt(res, ISonarLintFile.class,
-          "[FileSystemSynchronizer#visitDeltaPostChange] Try get file from event '" + res + "' (added)");
-
-        // INFO: When importing projects all files are considered to be "added", so don't suggest connections twice by
-        // providing SLCORE with the "added" SonarLint configuration files besides the
-        // SonarLintEclipseRpcClient.listFiles(...)!
-        if (slFile != null && !matchesSonarLintConfigurationFiles(slFile)) {
-          SonarLintLogger.get().debug("File added: " + slFile.getName());
-          changedOrAddedFiles.add(slFile);
-        }
-        break;
-      case IResourceDelta.REMOVED:
-        var fileUri = res.getLocationURI();
-        if (fileUri != null) {
-          removedFiles.add(fileUri);
-          SonarLintLogger.get().debug("File removed: " + fileUri);
-        }
-        break;
-      case IResourceDelta.CHANGED:
-        var changedSlFile = SonarLintUtils.adapt(res, ISonarLintFile.class,
-          "[FileSystemSynchronizer#visitDeltaPostChange] Try get file from event '" + res + "' (changed)");
-        if (changedSlFile != null) {
-          var interestingChangeForSlBackend = false;
-          var flags = delta.getFlags();
-          if ((flags & IResourceDelta.CONTENT) != 0) {
-            interestingChangeForSlBackend = true;
-            SonarLintLogger.get().debug("File content changed: " + changedSlFile.getName());
-          }
-          if ((flags & IResourceDelta.REPLACED) != 0) {
-            interestingChangeForSlBackend = true;
-            SonarLintLogger.get().debug("File content replaced: " + changedSlFile.getName());
-          }
-          if ((flags & IResourceDelta.ENCODING) != 0) {
-            interestingChangeForSlBackend = true;
-            SonarLintLogger.get().debug("File encoding changed: " + changedSlFile.getName());
-          }
-          if (interestingChangeForSlBackend) {
-            changedOrAddedFiles.add(changedSlFile);
-          }
-        }
-        break;
-      default:
-        break;
+    var fullPath = res.getFullPath();
+
+    // Immediately rule out files in the VCS and files related to Node.js "metadata" / storage. We don't care for these
+    // ones no matter if removed, changed, or added!
+    if (SonarLintUtils.insideVCSFolder(fullPath) || SonarLintUtils.isNodeJsRelated(fullPath)) {
+      return false;
+    }
+
+    if (delta.getKind() == IResourceDelta.REMOVED) {
+      // When something got removed, we don't care for exclusions and the adaption to ISonarLintFile. This happens not
+      // as often as adding or changing files to we will just let SLCORE handle it and don't "care" anymore about it.
+      var fileUri = res.getLocationURI();
+      if (fileUri != null) {
+        removedFiles.add(fileUri);
+        SonarLintLogger.get().debug("File removed: " + fileUri);
+      }
+      return true;
+    }
+
+    var slFile = SonarLintUtils.adapt(res, ISonarLintFile.class,
+      "[FileSystemSynchronizer#visitDeltaPostChange] Try get file from event '" + res + "' (added/changed)");
+    if (slFile == null) {
+      // Whatever happened here, try to dig deeper. If nothing is there, then okey - if there is, we can check anyway
+      // and care in the next iteration of this method with the child element.
+      return false;
     }
+
+    var project = slFile.getProject();
+    var configScopeId = ConfigScopeSynchronizer.getConfigScopeId(project);
+    var exclusions = IProjectScopeProviderCache.INSTANCE.getEntry(configScopeId);
+    if (exclusions == null) {
+      exclusions = getExclusions((IProject) project.getResource());
+      IProjectScopeProviderCache.INSTANCE.putEntry(configScopeId, exclusions);
+    }
+
+    // Compared to "DefaultSonarLintProjectAdapter#files" this is only on a resource delta, therefore we won't visit
+    // the folders containing the files that were added / changed. And therefore we have to iterate over the exclusions
+    // instead of just checking whether the "whole" path is in there (as it would be the case for a folder).
+    for (var exclusion : exclusions) {
+      if (SonarLintUtils.isChild(fullPath, exclusion)) {
+        return false;
+      }
+    }
+
+    if (delta.getKind() == IResourceDelta.ADDED) {
+      SonarLintLogger.get().debug("File added: " + slFile.getName());
+      changedOrAddedFiles.add(slFile);
+    } else if (delta.getKind() == IResourceDelta.CHANGED) {
+      var interestingChangeForSlBackend = false;
+      var flags = delta.getFlags();
+      if ((flags & IResourceDelta.CONTENT) != 0) {
+        interestingChangeForSlBackend = true;
+        SonarLintLogger.get().debug("File content changed: " + slFile.getName());
+      }
+      if ((flags & IResourceDelta.REPLACED) != 0) {
+        interestingChangeForSlBackend = true;
+        SonarLintLogger.get().debug("File content replaced: " + slFile.getName());
+      }
+      if ((flags & IResourceDelta.ENCODING) != 0) {
+        interestingChangeForSlBackend = true;
+        SonarLintLogger.get().debug("File encoding changed: " + slFile.getName());
+      }
+      if (interestingChangeForSlBackend) {
+        changedOrAddedFiles.add(slFile);
+      }
+    }
+
     return true;
   }
 
@@ -256,4 +286,12 @@ private static Language tryDetectLanguage(ISonarLintFile file) {
     }
     return language != null ? Language.valueOf(language.name()) : null;
   }
+
+  private static Set<IPath> getExclusions(IProject project) {
+    var exclusions = new HashSet<IPath>();
+    for (var projectScopeProvider : SonarLintExtensionTracker.getInstance().getProjectScopeProviders()) {
+      exclusions.addAll(projectScopeProvider.getExclusions(project));
+    }
+    return exclusions;
+  }
 }

org.sonarlint.eclipse.core/src/org/sonarlint/eclipse/core/internal/cache/AbstractConfigScopeIdCache.java

@@ -0,0 +1,77 @@
+/*
+ * SonarLint for Eclipse
+ * Copyright (C) 2015-2024 SonarSource SA
+ * [email protected]
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarlint.eclipse.core.internal.cache;
+
+import java.util.concurrent.ConcurrentHashMap;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.jdt.annotation.Nullable;
+import org.sonarlint.eclipse.core.internal.jobs.AbstractSonarJob;
+
+/**
+ *  This is the base for all caches that are linked to a configuration scope id. This can be both IProject and
+ *  ISonarLintProject as "configScopeId" is generated by `IProject#getLocationURI().toString()`. The actual cache
+ *  implementation will provide the duration for how long the cache is valid before the entry is removed
+ *
+ *  @param <T> values stored per configuration scope id
+ */
+public abstract class AbstractConfigScopeIdCache<T> {
+  private final ConcurrentHashMap<String, T> cache = new ConcurrentHashMap<>();
+
+  @Nullable
+  public synchronized T getEntry(String configScopeId) {
+    if (!cache.containsKey(configScopeId)) {
+      return null;
+    }
+    return cache.get(configScopeId);
+  }
+
+  public synchronized void putEntry(String configScopeId, T entry) {
+    cache.put(configScopeId, entry);
+    new DeleteCacheEntryJob<T>(this, configScopeId)
+      .schedule(getCacheDuration());
+  }
+
+  public synchronized void removeEntry(String configScopeId) {
+    cache.remove(configScopeId);
+  }
+
+  /** Should be a meaningful period based on the actual cache implementation */
+  protected abstract long getCacheDuration();
+
+  private static class DeleteCacheEntryJob<T> extends AbstractSonarJob {
+    private final AbstractConfigScopeIdCache<T> cache;
+    private final String configScopeId;
+
+    public DeleteCacheEntryJob(AbstractConfigScopeIdCache<T> cache, String configScopeId) {
+      super("Delete cache entry for: " + configScopeId);
+      this.cache = cache;
+      this.configScopeId = configScopeId;
+    }
+
+    @Override
+    protected IStatus doRun(IProgressMonitor monitor) throws CoreException {
+      cache.removeEntry(configScopeId);
+      return Status.OK_STATUS;
+    }
+  }
+}