SONARPY-2219 ProjectLevelSymbolTable descriptors creation out of V2 t…

…ypes

python-frontend/src/main/java/org/sonar/python/TestPythonVisitorRunner.java

@@ -93,6 +93,7 @@ public static ProjectLevelSymbolTable globalSymbols(List<File> files, File baseD
       var astRoot = parseFile(pythonFile);
       String packageName = pythonPackageName(file, baseDir.getAbsolutePath());
       projectLevelSymbolTable.addModule(astRoot, packageName, pythonFile);
+      projectLevelSymbolTable.addModuleV2(packageName, pythonFile, astRoot);
     }
     return projectLevelSymbolTable;
   }

python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java

@@ -24,6 +24,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -41,15 +42,25 @@
 import org.sonar.python.index.Descriptor;
 import org.sonar.python.index.DescriptorUtils;
 import org.sonar.python.index.VariableDescriptor;
+import org.sonar.python.semantic.v2.BasicTypeTable;
+import org.sonar.python.semantic.v2.SymbolTableBuilderV2;
+import org.sonar.python.semantic.v2.SymbolV2;
+import org.sonar.python.semantic.v2.TypeInferenceV2;
+import org.sonar.python.semantic.v2.UsageV2;
+import org.sonar.python.semantic.v2.converter.PythonTypeToDescriptorConverter;
 import org.sonar.python.semantic.v2.typeshed.TypeShedDescriptorsProvider;
+import org.sonar.python.types.v2.PythonType;
 
 import static org.sonar.python.tree.TreeUtils.getSymbolFromTree;
 import static org.sonar.python.tree.TreeUtils.nthArgumentOrKeyword;
 
 public class ProjectLevelSymbolTable {
 
+  private final PythonTypeToDescriptorConverter pythonTypeToDescriptorConverter = new PythonTypeToDescriptorConverter();
   private final Map<String, Set<Descriptor>> globalDescriptorsByModuleName;
+  private final Map<String, Set<Descriptor>> globalDescriptorsByModuleNameV2;
   private Map<String, Descriptor> globalDescriptorsByFQN;
+  private Map<String, Descriptor> globalDescriptorsByFQNV2;
   private final Set<String> djangoViewsFQN = new HashSet<>();
   private final Map<String, Set<String>> importsByModule = new HashMap<>();
   private final Set<String> projectBasePackages = new HashSet<>();
@@ -65,15 +76,19 @@ public static ProjectLevelSymbolTable from(Map<String, Set<Symbol>> globalSymbol
 
   public ProjectLevelSymbolTable() {
     this.globalDescriptorsByModuleName = new HashMap<>();
+    this.globalDescriptorsByModuleNameV2 = new HashMap<>();
   }
 
   private ProjectLevelSymbolTable(Map<String, Set<Symbol>> globalSymbolsByModuleName) {
     this.globalDescriptorsByModuleName = new HashMap<>();
+    this.globalDescriptorsByModuleNameV2 = new HashMap<>();
     globalSymbolsByModuleName.entrySet().forEach(entry -> {
       String moduleName = entry.getKey();
       Set<Symbol> symbols = entry.getValue();
       Set<Descriptor> globalDescriptors = symbols.stream().map(DescriptorUtils::descriptor).collect(Collectors.toSet());
       globalDescriptorsByModuleName.put(moduleName, globalDescriptors);
+      globalDescriptors = symbols.stream().map(DescriptorUtils::descriptor).collect(Collectors.toSet());
+      globalDescriptorsByModuleNameV2.put(moduleName, globalDescriptors);
     });
   }
 
@@ -82,6 +97,7 @@ public void removeModule(String packageName, String fileName) {
     globalDescriptorsByModuleName.remove(fullyQualifiedModuleName);
     // ensure globalDescriptorsByFQN is re-computed
     this.globalDescriptorsByFQN = null;
+    this.globalDescriptorsByFQNV2 = null;
   }
 
   public void addModule(FileInput fileInput, String packageName, PythonFile pythonFile) {
@@ -169,6 +185,11 @@ public Set<Descriptor> getDescriptorsFromModule(@Nullable String moduleName) {
     return globalDescriptorsByModuleName.get(moduleName);
   }
 
+  @CheckForNull
+  public Set<Descriptor> getDescriptorsFromModuleV2(@Nullable String moduleName) {
+    return globalDescriptorsByModuleNameV2.get(moduleName);
+  }
+
   public Map<String, Set<String>> importsByModule() {
     return Collections.unmodifiableMap(importsByModule);
   }
@@ -201,6 +222,33 @@ public TypeShedDescriptorsProvider typeShedDescriptorsProvider() {
     return typeShedDescriptorsProvider;
   }
 
+  public void addModuleV2(String fullyQualifiedModuleName, Set<Descriptor> moduleDescriptors) {
+    globalDescriptorsByModuleNameV2.put(fullyQualifiedModuleName, moduleDescriptors);
+  }
+
+  public void addModuleV2(String fullyQualifiedModuleName, Map<SymbolV2, Set<PythonType>> typesBySymbol) {
+    var moduleDescriptors = typesBySymbol.entrySet()
+      .stream()
+      .map(entry -> {
+          var descriptor = pythonTypeToDescriptorConverter.convert(fullyQualifiedModuleName, entry.getKey(), entry.getValue());
+          return Map.entry(entry.getKey(), descriptor);
+        }
+      )
+      .filter(entry -> !(!Objects.requireNonNull(entry.getValue().fullyQualifiedName()).startsWith(fullyQualifiedModuleName)
+        || entry.getKey().usages().stream().anyMatch(u -> u.kind().equals(UsageV2.Kind.IMPORT))))
+      .map(Map.Entry::getValue)
+      .collect(Collectors.toSet());
+    addModuleV2(fullyQualifiedModuleName, moduleDescriptors);
+  }
+
+  public void addModuleV2(String packageName, PythonFile pythonFile, FileInput astRoot) {
+    var fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName());
+    var symbolTable = new SymbolTableBuilderV2(astRoot).build();
+    var typeInferenceV2 = new TypeInferenceV2(new BasicTypeTable(), pythonFile, symbolTable);
+    var typesBySymbol = typeInferenceV2.inferTypes(astRoot);
+    addModuleV2(fullyQualifiedModuleName, typesBySymbol);
+  }
+
   private class DjangoViewsVisitor extends BaseTreeVisitor {
     @Override
     public void visitCallExpression(CallExpression callExpression) {

python-frontend/src/main/java/org/sonar/python/semantic/v2/BasicTypeTable.java

@@ -0,0 +1,46 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.semantic.v2;
+
+import java.util.List;
+import org.sonar.python.types.v2.PythonType;
+import org.sonar.python.types.v2.UnknownType;
+
+public class BasicTypeTable implements TypeTable {
+  @Override
+  public PythonType getBuiltinsModule() {
+    return new UnknownType.UnresolvedImportType("");
+  }
+
+  @Override
+  public PythonType getType(String typeFqn) {
+    return new UnknownType.UnresolvedImportType(typeFqn);
+  }
+
+  @Override
+  public PythonType getType(String... typeFqnParts) {
+    return new UnknownType.UnresolvedImportType(String.join(".", typeFqnParts));
+  }
+
+  @Override
+  public PythonType getType(List<String> typeFqnParts) {
+    return new UnknownType.UnresolvedImportType(String.join(".", typeFqnParts));
+  }
+}

python-frontend/src/main/java/org/sonar/python/semantic/v2/FunctionTypeBuilder.java

@@ -60,7 +60,7 @@ public class FunctionTypeBuilder implements TypeBuilder<FunctionType> {
   private static final String CLASS_METHOD_DECORATOR = "classmethod";
   private static final String STATIC_METHOD_DECORATOR = "staticmethod";
 
-  public FunctionTypeBuilder fromFunctionDef(FunctionDef functionDef, @Nullable String fileId, ProjectLevelTypeTable projectLevelTypeTable) {
+  public FunctionTypeBuilder fromFunctionDef(FunctionDef functionDef, @Nullable String fileId, TypeTable projectLevelTypeTable) {
     this.name = functionDef.name().name();
     this.attributes = new ArrayList<>();
     this.parameters = new ArrayList<>();
@@ -155,7 +155,7 @@ public FunctionTypeBuilder withOwner(PythonType owner) {
     return this;
   }
 
-  private void createParameterNames(List<AnyParameter> parameterTrees, @Nullable String fileId, ProjectLevelTypeTable projectLevelTypeTable) {
+  private void createParameterNames(List<AnyParameter> parameterTrees, @Nullable String fileId, TypeTable projectLevelTypeTable) {
     ParameterState parameterState = new ParameterState();
     parameterState.positionalOnly = parameterTrees.stream().anyMatch(param -> Optional.of(param)
       .filter(p -> p.is(Tree.Kind.PARAMETER))
@@ -174,7 +174,7 @@ private void createParameterNames(List<AnyParameter> parameterTrees, @Nullable S
     }
   }
 
-  private void addParameter(Parameter parameter, @Nullable String fileId, ParameterState parameterState, ProjectLevelTypeTable projectLevelTypeTable) {
+  private void addParameter(Parameter parameter, @Nullable String fileId, ParameterState parameterState, TypeTable projectLevelTypeTable) {
     Name parameterName = parameter.name();
     Token starToken = parameter.starToken();
     if (parameterName != null) {
@@ -197,7 +197,7 @@ private void addParameter(Parameter parameter, @Nullable String fileId, Paramete
     }
   }
 
-  private ParameterType getParameterType(Parameter parameter, ProjectLevelTypeTable projectLevelTypeTable) {
+  private ParameterType getParameterType(Parameter parameter, TypeTable projectLevelTypeTable) {
     boolean isPositionalVariadic = false;
     boolean isKeywordVariadic = false;
     Token starToken = parameter.starToken();

python-frontend/src/main/java/org/sonar/python/semantic/v2/LazyTypesContext.java

@@ -28,11 +28,11 @@
 
 public class LazyTypesContext {
   private final Map<String, LazyType> lazyTypes;
-  private final ProjectLevelTypeTable projectLevelTypeTable;
+  private final TypeTable typeTable;
 
-  public LazyTypesContext(ProjectLevelTypeTable projectLevelTypeTable) {
+  public LazyTypesContext(ProjectLevelTypeTable typeTable) {
     this.lazyTypes = new HashMap<>();
-    this.projectLevelTypeTable = projectLevelTypeTable;
+    this.typeTable = typeTable;
   }
 
   public TypeWrapper getOrCreateLazyTypeWrapper(String importPath) {
@@ -49,7 +49,7 @@ public LazyType getOrCreateLazyType(String importPath) {
   }
 
   public PythonType resolveLazyType(LazyType lazyType) {
-    PythonType resolved = projectLevelTypeTable.getType(lazyType.importPath());
+    PythonType resolved = typeTable.getType(lazyType.importPath());
     lazyType.resolve(resolved);
     lazyTypes.remove(lazyType.importPath());
     return resolved;

python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java

@@ -29,7 +29,7 @@
 import org.sonar.python.types.v2.PythonType;
 import org.sonar.python.types.v2.TypeWrapper;
 
-public class ProjectLevelTypeTable {
+public class ProjectLevelTypeTable implements TypeTable {
 
   private final SymbolsModuleTypeProvider symbolsModuleTypeProvider;
   private final ModuleType rootModule;
@@ -41,18 +41,22 @@ public ProjectLevelTypeTable(ProjectLevelSymbolTable projectLevelSymbolTable) {
     this.rootModule = this.symbolsModuleTypeProvider.createBuiltinModule();
   }
 
-  public ModuleType getBuiltinsModule() {
+  @Override
+  public PythonType getBuiltinsModule() {
     return rootModule;
   }
 
+  @Override
   public PythonType getType(String typeFqn) {
     return getType(typeFqn.split("\\."));
   }
 
+  @Override
   public PythonType getType(String... typeFqnParts) {
     return getType(List.of(typeFqnParts));
   }
 
+  @Override
   public PythonType getType(List<String> typeFqnParts) {
     var parent = (PythonType) rootModule;
     for (int i = 0; i < typeFqnParts.size(); i++) {

python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java

@@ -79,7 +79,7 @@ private static String getModuleFqnString(List<String> moduleFqn) {
   }
 
   private Optional<ModuleType> createModuleTypeFromProjectLevelSymbolTable(String moduleName, String moduleFqn, ModuleType parent) {
-    var retrieved = projectLevelSymbolTable.getDescriptorsFromModule(moduleFqn);
+    var retrieved = projectLevelSymbolTable.getDescriptorsFromModuleV2(moduleFqn);
     if (retrieved == null) {
       return Optional.empty();
     }

python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeInferenceV2.java

@@ -21,7 +21,6 @@
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -47,22 +46,21 @@
 
 public class TypeInferenceV2 {
 
-  private final ProjectLevelTypeTable projectLevelTypeTable;
+  private final TypeTable projectLevelTypeTable;
   private final SymbolTable symbolTable;
   private final PythonFile pythonFile;
-  private Map<SymbolV2, Set<PythonType>> typesBySymbol = new HashMap<>();
 
-  public TypeInferenceV2(ProjectLevelTypeTable projectLevelTypeTable, PythonFile pythonFile, SymbolTable symbolTable) {
+  public TypeInferenceV2(TypeTable projectLevelTypeTable, PythonFile pythonFile, SymbolTable symbolTable) {
     this.projectLevelTypeTable = projectLevelTypeTable;
     this.symbolTable = symbolTable;
     this.pythonFile = pythonFile;
   }
 
-  public void inferTypes(FileInput fileInput) {
+  public Map<SymbolV2, Set<PythonType>> inferTypes(FileInput fileInput) {
     TrivialTypeInferenceVisitor trivialTypeInferenceVisitor = new TrivialTypeInferenceVisitor(projectLevelTypeTable, pythonFile);
     fileInput.accept(trivialTypeInferenceVisitor);
 
-    inferTypesAndMemberAccessSymbols(fileInput);
+    var typesBySymbol = inferTypesAndMemberAccessSymbols(fileInput);
 
     fileInput.accept(new BaseTreeVisitor() {
       @Override
@@ -71,20 +69,17 @@ public void visitFunctionDef(FunctionDef funcDef) {
         inferTypesAndMemberAccessSymbols(funcDef);
       }
     });
-  }
-
-  public Map<SymbolV2, Set<PythonType>> getTypesBySymbol() {
     return typesBySymbol;
   }
 
-  private void inferTypesAndMemberAccessSymbols(FileInput fileInput) {
+  private Map<SymbolV2, Set<PythonType>> inferTypesAndMemberAccessSymbols(FileInput fileInput) {
     StatementList statements = fileInput.statements();
     if (statements == null) {
-      return;
+      return Map.of();
     }
     var moduleSymbols = symbolTable.getSymbolsByRootTree(fileInput);
 
-    typesBySymbol = inferTypesAndMemberAccessSymbols(
+    return inferTypesAndMemberAccessSymbols(
       fileInput,
       statements,
       moduleSymbols,

python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeTable.java

@@ -0,0 +1,33 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.semantic.v2;
+
+import java.util.List;
+import org.sonar.python.types.v2.PythonType;
+
+public interface TypeTable {
+  PythonType getBuiltinsModule();
+
+  PythonType getType(String typeFqn);
+
+  PythonType getType(String... typeFqnParts);
+
+  PythonType getType(List<String> typeFqnParts);
+}