From 99f28707c672a0d4d2c4dbc72749995e82c6825b Mon Sep 17 00:00:00 2001 From: Maksim Grebeniuk Date: Fri, 18 Oct 2024 15:44:39 +0200 Subject: [PATCH] SONARPY-2219 ProjectLevelSymbolTable descriptors creation out of V2 types --- .../sonar/python/TestPythonVisitorRunner.java | 1 + .../semantic/ProjectLevelSymbolTable.java | 48 ++++++ .../python/semantic/v2/BasicTypeTable.java | 46 ++++++ .../semantic/v2/FunctionTypeBuilder.java | 8 +- .../python/semantic/v2/LazyTypesContext.java | 8 +- .../semantic/v2/ProjectLevelTypeTable.java | 8 +- .../v2/SymbolsModuleTypeProvider.java | 2 +- .../python/semantic/v2/TypeInferenceV2.java | 19 +-- .../sonar/python/semantic/v2/TypeTable.java | 33 ++++ .../PythonTypeToDescriptorConverter.java | 148 ++++++++++++++++++ .../v2/types/FlowSensitiveTypeInference.java | 3 +- .../semantic/v2/types/IsInstanceVisitor.java | 3 +- .../v2/types/TrivialTypeInferenceVisitor.java | 18 +-- .../sonar/python/types/v2/UnknownType.java | 12 ++ .../semantic/v2/TypeInferenceV2Test.java | 5 +- .../python/types/v2/TypeCheckerTest.java | 2 + .../sonar/python/types/v2/TypesTestUtils.java | 2 +- .../plugins/python/indexer/PythonIndexer.java | 1 + ...erProjectLevelSymbolTableBuildingTest.java | 121 ++++++++++++++ .../sonar/plugins/python/indexer/v2/mod1.py | 2 + .../sonar/plugins/python/indexer/v2/mod2.py | 3 + .../sonar/plugins/python/indexer/v2/script.py | 15 ++ 22 files changed, 471 insertions(+), 37 deletions(-) create mode 100644 python-frontend/src/main/java/org/sonar/python/semantic/v2/BasicTypeTable.java create mode 100644 python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeTable.java create mode 100644 python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java create mode 100644 sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest.java create mode 100644 sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod1.py create mode 100644 sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod2.py create mode 100644 sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/script.py diff --git a/python-frontend/src/main/java/org/sonar/python/TestPythonVisitorRunner.java b/python-frontend/src/main/java/org/sonar/python/TestPythonVisitorRunner.java index 7a2bdb098f..20010856d0 100644 --- a/python-frontend/src/main/java/org/sonar/python/TestPythonVisitorRunner.java +++ b/python-frontend/src/main/java/org/sonar/python/TestPythonVisitorRunner.java @@ -93,6 +93,7 @@ public static ProjectLevelSymbolTable globalSymbols(List files, File baseD var astRoot = parseFile(pythonFile); String packageName = pythonPackageName(file, baseDir.getAbsolutePath()); projectLevelSymbolTable.addModule(astRoot, packageName, pythonFile); + projectLevelSymbolTable.addModuleV2(packageName, pythonFile, astRoot); } return projectLevelSymbolTable; } diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java b/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java index 96f933f951..9ee6db4bb2 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -41,15 +42,25 @@ import org.sonar.python.index.Descriptor; import org.sonar.python.index.DescriptorUtils; import org.sonar.python.index.VariableDescriptor; +import org.sonar.python.semantic.v2.BasicTypeTable; +import org.sonar.python.semantic.v2.SymbolTableBuilderV2; +import org.sonar.python.semantic.v2.SymbolV2; +import org.sonar.python.semantic.v2.TypeInferenceV2; +import org.sonar.python.semantic.v2.UsageV2; +import org.sonar.python.semantic.v2.converter.PythonTypeToDescriptorConverter; import org.sonar.python.semantic.v2.typeshed.TypeShedDescriptorsProvider; +import org.sonar.python.types.v2.PythonType; import static org.sonar.python.tree.TreeUtils.getSymbolFromTree; import static org.sonar.python.tree.TreeUtils.nthArgumentOrKeyword; public class ProjectLevelSymbolTable { + private final PythonTypeToDescriptorConverter pythonTypeToDescriptorConverter = new PythonTypeToDescriptorConverter(); private final Map> globalDescriptorsByModuleName; + private final Map> globalDescriptorsByModuleNameV2; private Map globalDescriptorsByFQN; + private Map globalDescriptorsByFQNV2; private final Set djangoViewsFQN = new HashSet<>(); private final Map> importsByModule = new HashMap<>(); private final Set projectBasePackages = new HashSet<>(); @@ -65,15 +76,19 @@ public static ProjectLevelSymbolTable from(Map> globalSymbol public ProjectLevelSymbolTable() { this.globalDescriptorsByModuleName = new HashMap<>(); + this.globalDescriptorsByModuleNameV2 = new HashMap<>(); } private ProjectLevelSymbolTable(Map> globalSymbolsByModuleName) { this.globalDescriptorsByModuleName = new HashMap<>(); + this.globalDescriptorsByModuleNameV2 = new HashMap<>(); globalSymbolsByModuleName.entrySet().forEach(entry -> { String moduleName = entry.getKey(); Set symbols = entry.getValue(); Set globalDescriptors = symbols.stream().map(DescriptorUtils::descriptor).collect(Collectors.toSet()); globalDescriptorsByModuleName.put(moduleName, globalDescriptors); + globalDescriptors = symbols.stream().map(DescriptorUtils::descriptor).collect(Collectors.toSet()); + globalDescriptorsByModuleNameV2.put(moduleName, globalDescriptors); }); } @@ -82,6 +97,7 @@ public void removeModule(String packageName, String fileName) { globalDescriptorsByModuleName.remove(fullyQualifiedModuleName); // ensure globalDescriptorsByFQN is re-computed this.globalDescriptorsByFQN = null; + this.globalDescriptorsByFQNV2 = null; } public void addModule(FileInput fileInput, String packageName, PythonFile pythonFile) { @@ -169,6 +185,11 @@ public Set getDescriptorsFromModule(@Nullable String moduleName) { return globalDescriptorsByModuleName.get(moduleName); } + @CheckForNull + public Set getDescriptorsFromModuleV2(@Nullable String moduleName) { + return globalDescriptorsByModuleNameV2.get(moduleName); + } + public Map> importsByModule() { return Collections.unmodifiableMap(importsByModule); } @@ -201,6 +222,33 @@ public TypeShedDescriptorsProvider typeShedDescriptorsProvider() { return typeShedDescriptorsProvider; } + public void addModuleV2(String fullyQualifiedModuleName, Set moduleDescriptors) { + globalDescriptorsByModuleNameV2.put(fullyQualifiedModuleName, moduleDescriptors); + } + + public void addModuleV2(String fullyQualifiedModuleName, Map> typesBySymbol) { + var moduleDescriptors = typesBySymbol.entrySet() + .stream() + .map(entry -> { + var descriptor = pythonTypeToDescriptorConverter.convert(fullyQualifiedModuleName, entry.getKey(), entry.getValue()); + return Map.entry(entry.getKey(), descriptor); + } + ) + .filter(entry -> !(!Objects.requireNonNull(entry.getValue().fullyQualifiedName()).startsWith(fullyQualifiedModuleName) + || entry.getKey().usages().stream().anyMatch(u -> u.kind().equals(UsageV2.Kind.IMPORT)))) + .map(Map.Entry::getValue) + .collect(Collectors.toSet()); + addModuleV2(fullyQualifiedModuleName, moduleDescriptors); + } + + public void addModuleV2(String packageName, PythonFile pythonFile, FileInput astRoot) { + var fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName()); + var symbolTable = new SymbolTableBuilderV2(astRoot).build(); + var typeInferenceV2 = new TypeInferenceV2(new BasicTypeTable(), pythonFile, symbolTable); + var typesBySymbol = typeInferenceV2.inferTypes(astRoot); + addModuleV2(fullyQualifiedModuleName, typesBySymbol); + } + private class DjangoViewsVisitor extends BaseTreeVisitor { @Override public void visitCallExpression(CallExpression callExpression) { diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/BasicTypeTable.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/BasicTypeTable.java new file mode 100644 index 0000000000..291dc927b5 --- /dev/null +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/BasicTypeTable.java @@ -0,0 +1,46 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.python.semantic.v2; + +import java.util.List; +import org.sonar.python.types.v2.PythonType; +import org.sonar.python.types.v2.UnknownType; + +public class BasicTypeTable implements TypeTable { + @Override + public PythonType getBuiltinsModule() { + return new UnknownType.UnresolvedImportType(""); + } + + @Override + public PythonType getType(String typeFqn) { + return new UnknownType.UnresolvedImportType(typeFqn); + } + + @Override + public PythonType getType(String... typeFqnParts) { + return new UnknownType.UnresolvedImportType(String.join(".", typeFqnParts)); + } + + @Override + public PythonType getType(List typeFqnParts) { + return new UnknownType.UnresolvedImportType(String.join(".", typeFqnParts)); + } +} diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/FunctionTypeBuilder.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/FunctionTypeBuilder.java index 99e0d0f2f4..7eeaee6604 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/FunctionTypeBuilder.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/FunctionTypeBuilder.java @@ -60,7 +60,7 @@ public class FunctionTypeBuilder implements TypeBuilder { private static final String CLASS_METHOD_DECORATOR = "classmethod"; private static final String STATIC_METHOD_DECORATOR = "staticmethod"; - public FunctionTypeBuilder fromFunctionDef(FunctionDef functionDef, @Nullable String fileId, ProjectLevelTypeTable projectLevelTypeTable) { + public FunctionTypeBuilder fromFunctionDef(FunctionDef functionDef, @Nullable String fileId, TypeTable projectLevelTypeTable) { this.name = functionDef.name().name(); this.attributes = new ArrayList<>(); this.parameters = new ArrayList<>(); @@ -155,7 +155,7 @@ public FunctionTypeBuilder withOwner(PythonType owner) { return this; } - private void createParameterNames(List parameterTrees, @Nullable String fileId, ProjectLevelTypeTable projectLevelTypeTable) { + private void createParameterNames(List parameterTrees, @Nullable String fileId, TypeTable projectLevelTypeTable) { ParameterState parameterState = new ParameterState(); parameterState.positionalOnly = parameterTrees.stream().anyMatch(param -> Optional.of(param) .filter(p -> p.is(Tree.Kind.PARAMETER)) @@ -174,7 +174,7 @@ private void createParameterNames(List parameterTrees, @Nullable S } } - private void addParameter(Parameter parameter, @Nullable String fileId, ParameterState parameterState, ProjectLevelTypeTable projectLevelTypeTable) { + private void addParameter(Parameter parameter, @Nullable String fileId, ParameterState parameterState, TypeTable projectLevelTypeTable) { Name parameterName = parameter.name(); Token starToken = parameter.starToken(); if (parameterName != null) { @@ -197,7 +197,7 @@ private void addParameter(Parameter parameter, @Nullable String fileId, Paramete } } - private ParameterType getParameterType(Parameter parameter, ProjectLevelTypeTable projectLevelTypeTable) { + private ParameterType getParameterType(Parameter parameter, TypeTable projectLevelTypeTable) { boolean isPositionalVariadic = false; boolean isKeywordVariadic = false; Token starToken = parameter.starToken(); diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/LazyTypesContext.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/LazyTypesContext.java index 63d9c46751..352da5253c 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/LazyTypesContext.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/LazyTypesContext.java @@ -28,11 +28,11 @@ public class LazyTypesContext { private final Map lazyTypes; - private final ProjectLevelTypeTable projectLevelTypeTable; + private final TypeTable typeTable; - public LazyTypesContext(ProjectLevelTypeTable projectLevelTypeTable) { + public LazyTypesContext(ProjectLevelTypeTable typeTable) { this.lazyTypes = new HashMap<>(); - this.projectLevelTypeTable = projectLevelTypeTable; + this.typeTable = typeTable; } public TypeWrapper getOrCreateLazyTypeWrapper(String importPath) { @@ -49,7 +49,7 @@ public LazyType getOrCreateLazyType(String importPath) { } public PythonType resolveLazyType(LazyType lazyType) { - PythonType resolved = projectLevelTypeTable.getType(lazyType.importPath()); + PythonType resolved = typeTable.getType(lazyType.importPath()); lazyType.resolve(resolved); lazyTypes.remove(lazyType.importPath()); return resolved; diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java index 23afe2dcc4..9586245830 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java @@ -29,7 +29,7 @@ import org.sonar.python.types.v2.PythonType; import org.sonar.python.types.v2.TypeWrapper; -public class ProjectLevelTypeTable { +public class ProjectLevelTypeTable implements TypeTable { private final SymbolsModuleTypeProvider symbolsModuleTypeProvider; private final ModuleType rootModule; @@ -41,18 +41,22 @@ public ProjectLevelTypeTable(ProjectLevelSymbolTable projectLevelSymbolTable) { this.rootModule = this.symbolsModuleTypeProvider.createBuiltinModule(); } - public ModuleType getBuiltinsModule() { + @Override + public PythonType getBuiltinsModule() { return rootModule; } + @Override public PythonType getType(String typeFqn) { return getType(typeFqn.split("\\.")); } + @Override public PythonType getType(String... typeFqnParts) { return getType(List.of(typeFqnParts)); } + @Override public PythonType getType(List typeFqnParts) { var parent = (PythonType) rootModule; for (int i = 0; i < typeFqnParts.size(); i++) { diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java index 95600dcc91..c23604c24c 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java @@ -79,7 +79,7 @@ private static String getModuleFqnString(List moduleFqn) { } private Optional createModuleTypeFromProjectLevelSymbolTable(String moduleName, String moduleFqn, ModuleType parent) { - var retrieved = projectLevelSymbolTable.getDescriptorsFromModule(moduleFqn); + var retrieved = projectLevelSymbolTable.getDescriptorsFromModuleV2(moduleFqn); if (retrieved == null) { return Optional.empty(); } diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeInferenceV2.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeInferenceV2.java index 3a08945b40..5fca6862dc 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeInferenceV2.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeInferenceV2.java @@ -21,7 +21,6 @@ import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -47,22 +46,21 @@ public class TypeInferenceV2 { - private final ProjectLevelTypeTable projectLevelTypeTable; + private final TypeTable projectLevelTypeTable; private final SymbolTable symbolTable; private final PythonFile pythonFile; - private Map> typesBySymbol = new HashMap<>(); - public TypeInferenceV2(ProjectLevelTypeTable projectLevelTypeTable, PythonFile pythonFile, SymbolTable symbolTable) { + public TypeInferenceV2(TypeTable projectLevelTypeTable, PythonFile pythonFile, SymbolTable symbolTable) { this.projectLevelTypeTable = projectLevelTypeTable; this.symbolTable = symbolTable; this.pythonFile = pythonFile; } - public void inferTypes(FileInput fileInput) { + public Map> inferTypes(FileInput fileInput) { TrivialTypeInferenceVisitor trivialTypeInferenceVisitor = new TrivialTypeInferenceVisitor(projectLevelTypeTable, pythonFile); fileInput.accept(trivialTypeInferenceVisitor); - inferTypesAndMemberAccessSymbols(fileInput); + var typesBySymbol = inferTypesAndMemberAccessSymbols(fileInput); fileInput.accept(new BaseTreeVisitor() { @Override @@ -71,20 +69,17 @@ public void visitFunctionDef(FunctionDef funcDef) { inferTypesAndMemberAccessSymbols(funcDef); } }); - } - - public Map> getTypesBySymbol() { return typesBySymbol; } - private void inferTypesAndMemberAccessSymbols(FileInput fileInput) { + private Map> inferTypesAndMemberAccessSymbols(FileInput fileInput) { StatementList statements = fileInput.statements(); if (statements == null) { - return; + return Map.of(); } var moduleSymbols = symbolTable.getSymbolsByRootTree(fileInput); - typesBySymbol = inferTypesAndMemberAccessSymbols( + return inferTypesAndMemberAccessSymbols( fileInput, statements, moduleSymbols, diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeTable.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeTable.java new file mode 100644 index 0000000000..b52c092789 --- /dev/null +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeTable.java @@ -0,0 +1,33 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.python.semantic.v2; + +import java.util.List; +import org.sonar.python.types.v2.PythonType; + +public interface TypeTable { + PythonType getBuiltinsModule(); + + PythonType getType(String typeFqn); + + PythonType getType(String... typeFqnParts); + + PythonType getType(List typeFqnParts); +} diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java new file mode 100644 index 0000000000..2357583ce5 --- /dev/null +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java @@ -0,0 +1,148 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.python.semantic.v2.converter; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.CheckForNull; +import org.sonar.python.index.AmbiguousDescriptor; +import org.sonar.python.index.ClassDescriptor; +import org.sonar.python.index.Descriptor; +import org.sonar.python.index.FunctionDescriptor; +import org.sonar.python.index.VariableDescriptor; +import org.sonar.python.semantic.v2.SymbolV2; +import org.sonar.python.types.v2.ClassType; +import org.sonar.python.types.v2.FunctionType; +import org.sonar.python.types.v2.ParameterV2; +import org.sonar.python.types.v2.PythonType; +import org.sonar.python.types.v2.TypeWrapper; +import org.sonar.python.types.v2.UnionType; +import org.sonar.python.types.v2.UnknownType; + +public class PythonTypeToDescriptorConverter { + + public Descriptor convert(String moduleFqn, SymbolV2 symbol, Set types) { + var candidates = types.stream() + .map(type -> convert(moduleFqn, symbol.name(), type)) + .collect(Collectors.toSet()); + + if (candidates.size() == 1) { + return candidates.iterator().next(); + } + return new AmbiguousDescriptor(symbol.name(), symbolFqn(moduleFqn, symbol.name()), candidates); + } + + private Descriptor convert(String moduleFqn, String symbolName, PythonType type) { + if (type instanceof FunctionType functionType) { + return convert(moduleFqn, symbolName, functionType); + } + if (type instanceof ClassType classType) { + return convert(moduleFqn, symbolName, classType); + } + if (type instanceof UnionType unionType) { + return convert(moduleFqn, symbolName, unionType); + } + if (type instanceof UnknownType.UnresolvedImportType unresolvedImportType) { + return convert(moduleFqn, symbolName, unresolvedImportType); + } + return new VariableDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), null); + } + + private Descriptor convert(String moduleFqn, String symbolName, FunctionType type) { + + var parameters = type.parameters() + .stream() + .map(parameter -> convert(moduleFqn, parameter)) + .toList(); + + return new FunctionDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), + parameters, + type.isAsynchronous(), + type.isInstanceMethod(), + List.of(), + type.hasDecorators(), + type.definitionLocation().orElse(null), + null, + null + ); + } + + private Descriptor convert(String moduleFqn, String symbolName, ClassType type) { + Set memberDescriptors = type.members().stream().map(m -> convert(moduleFqn, m.name(), m.type())).collect(Collectors.toSet()); + List superClasses = type.superClasses().stream().map(TypeWrapper::type).map(t -> typeFqn(moduleFqn, t)).toList(); + return new ClassDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), + superClasses, + memberDescriptors, + type.hasDecorators(), + type.definitionLocation().orElse(null), + false, + type.hasMetaClass(), + null, + false + ); + } + + private Descriptor convert(String moduleFqn, String symbolName, UnionType type) { + var candidates = type.candidates().stream() + .map(candidateType -> convert(moduleFqn, symbolName, candidateType)) + .collect(Collectors.toSet()); + return new AmbiguousDescriptor(symbolName, + symbolFqn(moduleFqn, symbolName), + candidates + ); + } + + private static Descriptor convert(String moduleFqn, String symbolName, UnknownType.UnresolvedImportType type) { + return new VariableDescriptor(symbolName, + symbolFqn(moduleFqn, symbolName), + type.importPath() + ); + } + + public FunctionDescriptor.Parameter convert(String moduleFqn, ParameterV2 parameter) { + var type = parameter.declaredType().type().unwrappedType(); + var annotatedType = typeFqn(moduleFqn, type); + + return new FunctionDescriptor.Parameter(parameter.name(), + annotatedType, + parameter.hasDefaultValue(), + parameter.isKeywordOnly(), + parameter.isPositionalOnly(), + parameter.isKeywordVariadic(), + parameter.isPositionalVariadic(), + parameter.location()); + } + + @CheckForNull + private static String typeFqn(String moduleFqn, PythonType type) { + if (type instanceof UnknownType.UnresolvedImportType importType) { + return importType.importPath(); + } else if (type instanceof ClassType classType) { + return moduleFqn + "." + classType.name(); + } + return null; + } + + private static String symbolFqn(String moduleFqn, String symbolName) { + return moduleFqn + "." + symbolName; + } + +} diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/FlowSensitiveTypeInference.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/FlowSensitiveTypeInference.java index d528e66942..4dfee21e7b 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/FlowSensitiveTypeInference.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/FlowSensitiveTypeInference.java @@ -41,6 +41,7 @@ import org.sonar.python.cfg.fixpoint.ProgramState; import org.sonar.python.semantic.v2.ProjectLevelTypeTable; import org.sonar.python.semantic.v2.SymbolV2; +import org.sonar.python.semantic.v2.TypeTable; import org.sonar.python.types.v2.PythonType; public class FlowSensitiveTypeInference extends ForwardAnalysis { @@ -51,7 +52,7 @@ public class FlowSensitiveTypeInference extends ForwardAnalysis { private final IsInstanceVisitor isInstanceVisitor; public FlowSensitiveTypeInference( - ProjectLevelTypeTable projectLevelTypeTable, Set trackedVars, + TypeTable projectLevelTypeTable, Set trackedVars, Map assignmentsByAssignmentStatement, Map> definitionsByDefinitionStatement, Map parameterTypesByName diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/IsInstanceVisitor.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/IsInstanceVisitor.java index 803d76b384..e9016dafb1 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/IsInstanceVisitor.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/IsInstanceVisitor.java @@ -28,6 +28,7 @@ import org.sonar.plugins.python.api.tree.RegularArgument; import org.sonar.python.semantic.v2.ProjectLevelTypeTable; import org.sonar.python.semantic.v2.SymbolV2; +import org.sonar.python.semantic.v2.TypeTable; import org.sonar.python.types.v2.PythonType; import org.sonar.python.types.v2.TypeSource; import org.sonar.python.types.v2.UnknownType; @@ -36,7 +37,7 @@ public class IsInstanceVisitor extends BaseTreeVisitor { private final PythonType isInstanceFunctionType; private TypeInferenceProgramState state; - public IsInstanceVisitor(ProjectLevelTypeTable projectLevelTypeTable) { + public IsInstanceVisitor(TypeTable projectLevelTypeTable) { isInstanceFunctionType = projectLevelTypeTable.getType("isinstance"); } diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java index 8a4027928f..e8a60bf2cb 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java @@ -59,8 +59,8 @@ import org.sonar.plugins.python.api.tree.TypeAnnotation; import org.sonar.python.semantic.v2.ClassTypeBuilder; import org.sonar.python.semantic.v2.FunctionTypeBuilder; -import org.sonar.python.semantic.v2.ProjectLevelTypeTable; import org.sonar.python.semantic.v2.SymbolV2; +import org.sonar.python.semantic.v2.TypeTable; import org.sonar.python.semantic.v2.UsageV2; import org.sonar.python.tree.ComprehensionExpressionImpl; import org.sonar.python.tree.DictCompExpressionImpl; @@ -88,12 +88,12 @@ public class TrivialTypeInferenceVisitor extends BaseTreeVisitor { - private final ProjectLevelTypeTable projectLevelTypeTable; + private final TypeTable projectLevelTypeTable; private final String fileId; private final Deque typeStack = new ArrayDeque<>(); - public TrivialTypeInferenceVisitor(ProjectLevelTypeTable projectLevelTypeTable, PythonFile pythonFile) { + public TrivialTypeInferenceVisitor(TypeTable projectLevelTypeTable, PythonFile pythonFile) { this.projectLevelTypeTable = projectLevelTypeTable; Path path = pathOf(pythonFile); this.fileId = path != null ? path.toString() : pythonFile.toString(); @@ -108,7 +108,7 @@ public void visitFileInput(FileInput fileInput) { @Override public void visitStringLiteral(StringLiteral stringLiteral) { - ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule(); + var builtins = this.projectLevelTypeTable.getBuiltinsModule(); // TODO: SONARPY-1867 multiple object types to represent str instance? PythonType strType = builtins.resolveMember("str").orElse(PythonType.UNKNOWN); ((StringLiteralImpl) stringLiteral).typeV2(new ObjectType(strType, new ArrayList<>(), new ArrayList<>())); @@ -122,7 +122,7 @@ public void visitTuple(Tuple tuple) { if (contentTypes.size() == 1 && !contentTypes.get(0).equals(PythonType.UNKNOWN)) { attributes = contentTypes; } - ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule(); + var builtins = this.projectLevelTypeTable.getBuiltinsModule(); PythonType tupleType = builtins.resolveMember("tuple").orElse(PythonType.UNKNOWN); ((TupleImpl) tuple).typeV2(new ObjectType(tupleType, attributes, new ArrayList<>())); } @@ -130,7 +130,7 @@ public void visitTuple(Tuple tuple) { @Override public void visitDictionaryLiteral(DictionaryLiteral dictionaryLiteral) { super.visitDictionaryLiteral(dictionaryLiteral); - ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule(); + var builtins = this.projectLevelTypeTable.getBuiltinsModule(); PythonType dictType = builtins.resolveMember("dict").orElse(PythonType.UNKNOWN); ((DictionaryLiteralImpl) dictionaryLiteral).typeV2(new ObjectType(dictType, new ArrayList<>(), new ArrayList<>())); } @@ -138,14 +138,14 @@ public void visitDictionaryLiteral(DictionaryLiteral dictionaryLiteral) { @Override public void visitSetLiteral(SetLiteral setLiteral) { super.visitSetLiteral(setLiteral); - ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule(); + var builtins = this.projectLevelTypeTable.getBuiltinsModule(); PythonType setType = builtins.resolveMember("set").orElse(PythonType.UNKNOWN); ((SetLiteralImpl) setLiteral).typeV2(new ObjectType(setType, new ArrayList<>(), new ArrayList<>())); } @Override public void visitNumericLiteral(NumericLiteral numericLiteral) { - ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule(); + var builtins = this.projectLevelTypeTable.getBuiltinsModule(); NumericLiteralImpl numericLiteralImpl = (NumericLiteralImpl) numericLiteral; NumericLiteralImpl.NumericKind numericKind = numericLiteralImpl.numericKind(); PythonType pythonType = builtins.resolveMember(numericKind.value()).orElse(PythonType.UNKNOWN); @@ -154,7 +154,7 @@ public void visitNumericLiteral(NumericLiteral numericLiteral) { @Override public void visitNone(NoneExpression noneExpression) { - ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule(); + var builtins = this.projectLevelTypeTable.getBuiltinsModule(); // TODO: SONARPY-1867 multiple object types to represent str instance? PythonType noneType = builtins.resolveMember("NoneType").orElse(PythonType.UNKNOWN); ((NoneExpressionImpl) noneExpression).typeV2(new ObjectType(noneType, new ArrayList<>(), new ArrayList<>())); diff --git a/python-frontend/src/main/java/org/sonar/python/types/v2/UnknownType.java b/python-frontend/src/main/java/org/sonar/python/types/v2/UnknownType.java index 1a35ed2369..a068043a42 100644 --- a/python-frontend/src/main/java/org/sonar/python/types/v2/UnknownType.java +++ b/python-frontend/src/main/java/org/sonar/python/types/v2/UnknownType.java @@ -20,6 +20,10 @@ */ package org.sonar.python.types.v2; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.sonar.api.Beta; @Beta @@ -36,5 +40,13 @@ final class UnknownTypeImpl implements UnknownType { } record UnresolvedImportType(String importPath) implements UnknownType { + + @Override + public Optional resolveMember(String memberName) { + var memberFqn = Stream.of(importPath, memberName) + .filter(Predicate.not(String::isEmpty)) + .collect(Collectors.joining(".")); + return Optional.of(new UnresolvedImportType(memberFqn)); + } } } diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java index 6b1222d38e..f3d705edba 100644 --- a/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java +++ b/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java @@ -711,6 +711,7 @@ void inferFunctionParameterTypesMultiFile() { ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable(); var modFile = pythonFile("mod.py"); projectLevelSymbolTable.addModule(tree, "", modFile); + projectLevelSymbolTable.addModuleV2("", modFile, tree); ProjectLevelTypeTable projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable); var modFileId = SymbolUtils.pathOf(modFile).toString(); @@ -2100,6 +2101,7 @@ void type_origin_of_project_function() { ); ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable(); projectLevelSymbolTable.addModule(tree, "", pythonFile("mod.py")); + projectLevelSymbolTable.addModuleV2("", pythonFile("mod.py"), tree); ProjectLevelTypeTable projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable); var lines = """ @@ -2660,8 +2662,7 @@ private static Map> inferTypesBySymbol(String lines) { FileInput root = parse(lines); var symbolTable = new SymbolTableBuilderV2(root).build(); var typeInferenceV2 = new TypeInferenceV2(PROJECT_LEVEL_TYPE_TABLE, pythonFile, symbolTable); - typeInferenceV2.inferTypes(root); - return typeInferenceV2.getTypesBySymbol(); + return typeInferenceV2.inferTypes(root); } private static FileInput inferTypes(String lines) { diff --git a/python-frontend/src/test/java/org/sonar/python/types/v2/TypeCheckerTest.java b/python-frontend/src/test/java/org/sonar/python/types/v2/TypeCheckerTest.java index 9cd58ac07f..ac333757e9 100644 --- a/python-frontend/src/test/java/org/sonar/python/types/v2/TypeCheckerTest.java +++ b/python-frontend/src/test/java/org/sonar/python/types/v2/TypeCheckerTest.java @@ -275,12 +275,14 @@ class B: pass ); PythonFile pythonFile = PythonTestUtils.pythonFile("mod.py"); projectLevelSymbolTable.addModule(tree, "my_package", pythonFile); + projectLevelSymbolTable.addModuleV2("my_package", pythonFile, tree); ProjectLevelTypeTable projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable); TypeChecker localTypeChecker = new TypeChecker(projectLevelTypeTable); FileInput initTree = parseWithoutSymbols(""); PythonFile initFile = PythonTestUtils.pythonFile("__init__.py"); projectLevelSymbolTable.addModule(initTree, "my_package", initFile); + projectLevelSymbolTable.addModuleV2("my_package", initFile, initTree); var fileInput = parseAndInferTypes(projectLevelTypeTable, pythonFile, """ from my_package.mod import A diff --git a/python-frontend/src/test/java/org/sonar/python/types/v2/TypesTestUtils.java b/python-frontend/src/test/java/org/sonar/python/types/v2/TypesTestUtils.java index 2ff5edcdea..d839b08774 100644 --- a/python-frontend/src/test/java/org/sonar/python/types/v2/TypesTestUtils.java +++ b/python-frontend/src/test/java/org/sonar/python/types/v2/TypesTestUtils.java @@ -30,7 +30,7 @@ public class TypesTestUtils { public static final ProjectLevelTypeTable PROJECT_LEVEL_TYPE_TABLE = new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty()); - public static final ModuleType BUILTINS = PROJECT_LEVEL_TYPE_TABLE.getBuiltinsModule(); + public static final PythonType BUILTINS = PROJECT_LEVEL_TYPE_TABLE.getBuiltinsModule(); public static final PythonType INT_TYPE = BUILTINS.resolveMember("int").get(); public static final PythonType FLOAT_TYPE = BUILTINS.resolveMember("float").get(); diff --git a/sonar-python-plugin/src/main/java/org/sonar/plugins/python/indexer/PythonIndexer.java b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/indexer/PythonIndexer.java index 71331f8f74..ea21cf1675 100644 --- a/sonar-python-plugin/src/main/java/org/sonar/plugins/python/indexer/PythonIndexer.java +++ b/sonar-python-plugin/src/main/java/org/sonar/plugins/python/indexer/PythonIndexer.java @@ -94,6 +94,7 @@ void addFile(PythonInputFile inputFile) throws IOException { projectLevelSymbolTable.addProjectPackage(packageName); PythonFile pythonFile = SonarQubePythonFile.create(inputFile.wrappedFile()); projectLevelSymbolTable.addModule(astRoot, packageName, pythonFile); + projectLevelSymbolTable.addModuleV2(packageName, pythonFile, astRoot); } public abstract void buildOnce(SensorContext context); diff --git a/sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest.java b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest.java new file mode 100644 index 0000000000..5aeb799734 --- /dev/null +++ b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest.java @@ -0,0 +1,121 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.plugins.python.indexer; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.event.Level; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.TestInputFileBuilder; +import org.sonar.api.batch.sensor.internal.SensorContextTester; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; +import org.sonar.plugins.python.Python; +import org.sonar.plugins.python.PythonInputFile; +import org.sonar.plugins.python.PythonInputFileImpl; +import org.sonar.plugins.python.TestUtils; +import org.sonar.python.index.ClassDescriptor; +import org.sonar.python.index.Descriptor; +import org.sonar.python.index.FunctionDescriptor; +import org.sonar.python.semantic.ProjectLevelSymbolTable; + +import static org.assertj.core.api.Assertions.assertThat; + +class SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest { + + private final File baseDir = new File("src/test/resources/org/sonar/plugins/python/indexer/v2").getAbsoluteFile(); + + @RegisterExtension + public LogTesterJUnit5 logTester = new LogTesterJUnit5().setLevel(Level.DEBUG); + + @Test + void single_file_simple_test() throws IOException { + var projectLevelSymbolTable = buildProjectLevelSymbolTable("script.py"); + assertThat(projectLevelSymbolTable.getDescriptorsFromModule("script")).hasSize(4); + Set moduleDescriptors = projectLevelSymbolTable.getDescriptorsFromModuleV2("script"); + assertThat(moduleDescriptors).hasSize(4); + + var aClassDescriptor = moduleDescriptors + .stream() + .filter(d -> d.name().equals("A")) + .findFirst() + .filter(ClassDescriptor.class::isInstance) + .map(ClassDescriptor.class::cast) + .orElse(null); + assertThat(aClassDescriptor).isNotNull(); + assertThat(aClassDescriptor.members()).hasSize(1); + assertThat(aClassDescriptor.superClasses()).containsOnly("script.Parent", "int"); + + var doSomethingDescriptor = aClassDescriptor.members() + .stream() + .filter(d -> d.name().equals("do_something")) + .findFirst() + .filter(FunctionDescriptor.class::isInstance) + .map(FunctionDescriptor.class::cast) + .orElse(null); + assertThat(doSomethingDescriptor).isNotNull(); + assertThat(doSomethingDescriptor.parameters()).hasSize(2); + } + + @Test + void multiple_files_simple_test() throws IOException { + var projectLevelSymbolTable = buildProjectLevelSymbolTable("mod1.py", "mod2.py"); + assertThat(projectLevelSymbolTable.getDescriptorsFromModule("mod1")).hasSize(1); + assertThat(projectLevelSymbolTable.getDescriptorsFromModuleV2("mod2")).hasSize(1); + } + + private ProjectLevelSymbolTable buildProjectLevelSymbolTable(String... files) throws IOException { + var context = SensorContextTester.create(baseDir); + var workDir = Files.createTempDirectory("workDir"); + context.fileSystem().setWorkDir(workDir); + var inputFiles = Stream.of(files) + .map(fileName -> inputFile(context, fileName)) + .toList(); + var moduleFileSystem = new TestModuleFileSystem(inputFiles); + var pythonIndexer = new SonarLintPythonIndexer(moduleFileSystem); + pythonIndexer.buildOnce(context); + return pythonIndexer.projectLevelSymbolTable(); + } + + private PythonInputFile inputFile(SensorContextTester context, String name) { + var inputFile = createInputFile(name); + context.fileSystem().add(inputFile.wrappedFile()); + return inputFile; + } + + private PythonInputFile createInputFile(String name) { + return createInputFile(name, Python.KEY); + } + + private PythonInputFile createInputFile(String name, String languageKey) { + return new PythonInputFileImpl(TestInputFileBuilder.create("moduleKey", name) + .setModuleBaseDir(baseDir.toPath()) + .setCharset(StandardCharsets.UTF_8) + .setType(InputFile.Type.MAIN) + .setLanguage(languageKey) + .initMetadata(TestUtils.fileContent(new File(baseDir, name), StandardCharsets.UTF_8)) + .build()); + } +} diff --git a/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod1.py b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod1.py new file mode 100644 index 0000000000..52a001a63d --- /dev/null +++ b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod1.py @@ -0,0 +1,2 @@ + +def func1(): ... diff --git a/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod2.py b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod2.py new file mode 100644 index 0000000000..6cdf70b676 --- /dev/null +++ b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod2.py @@ -0,0 +1,3 @@ +from typing import List + +def func2(a : List): ... diff --git a/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/script.py b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/script.py new file mode 100644 index 0000000000..092f751147 --- /dev/null +++ b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/script.py @@ -0,0 +1,15 @@ +def foo(): ... + +class Parent: + ... + +class A(Parent, int): + def do_something(self, a: int): + ... + +if something: + class B: + def method_one(self): ... +else: + class B: + def method_two(self): ...