Skip to content

Commit

Permalink
SONARPY-2110 Make the ProjectLevelSymbolTable return the Descriptor m…
Browse files Browse the repository at this point in the history 
…odel instead of the v1 Symbol model (#2070)
  • Loading branch information
@ghislainpiot
ghislainpiot authored Oct 14, 2024
1 parent d1869c7 commit 2970882
Show file tree
Hide file tree
Showing 8 changed files with 38 additions and 170 deletions.
5 changes: 5 additions & 0 deletions python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,11 @@ public Set<Symbol> getSymbolsFromModule(@Nullable String moduleName) {
.map(desc -> DescriptorUtils.symbolFromDescriptor(desc, this, null, createdSymbolsByDescriptor, createdSymbolsByFqn)).collect(Collectors.toSet());
}

@CheckForNull
public Set<Descriptor> getDescriptorsFromModule(@Nullable String moduleName) {
return globalDescriptorsByModuleName.get(moduleName);
}

public Map<String, Set<String>> importsByModule() {
return Collections.unmodifiableMap(importsByModule);
}
Expand Down
165 changes: 9 additions & 156 deletions python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,38 +19,18 @@
*/
package org.sonar.python.semantic.v2;

import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.sonar.plugins.python.api.symbols.AmbiguousSymbol;
import org.sonar.plugins.python.api.symbols.ClassSymbol;
import org.sonar.plugins.python.api.symbols.FunctionSymbol;
import org.sonar.plugins.python.api.symbols.Symbol;
import org.sonar.python.semantic.ClassSymbolImpl;
import org.sonar.python.semantic.FunctionSymbolImpl;
import org.sonar.python.index.Descriptor;
import org.sonar.python.semantic.ProjectLevelSymbolTable;
import org.sonar.python.semantic.v2.converter.AnyDescriptorToPythonTypeConverter;
import org.sonar.python.types.v2.ClassType;
import org.sonar.python.types.v2.FunctionType;
import org.sonar.python.types.v2.LazyTypeWrapper;
import org.sonar.python.types.v2.Member;
import org.sonar.python.types.v2.ModuleType;
import org.sonar.python.types.v2.ParameterV2;
import org.sonar.python.types.v2.PythonType;
import org.sonar.python.types.v2.TypeOrigin;
import org.sonar.python.types.v2.TypeWrapper;
import org.sonar.python.types.v2.UnionType;

public class SymbolsModuleTypeProvider {
public static final String OBJECT_TYPE_FQN = "object";
private final ProjectLevelSymbolTable projectLevelSymbolTable;
private final ModuleType rootModule;
private final LazyTypesContext lazyTypesContext;
Expand All @@ -64,7 +44,7 @@ public SymbolsModuleTypeProvider(ProjectLevelSymbolTable projectLevelSymbolTable
var rootModuleMembers = projectLevelSymbolTable.typeShedDescriptorsProvider().builtinDescriptors()
.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> anyDescriptorToPythonTypeConverter.convert(e.getValue())));
.collect(Collectors.toMap(Map.Entry::getKey, e -> anyDescriptorToPythonTypeConverter.convert(e.getValue(), TypeOrigin.STUB)));
this.rootModule = new ModuleType(null, null, rootModuleMembers);
}

Expand All @@ -88,147 +68,20 @@ private static String getModuleFqnString(List<String> moduleFqn) {
}

private Optional<ModuleType> createModuleTypeFromProjectLevelSymbolTable(String moduleName, String moduleFqn, ModuleType parent) {
return Optional.ofNullable(projectLevelSymbolTable.getSymbolsFromModule(moduleFqn))
.map(projectModuleSymbols -> createModuleFromSymbols(moduleName, parent, projectModuleSymbols));
var retrieved = projectLevelSymbolTable.getDescriptorsFromModule(moduleFqn);
if (retrieved == null) {
return Optional.empty();
}
var members = retrieved.stream().collect(Collectors.toMap(Descriptor::name, d -> anyDescriptorToPythonTypeConverter.convert(d, TypeOrigin.LOCAL)));
return Optional.of(new ModuleType(moduleName, parent, members));
}

private Optional<ModuleType> createModuleTypeFromTypeShed(String moduleName, String moduleFqn, ModuleType parent) {
var moduleMembers = projectLevelSymbolTable.typeShedDescriptorsProvider().descriptorsForModule(moduleFqn)
.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> anyDescriptorToPythonTypeConverter.convert(e.getValue())));
.collect(Collectors.toMap(Map.Entry::getKey, e -> anyDescriptorToPythonTypeConverter.convert(e.getValue(), TypeOrigin.STUB)));
return Optional.of(moduleMembers).filter(m -> !m.isEmpty())
.map(m -> new ModuleType(moduleName, parent, m));
}

private ModuleType createModuleFromSymbols(@Nullable String name, @Nullable ModuleType parent, Collection<Symbol> symbols) {
var members = new HashMap<String, PythonType>();
Map<Symbol, PythonType> createdTypesBySymbol = new HashMap<>();
symbols.forEach(symbol -> {
var type = convertToType(symbol, createdTypesBySymbol);
members.put(symbol.name(), type);
});
var module = new ModuleType(name, parent);
module.members().putAll(members);

Optional.ofNullable(parent)
.map(ModuleType::members)
.ifPresent(m -> m.put(name, module));

return module;
}

private PythonType convertToFunctionType(FunctionSymbol symbol, Map<Symbol, PythonType> createdTypesBySymbol) {
if (createdTypesBySymbol.containsKey(symbol)) {
return createdTypesBySymbol.get(symbol);
}

var parameters = symbol.parameters()
.stream()
.map(this::convertParameter)
.toList();

var returnType = PythonType.UNKNOWN;

TypeOrigin typeOrigin = symbol.isStub() ? TypeOrigin.STUB : TypeOrigin.LOCAL;

FunctionTypeBuilder functionTypeBuilder =
new FunctionTypeBuilder(symbol.name())
.withAttributes(List.of())
.withParameters(parameters)
.withReturnType(returnType)
.withTypeOrigin(typeOrigin)
.withAsynchronous(symbol.isAsynchronous())
.withHasDecorators(symbol.hasDecorators())
.withInstanceMethod(symbol.isInstanceMethod())
.withHasVariadicParameter(symbol.hasVariadicParameter())
.withDefinitionLocation(symbol.definitionLocation());
FunctionType functionType = functionTypeBuilder.build();
createdTypesBySymbol.put(symbol, functionType);
return functionType;
}

PythonType resolvePossibleLazyType(String fullyQualifiedName) {
if (rootModule == null) {
// If root module has not been created yet, return lazy type
return lazyTypesContext.getOrCreateLazyType(fullyQualifiedName);
}
PythonType currentType = rootModule;
String[] fqnParts = fullyQualifiedName.split("\\.");
var fqnPartsQueue = new ArrayDeque<>(Arrays.asList(fqnParts));
while (!fqnPartsQueue.isEmpty()) {
var memberName = fqnPartsQueue.poll();
var memberOpt = currentType.resolveMember(memberName);
if (memberOpt.isEmpty()) {
if (currentType instanceof ModuleType) {
// The type is part of an unresolved submodule
// Create a lazy type for it
return lazyTypesContext.getOrCreateLazyType(fullyQualifiedName);
} else {
// The type is an unknown member of an already resolved type
// Default to UNKNOWN
return PythonType.UNKNOWN;
}
}
currentType = memberOpt.get();
}
return currentType;
}

private PythonType convertToClassType(ClassSymbol symbol, Map<Symbol, PythonType> createdTypesBySymbol) {
if (createdTypesBySymbol.containsKey(symbol)) {
return createdTypesBySymbol.get(symbol);
}
ClassType classType = new ClassType(symbol.name(), symbol.definitionLocation());
createdTypesBySymbol.put(symbol, classType);
Set<Member> members =
symbol.declaredMembers().stream().map(m -> new Member(m.name(), convertToType(m, createdTypesBySymbol))).collect(Collectors.toSet());
classType.members().addAll(members);

Optional.of(symbol)
.filter(ClassSymbolImpl.class::isInstance)
.map(ClassSymbolImpl.class::cast)
.filter(ClassSymbolImpl::shouldSearchHierarchyInTypeshed)
.map(ClassSymbol::superClassesFqn)
.map(fqns -> fqns.stream().map(this::resolvePossibleLazyType))
.or(() -> Optional.of(symbol)
.map(ClassSymbol::superClasses)
.map(Collection::stream)
.map(symbols -> symbols.map(s -> convertToType(s, createdTypesBySymbol))))
.stream()
.flatMap(Function.identity())
.map(LazyTypeWrapper::new)
.forEach(classType.superClasses()::add);

return classType;
}

private ParameterV2 convertParameter(FunctionSymbol.Parameter parameter) {
var typeWrapper = Optional.ofNullable(((FunctionSymbolImpl.ParameterImpl) parameter).annotatedTypeName())
.map(lazyTypesContext::getOrCreateLazyTypeWrapper)
.orElse(TypeWrapper.UNKNOWN_TYPE_WRAPPER);

return new ParameterV2(parameter.name(),
typeWrapper,
parameter.hasDefaultValue(),
parameter.isKeywordOnly(),
parameter.isPositionalOnly(),
parameter.isKeywordVariadic(),
parameter.isPositionalVariadic(),
parameter.location());
}

private PythonType convertToUnionType(AmbiguousSymbol ambiguousSymbol, Map<Symbol, PythonType> createdTypesBySymbol) {
Set<PythonType> pythonTypes = ambiguousSymbol.alternatives().stream().map(a -> convertToType(a, createdTypesBySymbol)).collect(Collectors.toSet());
return new UnionType(pythonTypes);
}

private PythonType convertToType(Symbol symbol, Map<Symbol, PythonType> createdTypesBySymbol) {
return switch (symbol.kind()) {
case CLASS -> convertToClassType((ClassSymbol) symbol, createdTypesBySymbol);
case FUNCTION -> convertToFunctionType((FunctionSymbol) symbol, createdTypesBySymbol);
case AMBIGUOUS -> convertToUnionType((AmbiguousSymbol) symbol, createdTypesBySymbol);
// Symbols that are neither classes or function nor ambiguous symbols whose alternatives are all classes or functions are considered of unknown type
case OTHER -> PythonType.UNKNOWN;
};
}
}
6 changes: 4 additions & 2 deletions .../main/java/org/sonar/python/semantic/v2/converter/AnyDescriptorToPythonTypeConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.sonar.python.index.Descriptor;
import org.sonar.python.semantic.v2.LazyTypesContext;
import org.sonar.python.types.v2.PythonType;
import org.sonar.python.types.v2.TypeOrigin;

public class AnyDescriptorToPythonTypeConverter {

Expand All @@ -39,10 +40,11 @@ Descriptor.Kind.FUNCTION, new FunctionDescriptorToPythonTypeConverter(),
Descriptor.Kind.VARIABLE, new VariableDescriptorToPythonTypeConverter(),
Descriptor.Kind.AMBIGUOUS, new AmbiguousDescriptorToPythonTypeConverter()
));

}

public PythonType convert(Descriptor from) {
var ctx = new ConversionContext(lazyTypesContext, this::convert);
public PythonType convert(Descriptor from, TypeOrigin typeOrigin) {
var ctx = new ConversionContext(lazyTypesContext, this::convert, typeOrigin);
return convert(ctx, from);
}

Expand Down
9 changes: 8 additions & 1 deletion python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/ConversionContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,29 @@
import org.sonar.python.index.Descriptor;
import org.sonar.python.semantic.v2.LazyTypesContext;
import org.sonar.python.types.v2.PythonType;
import org.sonar.python.types.v2.TypeOrigin;

public class ConversionContext {
private final LazyTypesContext lazyTypesContext;
private final DescriptorToPythonTypeConverter converter;
private final Deque<PythonType> parents;
private final TypeOrigin typeOrigin;

public ConversionContext(LazyTypesContext lazyTypesContext, DescriptorToPythonTypeConverter converter) {
public ConversionContext(LazyTypesContext lazyTypesContext, DescriptorToPythonTypeConverter converter, TypeOrigin typeOrigin) {
this.lazyTypesContext = lazyTypesContext;
this.converter = converter;
this.parents = new ArrayDeque<>();
this.typeOrigin = typeOrigin;
}

public LazyTypesContext lazyTypesContext() {
return lazyTypesContext;
}

public TypeOrigin typeOrigin() {
return typeOrigin;
}

public PythonType convert(Descriptor from) {
return converter.convert(this, from);
}
Expand Down
3 changes: 1 addition & 2 deletions .../java/org/sonar/python/semantic/v2/converter/FunctionDescriptorToPythonTypeConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.sonar.python.semantic.v2.FunctionTypeBuilder;
import org.sonar.python.types.v2.ParameterV2;
import org.sonar.python.types.v2.PythonType;
import org.sonar.python.types.v2.TypeOrigin;
import org.sonar.python.types.v2.TypeUtils;

public class FunctionDescriptorToPythonTypeConverter implements DescriptorToPythonTypeConverter {
Expand All @@ -50,7 +49,7 @@ public PythonType convert(ConversionContext ctx, FunctionDescriptor from) {
.map(TypeUtils::ensureWrappedObjectType)
.orElse(PythonType.UNKNOWN);

var typeOrigin = TypeOrigin.STUB;
var typeOrigin = ctx.typeOrigin();

var hasVariadicParameter = hasVariadicParameter(parameters);

Expand Down
4 changes: 2 additions & 2 deletions python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -722,7 +722,7 @@ void inferFunctionParameterTypesMultiFile() {
assertThat(fooType.parameters().get(0).declaredType().type().unwrappedType()).isEqualTo(intType);

FunctionType foo2Type = (FunctionType) ((ExpressionStatement) fileInput.statements().statements().get(2)).expressions().get(0).typeV2();
assertThat(foo2Type.parameters()).extracting(ParameterV2::declaredType).extracting(TypeWrapper::type).containsExactly(dictType, aType);
assertThat(foo2Type.parameters()).extracting(ParameterV2::declaredType).extracting(TypeWrapper::type).extracting(PythonType::unwrappedType).containsExactly(dictType, aType);
assertThat(foo2Type.parameters()).extracting(ParameterV2::location).containsExactly(
new LocationInFile(modFileId, 3, 9, 3, 17),
new LocationInFile(modFileId, 3, 19, 3, 24));
Expand Down Expand Up @@ -2321,7 +2321,7 @@ void resolveIncorrectLazyType2() {

ClassSymbol symbol = Mockito.mock(ClassSymbolImpl.class);
Mockito.when(symbol.kind()).thenReturn(Symbol.Kind.OTHER);
assertThat(symbolsModuleTypeProvider.resolvePossibleLazyType("typing.Iterable.unknown")).isEqualTo(PythonType.UNKNOWN);
assertThat(PROJECT_LEVEL_TYPE_TABLE.lazyTypesContext().getOrCreateLazyType("typing.Iterable.unknown").resolve()).isEqualTo(PythonType.UNKNOWN);
}

@Test
Expand Down
7 changes: 4 additions & 3 deletions ...-frontend/src/test/java/org/sonar/python/semantic/v2/converter/ConversionContextTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,15 @@
import org.sonar.python.index.Descriptor;
import org.sonar.python.semantic.v2.LazyTypesContext;
import org.sonar.python.types.v2.PythonType;
import org.sonar.python.types.v2.TypeOrigin;

class ConversionContextTest {

@Test
void lazyTypeContextTest() {
var expectedLazyTypeContext = Mockito.mock(LazyTypesContext.class);
var rootConverter = Mockito.mock(DescriptorToPythonTypeConverter.class);
var ctx = new ConversionContext(expectedLazyTypeContext, rootConverter);
var ctx = new ConversionContext(expectedLazyTypeContext, rootConverter, TypeOrigin.LOCAL);
var lazyTypesContext = ctx.lazyTypesContext();
Assertions.assertSame(expectedLazyTypeContext, lazyTypesContext);
}
Expand All @@ -41,7 +42,7 @@ void lazyTypeContextTest() {
void parentsTest() {
var expectedLazyTypeContext = Mockito.mock(LazyTypesContext.class);
var rootConverter = Mockito.mock(DescriptorToPythonTypeConverter.class);
var ctx = new ConversionContext(expectedLazyTypeContext, rootConverter);
var ctx = new ConversionContext(expectedLazyTypeContext, rootConverter, TypeOrigin.LOCAL);
var firstParent = Mockito.mock(PythonType.class);
var secondParent = Mockito.mock(PythonType.class);

Expand All @@ -63,7 +64,7 @@ void convertTest() {

var lazyTypeContext = Mockito.mock(LazyTypesContext.class);
var rootConverter = Mockito.mock(DescriptorToPythonTypeConverter.class);
var ctx = new ConversionContext(lazyTypeContext, rootConverter);
var ctx = new ConversionContext(lazyTypeContext, rootConverter, TypeOrigin.LOCAL);

Mockito.when(rootConverter.convert(ctx, descriptor))
.thenReturn(expectedType);
Expand Down
9 changes: 5 additions & 4 deletions ...test/java/org/sonar/python/semantic/v2/converter/DescriptorToPythonTypeConverterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.sonar.python.types.v2.ObjectType;
import org.sonar.python.types.v2.ParameterV2;
import org.sonar.python.types.v2.PythonType;
import org.sonar.python.types.v2.TypeOrigin;
import org.sonar.python.types.v2.TypeWrapper;
import org.sonar.python.types.v2.UnionType;

Expand Down Expand Up @@ -103,7 +104,7 @@ void ambiguousDescriptorConversionTest() {
Mockito.when(descriptor.kind()).thenReturn(Descriptor.Kind.AMBIGUOUS);
Mockito.when(descriptor.alternatives()).thenReturn(Set.of(descriptorAlternative1, descriptorAlternative2));

var type = (UnionType) converter.convert(descriptor);
var type = (UnionType) converter.convert(descriptor, TypeOrigin.LOCAL);
Assertions.assertThat(type.candidates())
.hasSize(2);

Expand Down Expand Up @@ -140,7 +141,7 @@ void classDescriptorConversionTest() {
Mockito.when(lazyTypesContext.resolveLazyType(Mockito.argThat(lt -> parentClassName.equals(lt.importPath()))))
.thenReturn(resolvedParent);

var type = (ClassType) converter.convert(descriptor);
var type = (ClassType) converter.convert(descriptor, TypeOrigin.LOCAL);
Assertions.assertThat(type.name()).isEqualTo("Sample");

Assertions.assertThat(type.superClasses())
Expand Down Expand Up @@ -189,7 +190,7 @@ void functionDescriptorConversionTest() {
Mockito.when(lazyTypesContext.resolveLazyType(Mockito.argThat(lt -> returnTypeName.equals(lt.importPath()))))
.thenReturn(resolvedReturnType);

var type = (FunctionType) converter.convert(descriptor);
var type = (FunctionType) converter.convert(descriptor, TypeOrigin.LOCAL);
Assertions.assertThat(type.name()).isEqualTo("Sample");

Assertions.assertThat(type.parameters()).hasSize(1);
Expand Down Expand Up @@ -222,7 +223,7 @@ void variableDescriptorConversionTest() {
Mockito.when(lazyTypesContext.resolveLazyType(Mockito.argThat(lt -> variableTypeName.equals(lt.importPath()))))
.thenReturn(resolvedVariableType);

var type = (ObjectType) converter.convert(descriptor);
var type = (ObjectType) converter.convert(descriptor, TypeOrigin.LOCAL);
Assertions.assertThat(type)
.extracting(PythonType::unwrappedType)
.isSameAs(resolvedVariableType);
Expand Down

0 comments on commit 2970882

Please sign in to comment.