Skip to content

Commit

Permalink
New Go frontend
Browse files Browse the repository at this point in the history 
This PR contains a complete re-write of the Go language frontend based on JNA instead of a pure Go implementation. While this has a slight performance drawback (I am working on providing some numbers), it has infinitly better debugging capabilities. The handlers and most of the frontend itself is written in Kotlin using our classes and we than use a JNA bridge to a C API wrapper around the Go `go/ast` package.
  • Loading branch information
@oxisto
oxisto committed Jul 27, 2023
1 parent d890df8 commit 6687080
Show file tree
Hide file tree
Showing 56 changed files with 3,955 additions and 4,106 deletions.
8 changes: 1 addition & 7 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,10 @@ jobs:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of SonarQube analysis
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: 1.18
- uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
go-version: "1.20"
- name: Build
run: |
cd cpg-language-go/src/main/golang
Expand Down
16 changes: 8 additions & 8 deletions cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluatorTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,8 +232,8 @@ class ValueEvaluatorTest {
binOp.rhs = newLiteral(2.4, primitiveType("double"))
assertEquals(5.4, ValueEvaluator().evaluate(binOp))

binOp.lhs = newLiteral("Hello", primitiveType("String"))
binOp.rhs = newLiteral(" world", primitiveType("String"))
binOp.lhs = newLiteral("Hello", primitiveType("string"))
binOp.rhs = newLiteral(" world", primitiveType("string"))
assertEquals("Hello world", ValueEvaluator().evaluate(binOp))

binOp.rhs = newLiteral(2, primitiveType("int"))
Expand Down Expand Up @@ -293,8 +293,8 @@ class ValueEvaluatorTest {
binOp.rhs = newLiteral(2.4, primitiveType("double"))
assertEquals(3 - 2.4, ValueEvaluator().evaluate(binOp))

binOp.lhs = newLiteral("Hello", primitiveType("String"))
binOp.rhs = newLiteral(" world", primitiveType("String"))
binOp.lhs = newLiteral("Hello", primitiveType("string"))
binOp.rhs = newLiteral(" world", primitiveType("string"))
assertEquals("{-}", ValueEvaluator().evaluate(binOp))
}
}
Expand Down Expand Up @@ -351,8 +351,8 @@ class ValueEvaluatorTest {
binOp.rhs = newLiteral(2.4, primitiveType("double"))
assertEquals(3 * 2.4, ValueEvaluator().evaluate(binOp))

binOp.lhs = newLiteral("Hello", primitiveType("String"))
binOp.rhs = newLiteral(" world", primitiveType("String"))
binOp.lhs = newLiteral("Hello", primitiveType("string"))
binOp.rhs = newLiteral(" world", primitiveType("string"))
assertEquals("{*}", ValueEvaluator().evaluate(binOp))
}
}
Expand Down Expand Up @@ -414,8 +414,8 @@ class ValueEvaluatorTest {
binOp.rhs = newLiteral(2.4, primitiveType("double"))
assertEquals(3 / 2.4, ValueEvaluator().evaluate(binOp))

binOp.lhs = newLiteral("Hello", primitiveType("String"))
binOp.rhs = newLiteral(" world", primitiveType("String"))
binOp.lhs = newLiteral("Hello", primitiveType("string"))
binOp.rhs = newLiteral(" world", primitiveType("string"))
assertEquals("{/}", ValueEvaluator().evaluate(binOp))
}
}
Expand Down
75 changes: 34 additions & 41 deletions cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ import de.fraunhofer.aisec.cpg.graph.types.*
import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin
import java.util.*
import java.util.function.Consumer
import java.util.regex.Pattern
import java.util.stream.Collectors
import org.apache.commons.lang3.builder.ToStringBuilder
import org.slf4j.LoggerFactory
Expand Down Expand Up @@ -190,7 +189,7 @@ class TypeManager {
fun createOrGetTypeParameter(
templateDeclaration: TemplateDeclaration,
typeName: String,
language: Language<out LanguageFrontend>?
language: Language<*>?
): ParameterizedType {
var parameterizedType = getTypeParameter(templateDeclaration, typeName)
if (parameterizedType == null) {
Expand All @@ -217,7 +216,7 @@ class TypeManager {
@Synchronized
fun cacheType(node: HasType, type: Type) {
if (!isUnknown(type)) {
val types = typeCache.computeIfAbsent(node) { n: HasType? -> ArrayList() }
val types = typeCache.computeIfAbsent(node) { mutableListOf() }
if (!types.contains(type)) {
types.add(type)
}
Expand Down Expand Up @@ -248,7 +247,7 @@ class TypeManager {
* with parameterizedTypes as generics false otherwise
*/
fun stopPropagation(type: Type, newType: Type): Boolean {
return if (type is ObjectType && newType is ObjectType && type.name.equals(newType.name)) {
return if (type is ObjectType && newType is ObjectType && type.name == newType.name) {
(containsParameterizedType(newType.generics) &&
!containsParameterizedType(type.generics))
} else false
Expand Down Expand Up @@ -324,7 +323,7 @@ class TypeManager {
wrapState.setPointerOrigin(pointerOrigins)
wrapState.isReference = reference
wrapState.referenceType = referenceType
return if (unwrappedTypes.isEmpty() && !original.isEmpty()) {
return if (unwrappedTypes.isEmpty() && original.isNotEmpty()) {
original
} else {
unwrappedTypes
Expand Down Expand Up @@ -373,7 +372,7 @@ class TypeManager {
val scope = provider.scope ?: return Optional.empty()

// We need to find the global scope
val globalScope = provider.scope!!.globalScope ?: return Optional.empty()
val globalScope = scope.globalScope ?: return Optional.empty()
for (child in globalScope.children) {
if (child is RecordScope && child.astNode is RecordDeclaration) {
typeToRecord[(child.astNode as RecordDeclaration?)!!.toType()] =
Expand All @@ -392,11 +391,9 @@ class TypeManager {
}
val allAncestors =
types
.stream()
.map { t: Type? -> typeToRecord.getOrDefault(t, null) }
.filter { obj: RecordDeclaration? -> Objects.nonNull(obj) }
.map { r: RecordDeclaration? -> getAncestors(r, 0) }
.toList()

// normalize/reverse depth: roots start at 0, increasing on each level
for (ancestors in allAncestors) {
Expand All @@ -412,17 +409,16 @@ class TypeManager {
commonAncestors.addAll(allAncestors[i])
} else {
val others = allAncestors[i]
val newCommonAncestors: MutableSet<Ancestor> = HashSet()
val newCommonAncestors = mutableSetOf<Ancestor>()
// like Collection#retainAll but swaps relevant items out if the other set's
// matching ancestor has a higher depth
for (curr in commonAncestors) {
val toRetain =
others
.stream()
.filter { a: Ancestor -> a == curr }
.map { a: Ancestor -> if (curr.depth >= a.depth) curr else a }
.findFirst()
toRetain.ifPresent { e: Ancestor -> newCommonAncestors.add(e) }
.firstOrNull()
toRetain?.let { newCommonAncestors.add(it) }
}
commonAncestors = newCommonAncestors
}
Expand Down Expand Up @@ -498,7 +494,7 @@ class TypeManager {
}
val commonType = getCommonType(HashSet(java.util.List.of(superType, subType)), ctx)
return if (commonType.isPresent) {
commonType.get().equals(superType)
commonType.get() == superType
} else {
// If array depth matches: check whether these are types from the standard library
try {
Expand All @@ -524,7 +520,7 @@ class TypeManager {
val firstDepth = first.referenceDepth
val secondDepth = second.referenceDepth
return if (firstDepth == secondDepth) {
(first.root.name.equals(second.root.name) && first.isSimilar(second))
(first.root.name == second.root.name && first.isSimilar(second))
} else {
false
}
Expand All @@ -545,7 +541,7 @@ class TypeManager {
* @return the typedef declaration
*/
fun createTypeAlias(
frontend: LanguageFrontend,
frontend: LanguageFrontend<*, *>,
rawCode: String?,
target: Type,
alias: Type,
Expand Down Expand Up @@ -593,22 +589,25 @@ class TypeManager {
if (!newRoot.isFirstOrderType) {
return newRoot
}
return if (oldChain is ObjectType && newRoot is ObjectType) {
(newRoot.root as ObjectType).generics = oldChain.generics
newRoot
} else if (oldChain is ReferenceType) {
val reference = reWrapType(oldChain.elementType, newRoot)
val newChain = oldChain.duplicate() as ReferenceType
newChain.elementType = reference
newChain.refreshName()
newChain
} else if (oldChain is PointerType) {
val newChain = oldChain.duplicate() as PointerType
newChain.root = reWrapType(oldChain.root, newRoot)
newChain.refreshNames()
newChain
} else {
newRoot
return when {
oldChain is ObjectType && newRoot is ObjectType -> {
(newRoot.root as ObjectType).generics = oldChain.generics
newRoot
}
oldChain is ReferenceType -> {
val reference = reWrapType(oldChain.elementType, newRoot)
val newChain = oldChain.duplicate() as ReferenceType
newChain.elementType = reference
newChain.refreshName()
newChain
}
oldChain is PointerType -> {
val newChain = oldChain.duplicate() as PointerType
newChain.root = reWrapType(oldChain.root, newRoot)
newChain.refreshNames()
newChain
}
else -> newRoot
}
}

Expand All @@ -618,14 +617,14 @@ class TypeManager {
return Objects.hash(record)
}

override fun equals(o: Any?): Boolean {
if (this === o) {
override fun equals(other: Any?): Boolean {
if (this === other) {
return true
}
if (o !is Ancestor) {
if (other !is Ancestor) {
return false
}
return record == o.record
return record == other.record
}

override fun toString(): String {
Expand All @@ -639,12 +638,6 @@ class TypeManager {
companion object {
private val log = LoggerFactory.getLogger(TypeManager::class.java)

// TODO: document/remove this regexp, merge with other pattern
private val funPointerPattern = Pattern.compile("\\(?\\*(?<alias>[^()]+)\\)?\\(.*\\)")
var isTypeSystemActive = true

fun isPrimitive(type: Type, language: Language<out LanguageFrontend>): Boolean {
return language.primitiveTypeNames.contains(type.typeName)
}
}
}
104 changes: 0 additions & 104 deletions cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,12 @@
package de.fraunhofer.aisec.cpg.graph

import de.fraunhofer.aisec.cpg.TranslationContext
import de.fraunhofer.aisec.cpg.TypeManager
import de.fraunhofer.aisec.cpg.frontends.*
import de.fraunhofer.aisec.cpg.graph.Node.Companion.EMPTY_NAME
import de.fraunhofer.aisec.cpg.graph.NodeBuilder.log
import de.fraunhofer.aisec.cpg.graph.scopes.Scope
import de.fraunhofer.aisec.cpg.graph.statements.expressions.*
import de.fraunhofer.aisec.cpg.graph.types.*
import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin.ARRAY
import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin.POINTER
import de.fraunhofer.aisec.cpg.passes.inference.IsInferredProvider
import org.slf4j.LoggerFactory

Expand Down Expand Up @@ -220,107 +217,6 @@ fun MetadataProvider.newAnnotationMember(
return node
}

/**
* Creates a new [UnknownType] and sets the appropriate language, if this [MetadataProvider]
* includes a [LanguageProvider].
*/
fun MetadataProvider?.unknownType(): Type {
return if (this is LanguageProvider) {
UnknownType.getUnknownType(language)
} else {
UnknownType.getUnknownType(null)
}
}

fun LanguageProvider.incompleteType(): Type {
return IncompleteType()
}

/** Returns a [PointerType] that describes an array reference to the current type. */
context(ContextProvider)

fun Type.array(): Type {
val c =
(this@ContextProvider).ctx
?: throw TranslationException(
"Could not create type: translation context not available"
)
val type = this.reference(ARRAY)

return c.typeManager.registerType(type)
}

/** Returns a [PointerType] that describes a pointer reference to the current type. */
context(ContextProvider)

fun Type.pointer(): Type {
val c =
(this@ContextProvider).ctx
?: throw TranslationException(
"Could not create type: translation context not available"
)
val type = this.reference(POINTER)

return c.typeManager.registerType(type)
}

context(ContextProvider)

fun Type.ref(): Type {
val c =
(this@ContextProvider).ctx
?: throw TranslationException(
"Could not create type: translation context not available"
)
val type = ReferenceType(this)

return c.typeManager.registerType(type)
}

/**
* This function creates a new [Type] with the given [name]. In order to avoid unnecessary
* allocation of simple types, we do a pre-check within this function, whether a built-in type exist
* with the particular name. If it not exists, a new [ObjectType] is created and registered with the
* [TypeManager].
*/
@JvmOverloads
fun LanguageProvider.objectType(name: CharSequence, generics: List<Type> = listOf()): Type {
// First, we check, whether this is a built-in type, to avoid necessary allocations of simple
// types
val builtIn = language?.getSimpleTypeOf(name.toString())
if (builtIn != null) {
return builtIn
}

// Otherwise, we need to create a new type and register it at the type manager
val c =
(this as? ContextProvider)?.ctx
?: throw TranslationException(
"Could not create type: translation context not available"
)
val type = ObjectType(name, generics, false, language)

return c.typeManager.registerType(type)
}

/**
* This function constructs a new primitive [Type]. Primitive or built-in types are defined in
* [Language.builtInTypes]. This function will look up the type by its name, if it fails to find an
* appropriate build-in type, a [TranslationException] is thrown. Therefore, this function should
* primarily be called by language frontends if they are sure that this type is a built-in type,
* e.g., when constructing literals. It can be useful, if frontends want to check, whether all
* literal types are correctly registered as built-in types.
*
* If the frontend is not sure, what kind of type it is, it should call [objectType], which also
* does a check, whether it is a known built-in type.
*/
fun LanguageProvider.primitiveType(name: CharSequence): Type {
return language?.getSimpleTypeOf(name.toString())
?: throw TranslationException(
"Cannot find primitive type $name in language ${language?.name}. This is either an error in the language frontend or the language definition is missing a type definition."
)
}

/** Returns a new [Name] based on the [localName] and the current namespace as parent. */
fun NamespaceProvider.fqn(localName: String): Name {
return this.namespace.fqn(localName)
Expand Down
Loading

0 comments on commit 6687080

Please sign in to comment.