From 5a77492a0482fa18d774bbc4fcd5e2b5edf5780c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Max=20Leuth=C3=A4user?=
<1417198+max-leuthaeuser@users.noreply.github.com>
Date: Mon, 3 Jun 2024 11:20:40 +0200
Subject: [PATCH] Revert "Updated joern, cpg, and re-used x2cpg API (#337)"
This reverts commit 6c2c3a59230579d929bb63881906b63591bdee7d.
---
.github/workflows/pr.yml | 4 +-
build.sbt | 11 +-
project/plugins.sbt | 2 +-
.../js2cpg/astcreation/AstCreator.scala | 742 ++++++++++--------
.../js2cpg/astcreation/AstEdgeBuilder.scala | 2 +-
.../js2cpg/astcreation/AstNodeBuilder.scala | 493 +++++++-----
.../io/shiftleft/js2cpg/core/Config.scala | 121 +--
.../io/shiftleft/js2cpg/core/Js2Cpg.scala | 73 +-
.../io/shiftleft/js2cpg/core/Js2CpgMain.scala | 22 +-
.../js2cpg/core/Js2cpgArgumentsParser.scala | 286 ++++---
.../io/shiftleft/js2cpg/core/Report.scala | 105 +++
.../js2cpg/datastructures/LineAndColumn.scala | 3 +
.../io/shiftleft/js2cpg/io/FileUtils.scala | 4 +-
.../io/shiftleft/js2cpg/io/JsFileChecks.scala | 8 +-
.../io/shiftleft/js2cpg/io/PathFilter.scala | 6 +-
.../js2cpg/parser/FreshJsonParser.scala | 2 +-
.../js2cpg/parser/JavaScriptParser.scala | 2 +-
.../io/shiftleft/js2cpg/parser/JsSource.scala | 119 ++-
.../js2cpg/passes/AstCreationPass.scala | 13 +-
.../js2cpg/passes/BuiltinTypesPass.scala | 2 +-
.../shiftleft/js2cpg/passes/ConfigPass.scala | 8 +-
.../js2cpg/passes/DependenciesPass.scala | 2 +-
.../js2cpg/passes/PrivateKeyFilePass.scala | 2 +-
.../js2cpg/preprocessing/NuxtTranspiler.scala | 2 +-
.../preprocessing/TranspilationRunner.scala | 2 +-
.../TranspilingEnvironment.scala | 2 +-
.../js2cpg/preprocessing/VueTranspiler.scala | 4 +-
.../js2cpg/utils/MemoryMetrics.scala | 5 +-
.../js2cpg/utils/SourceWrapper.scala | 8 +-
.../io/shiftleft/js2cpg/utils/TimeUtils.scala | 55 ++
src/test/resources/excludes/a.js | 0
src/test/resources/excludes/folder/b.js | 0
src/test/resources/excludes/folder/c.js | 0
src/test/resources/excludes/foo.bar/d.js | 0
src/test/resources/excludes/index.js | 0
src/test/resources/excludes/tests/a.spec.js | 0
src/test/resources/excludes/tests/b.mock.js | 0
src/test/resources/excludes/tests/c.e2e.js | 0
src/test/resources/excludes/tests/d.test.js | 0
.../io/shiftleft/js2cpg/io/ExcludeTest.scala | 169 ++--
.../js2cpg/io/ExternalCommandTest.scala | 9 +-
.../shiftleft/js2cpg/io/FileUtilsTest.scala | 4 +-
.../shiftleft/js2cpg/parser/ParserTest.scala | 8 +-
.../js2cpg/passes/AbstractPassTest.scala | 12 +-
.../js2cpg/passes/BuiltinTypesPassTest.scala | 2 +-
.../js2cpg/passes/CfgCreationPassTest.scala | 11 +-
.../js2cpg/passes/ConfigPassTest.scala | 4 +-
.../js2cpg/passes/DependenciesPassTest.scala | 10 +-
.../passes/SimpleAstCreationPassTest.scala | 4 +
.../TranspilationRunnerTest.scala | 20 +-
50 files changed, 1353 insertions(+), 1010 deletions(-)
create mode 100644 src/main/scala/io/shiftleft/js2cpg/core/Report.scala
create mode 100644 src/main/scala/io/shiftleft/js2cpg/datastructures/LineAndColumn.scala
create mode 100644 src/main/scala/io/shiftleft/js2cpg/utils/TimeUtils.scala
create mode 100644 src/test/resources/excludes/a.js
create mode 100644 src/test/resources/excludes/folder/b.js
create mode 100644 src/test/resources/excludes/folder/c.js
create mode 100644 src/test/resources/excludes/foo.bar/d.js
create mode 100644 src/test/resources/excludes/index.js
create mode 100644 src/test/resources/excludes/tests/a.spec.js
create mode 100644 src/test/resources/excludes/tests/b.mock.js
create mode 100644 src/test/resources/excludes/tests/c.e2e.js
create mode 100644 src/test/resources/excludes/tests/d.test.js
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 2d2c25e28..7704b4526 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -20,7 +20,7 @@ jobs:
java-version: '11'
cache: 'sbt'
- name: Compile and run tests
- run: sbt test
+ run: sbt clean +test
formatting:
runs-on: ubuntu-22.04
steps:
@@ -34,6 +34,6 @@ jobs:
java-version: '11'
cache: 'sbt'
- name: Check formatting
- run: sbt scalafmtCheck Test/scalafmtCheck
+ run: sbt scalafmtCheck test:scalafmtCheck
- run: echo "Previous step failed because code is not formatted. Run 'sbt scalafmt'"
if: ${{ failure() }}
diff --git a/build.sbt b/build.sbt
index 55c612d18..2983f561a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,5 +1,5 @@
-val cpgVersion = "1.6.13"
-val joernVersion = "2.0.392"
+val cpgVersion = "1.6.11"
+val joernVersion = "2.0.335"
val gitCommitString = SettingKey[String]("gitSha")
@@ -27,11 +27,10 @@ lazy val commonSettings = Seq(
"io.shiftleft" %% "codepropertygraph" % cpgVersion,
"io.joern" %% "x2cpg" % joernVersion,
"com.github.scopt" %% "scopt" % "4.1.0",
- // do not update to 23.x as this requires JDK >= 19
- "org.graalvm.js" % "js" % "22.3.5",
- "com.fasterxml.jackson.core" % "jackson-databind" % "2.17.1",
+ "org.graalvm.js" % "js" % "22.3.4",
+ "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.3",
"com.atlassian.sourcemap" % "sourcemap" % "2.0.0",
- "commons-io" % "commons-io" % "2.16.1",
+ "commons-io" % "commons-io" % "2.13.0",
"org.slf4j" % "slf4j-api" % "2.0.7",
"org.apache.logging.log4j" % "log4j-slf4j2-impl" % "2.20.0" % Optional,
"org.apache.logging.log4j" % "log4j-core" % "2.20.0" % Optional,
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 89d10763d..fa69a01f3 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -2,4 +2,4 @@ addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16")
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0")
addSbtPlugin("io.shiftleft" % "sbt-ci-release-early" % "2.0.27")
addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1")
-addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.12.0")
+addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0")
diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala
index 86422f909..b1e34ddb3 100644
--- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala
@@ -58,18 +58,9 @@ import io.shiftleft.codepropertygraph.generated.nodes.{
NewTypeRef
}
import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, ModifierTypes, Operators}
-import io.shiftleft.js2cpg.datastructures.Stack.*
-import io.shiftleft.js2cpg.datastructures.scope.Scope
-import io.shiftleft.js2cpg.datastructures.OrderTracker
-import io.shiftleft.js2cpg.datastructures.scope.MethodScope
-import io.shiftleft.js2cpg.datastructures.Parameter
-import io.shiftleft.js2cpg.datastructures.scope.BlockScope
-import io.shiftleft.js2cpg.datastructures.scope.BlockScopeElement
-import io.shiftleft.js2cpg.datastructures.scope.MethodScopeElement
-import io.shiftleft.js2cpg.datastructures.scope.ResolvedReference
-import io.shiftleft.js2cpg.datastructures.scope.ScopeElement
-import io.shiftleft.js2cpg.datastructures.scope.ScopeElementIterator
-import io.shiftleft.js2cpg.datastructures.scope.ScopeType
+import io.shiftleft.js2cpg.datastructures.Stack._
+import io.shiftleft.js2cpg.datastructures._
+import io.shiftleft.js2cpg.datastructures.scope._
import io.shiftleft.js2cpg.passes.{Defines, EcmaBuiltins, PassHelpers}
import io.shiftleft.js2cpg.passes.PassHelpers.ParamNodeInitKind
import io.shiftleft.js2cpg.parser.{GeneralizingAstVisitor, JsSource}
@@ -77,7 +68,7 @@ import overflowdb.BatchedUpdate.DiffGraphBuilder
import org.slf4j.LoggerFactory
import scala.collection.mutable
-import scala.jdk.CollectionConverters.*
+import scala.jdk.CollectionConverters._
object AstCreator {
@@ -89,14 +80,16 @@ object AstCreator {
}
-class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val usedIdentNodes: Set[String])
- extends GeneralizingAstVisitor[NewNode]
- with AstNodeBuilder
- with AstEdgeBuilder {
+class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: Set[String])
+ extends GeneralizingAstVisitor[NewNode] {
import AstCreator._
- protected val scope = new Scope()
+ private val scope = new Scope()
+
+ private val astEdgeBuilder = new AstEdgeBuilder(diffGraph)
+
+ private val astNodeBuilder = new AstNodeBuilder(diffGraph, astEdgeBuilder, source, scope)
// Nested methods are not put in the AST where they are defined.
// Instead we put them directly under the METHOD in which they are
@@ -117,23 +110,26 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
private val usedVariableNames = mutable.HashMap.empty[String, Int]
private def prepareFileWrapperFunction(): NewNamespaceBlock = {
- val fileName = source.filePath
- val fileNode = createFileNode(fileName)
- val namespaceBlock = createNamespaceBlockNode(fileName + ":" + Defines.GlobalNamespace)
- addAstEdge(namespaceBlock, fileNode)
+ val fileName = source.filePath
+ val fileNode = astNodeBuilder.createFileNode(fileName)
+
+ val namespaceBlock =
+ astNodeBuilder.createNamespaceBlockNode(fileName + ":" + Defines.GlobalNamespace)
+
+ astEdgeBuilder.addAstEdge(namespaceBlock, fileNode)
namespaceBlock
}
private def addLocalToAst(local: NewLocal): Unit = {
- addAstEdge(local, localAstParentStack.head, 0)
+ astEdgeBuilder.addAstEdge(local, localAstParentStack.head, 0)
}
private def addMethodToAst(method: NewMethod): Unit = {
- addAstEdge(method, methodAstParentStack.head, 0)
+ astEdgeBuilder.addAstEdge(method, methodAstParentStack.head, 0)
}
private def addTypeDeclToAst(typeDecl: NewTypeDecl): Unit = {
- addAstEdge(typeDecl, methodAstParentStack.head, 0)
+ astEdgeBuilder.addAstEdge(typeDecl, methodAstParentStack.head, 0)
}
/** Entry point for converting ASTs with this class.
@@ -155,46 +151,46 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
module.getImports.forEach { importNode =>
- val groupId = groupIdFromImportNode(importNode)
+ val groupId = astNodeBuilder.groupIdFromImportNode(importNode)
importNode.getModuleSpecifier match {
case null =>
val defaultBinding = importNode.getImportClause.getDefaultBinding
if (defaultBinding != null) {
- createDependencyNode(defaultBinding.getName, groupId, VERSION_IMPORT)
+ astNodeBuilder.createDependencyNode(defaultBinding.getName, groupId, VERSION_IMPORT)
createImportNodeAndAttachToAst(importNode)
}
val nameSpaceImport = importNode.getImportClause.getNameSpaceImport
val namedImports = importNode.getImportClause.getNamedImports
if (nameSpaceImport != null) {
- createDependencyNode(nameSpaceImport.getBindingIdentifier.getName, groupId, VERSION_IMPORT)
+ astNodeBuilder.createDependencyNode(nameSpaceImport.getBindingIdentifier.getName, groupId, VERSION_IMPORT)
createImportNodeAndAttachToAst(importNode)
} else if (namedImports != null) {
namedImports.getImportSpecifiers.forEach { namedImport =>
- createDependencyNode(namedImport.getBindingIdentifier.getName, groupId, VERSION_IMPORT)
+ astNodeBuilder.createDependencyNode(namedImport.getBindingIdentifier.getName, groupId, VERSION_IMPORT)
createImportNodeAndAttachToAst(importNode)
}
}
case module =>
- createDependencyNode(module.getString, groupId, VERSION_IMPORT)
+ astNodeBuilder.createDependencyNode(module.getString, groupId, VERSION_IMPORT)
createImportNodeAndAttachToAst(importNode)
}
}
}
private def createImportNodeAndAttachToAst(importNode: ImportNode) = {
- val impNode = createImportNode(importNode)
+ val impNode = astNodeBuilder.createImportNode(importNode)
methodAstParentStack.headOption.collect { case namespaceBlockNode: NewNamespaceBlock =>
- addAstEdge(impNode, namespaceBlockNode)
+ astEdgeBuilder.addAstEdge(impNode, namespaceBlockNode)
}
}
override def visit(breakNode: BreakNode): NewNode = {
- createControlStructureNode(breakNode, ControlStructureTypes.BREAK)
+ astNodeBuilder.createControlStructureNode(breakNode, ControlStructureTypes.BREAK)
}
override def visit(continueNode: ContinueNode): NewNode = {
- createControlStructureNode(continueNode, ControlStructureTypes.CONTINUE)
+ astNodeBuilder.createControlStructureNode(continueNode, ControlStructureTypes.CONTINUE)
}
private def createIdentifierNode(name: String, lineAndColumnProvider: Node): NewIdentifier = {
@@ -209,7 +205,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
None
}
- createIdentifierNode(name, lineAndColumnProvider, dynamicInstanceTypeOption)
+ astNodeBuilder.createIdentifierNode(name, lineAndColumnProvider, dynamicInstanceTypeOption)
}
private def handleDestructingParameter(
@@ -222,13 +218,13 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
): Unit = {
val name =
PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, s"param$index")
- val code_ = paramComponents.map(code).mkString("{", ", ", "}")
- createParameterInNode(name, code_, methodId, paramComponents.head, new OrderTracker(index))
+ val code = paramComponents.map(source.getCode).mkString("{", ", ", "}")
+ astNodeBuilder.createParameterInNode(name, code, methodId, paramComponents.head, new OrderTracker(index))
initStatements match {
case Some(initExpr: ExpressionStatement) =>
val destructingAssignmentId =
convertDestructingAssignment(initExpr.getExpression.asInstanceOf[BinaryNode], Some(name))
- addAstEdge(destructingAssignmentId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(destructingAssignmentId, blockId, blockOrder)
case None =>
paramComponents.foreach { param =>
val paramName = param.getName
@@ -236,14 +232,14 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val paramId = createIdentifierNode(name, param)
scope.addVariableReference(name, paramId)
- val localParamLocal = createLocalNode(paramName, Defines.Any)
+ val localParamLocal = astNodeBuilder.createLocalNode(paramName, Defines.Any)
addLocalToAst(localParamLocal)
scope.addVariable(paramName, localParamLocal, MethodScope)
- val keyId = createFieldIdentifierNode(paramName, param)
- val accessId = createFieldAccessCallNode(paramId, keyId, param)
+ val keyId = astNodeBuilder.createFieldIdentifierNode(paramName, param)
+ val accessId = astNodeBuilder.createFieldAccessNode(paramId, keyId, astNodeBuilder.lineAndColumn(param))
val assignmentCallId =
- createAssignmentNode(localParamId, accessId, param)
- addAstEdge(assignmentCallId, blockId, blockOrder)
+ astNodeBuilder.createAssignmentNode(localParamId, accessId, astNodeBuilder.lineAndColumn(param))
+ astEdgeBuilder.addAstEdge(assignmentCallId, blockId, blockOrder)
blockOrder.inc()
}
case _ =>
@@ -260,36 +256,36 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
blockOrder: OrderTracker
): Unit = {
val name, code = PassHelpers.cleanParameterNodeName(parameter)
- createParameterInNode(name, code, methodId, parameter, new OrderTracker(index))
+ astNodeBuilder.createParameterInNode(name, code, methodId, parameter, new OrderTracker(index))
initStatement match {
case Some(initExpr: VarNode) =>
val paramName = initExpr.getName.getName
val localParamId = createIdentifierNode(paramName, initExpr)
val rhs = createRhsForConditionalParameterInit(initExpr.getAssignmentSource, name, initExpr)
val assignmentCallId =
- createAssignmentNode(localParamId, rhs, initExpr)
- addAstEdge(assignmentCallId, blockId, blockOrder)
+ astNodeBuilder.createAssignmentNode(localParamId, rhs, astNodeBuilder.lineAndColumn(initExpr))
+ astEdgeBuilder.addAstEdge(assignmentCallId, blockId, blockOrder)
case Some(initExpr: ExpressionStatement) =>
val destructingAssignmentId =
convertDestructingAssignment(initExpr.getExpression.asInstanceOf[BinaryNode], Some(name))
- addAstEdge(destructingAssignmentId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(destructingAssignmentId, blockId, blockOrder)
case None =>
val paramName = name
val localParamId = createIdentifierNode(paramName, parameter)
- val localParamLocal = createLocalNode(paramName, Defines.Any)
+ val localParamLocal = astNodeBuilder.createLocalNode(paramName, Defines.Any)
addLocalToAst(localParamLocal)
scope.addVariable(paramName, localParamLocal, MethodScope)
val paramId = createIdentifierNode(name, parameter)
scope.addVariableReference(name, paramId)
- val keyId = createFieldIdentifierNode(paramName, parameter)
+ val keyId = astNodeBuilder.createFieldIdentifierNode(paramName, parameter)
- val accessId = createFieldAccessCallNode(paramId, keyId, parameter)
+ val accessId = astNodeBuilder.createFieldAccessNode(paramId, keyId, astNodeBuilder.lineAndColumn(parameter))
val assignmentCallId =
- createAssignmentNode(localParamId, accessId, parameter)
- addAstEdge(assignmentCallId, blockId, blockOrder)
+ astNodeBuilder.createAssignmentNode(localParamId, accessId, astNodeBuilder.lineAndColumn(parameter))
+ astEdgeBuilder.addAstEdge(assignmentCallId, blockId, blockOrder)
case _ =>
logger.debug(s"Unhandled parameter kind: $initStatement")
}
@@ -308,26 +304,26 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val (methodName, methodFullName) = calcMethodNameAndFullName(functionNode)
- val methodId = createMethodNode(methodName, methodFullName, functionNode)
+ val methodId = astNodeBuilder.createMethodNode(methodName, methodFullName, functionNode)
addMethodToAst(methodId)
if (!functionNode.isProgram) {
- val virtualModifierId = createModifierNode(ModifierTypes.VIRTUAL)
- addAstEdge(virtualModifierId, methodId)
+ val virtualModifierId = astNodeBuilder.createModifierNode(ModifierTypes.VIRTUAL)
+ astEdgeBuilder.addAstEdge(virtualModifierId, methodId)
}
val methodRefId =
if (!shouldCreateFunctionReference) {
None
} else {
- Some(createMethodRefNode(methodName, methodFullName, functionNode))
+ Some(astNodeBuilder.createMethodRefNode(methodName, methodFullName, functionNode))
}
methodAstParentStack.push(methodId)
val block = functionNode.getBody
- val blockId = createBlockNode(block, functionNode.isProgram)
- addAstEdge(blockId, methodId, 1)
+ val blockId = astNodeBuilder.createBlockNode(block, functionNode.isProgram)
+ astEdgeBuilder.addAstEdge(blockId, methodId, 1)
val capturingRefId =
if (shouldCreateFunctionReference) {
@@ -339,9 +335,15 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val parameterOrderTracker = new OrderTracker(0)
// We always create an instance parameter because in JS every function could get called with an instance.
- createParameterInNode("this", "this", methodId, functionNode, parameterOrderTracker)
+ astNodeBuilder.createParameterInNode("this", "this", methodId, functionNode, parameterOrderTracker)
functionNode.getParameters.forEach { parameter =>
- createParameterInNode(parameter.getName, source.getString(parameter), methodId, parameter, parameterOrderTracker)
+ astNodeBuilder.createParameterInNode(
+ parameter.getName,
+ source.getString(parameter),
+ methodId,
+ parameter,
+ parameterOrderTracker
+ )
}
val blockOrder = new OrderTracker()
@@ -368,15 +370,15 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
val methodReturnId =
- createMethodReturnNode(functionNode)
- addAstEdge(methodReturnId, methodId, 2)
+ astNodeBuilder.createMethodReturnNode(astNodeBuilder.lineAndColumn(functionNode))
+ astEdgeBuilder.addAstEdge(methodReturnId, methodId, 2)
val filteredFunctionBodyStatements =
functionBodyStatements.filterNot(PassHelpers.isSynthetic(_, destructingParameters.flatten ++ syntheticParameters))
visitStatements(
filteredFunctionBodyStatements.asJava,
statementId => {
- addAstEdge(statementId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(statementId, blockId, blockOrder)
}
)
localAstParentStack.pop()
@@ -385,7 +387,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
methodAstParentStack.pop()
- createFunctionTypeAndTypeDecl(functionNode, methodId, methodAstParentStack.head, methodName, methodFullName)
+ createFunctionTypeAndTypeDecl(methodId, methodAstParentStack.head, methodName, methodFullName)
(methodRefId, methodId)
}
@@ -403,7 +405,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
override def visit(debuggerNode: DebuggerNode): NewNode = {
// If no debugging is available, the debugger statement has no effect.
- createUnknownNode(debuggerNode)
+ astNodeBuilder.createUnknownNode(debuggerNode)
}
override def visit(functionNode: FunctionNode): NewNode = {
@@ -418,29 +420,28 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
private def createFunctionTypeAndTypeDecl(
- node: Node,
methodId: NewMethod,
parentNodeId: NewNode,
methodName: String,
methodFullName: String
): Unit = {
- createTypeNode(methodName, methodFullName)
+ astNodeBuilder.createTypeNode(methodName, methodFullName)
val astParentType = parentNodeId.label
val astParentFullName = parentNodeId.properties("FULL_NAME").toString
val functionTypeDeclId =
- createTypeDeclNode(node, methodName, methodFullName, astParentType, astParentFullName, Some(Defines.Any))
+ astNodeBuilder.createTypeDeclNode(methodName, methodFullName, astParentType, astParentFullName, Some(Defines.Any))
addTypeDeclToAst(functionTypeDeclId)
// Problem for https://github.com/ShiftLeftSecurity/codescience/issues/3626 here.
// As the type (thus, the signature) of the function node is unknown (i.e., ANY*)
// we can't generate the correct binding with signature.
- val functionBindingId = createBindingNode()
- addBindsEdge(functionBindingId, functionTypeDeclId)
+ val functionBindingId = astNodeBuilder.createBindingNode()
+ astEdgeBuilder.addBindsEdge(functionBindingId, functionTypeDeclId)
- addRefEdge(methodId, functionBindingId)
+ astEdgeBuilder.addRefEdge(methodId, functionBindingId)
}
override def visit(classNode: ClassNode): NewNode = {
@@ -448,7 +449,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val metaTypeName = s"$typeName"
val metaTypeFullName = s"$typeFullName"
- createTypeNode(typeName, typeFullName)
+ astNodeBuilder.createTypeNode(typeName, typeFullName)
// We do not need to look at classNode.getClassHeritage because
// the CPG only allows us to encode inheriting from fully known
@@ -459,13 +460,12 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString
val typeDeclId =
- createTypeDeclNode(classNode, typeName, typeFullName, astParentType, astParentFullName, inheritsFrom = None)
+ astNodeBuilder.createTypeDeclNode(typeName, typeFullName, astParentType, astParentFullName, inheritsFrom = None)
- createTypeNode(metaTypeName, metaTypeFullName)
+ astNodeBuilder.createTypeNode(metaTypeName, metaTypeFullName)
val metaTypeDeclId =
- createTypeDeclNode(
- classNode,
+ astNodeBuilder.createTypeDeclNode(
metaTypeName,
metaTypeFullName,
astParentType,
@@ -477,7 +477,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
addTypeDeclToAst(metaTypeDeclId)
val metaTypeRefId =
- createTypeRefNode(s"class $typeName", metaTypeFullName, classNode)
+ astNodeBuilder.createTypeRefNode(s"class $typeName", metaTypeFullName, classNode)
methodAstParentStack.push(typeDeclId)
dynamicInstanceTypeStack.push(typeFullName)
@@ -488,10 +488,10 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val constructor = classNode.getConstructor.getValue.asInstanceOf[FunctionNode]
val constructorId = createFunctionNode(constructor, shouldCreateFunctionReference = false)._2
- val constructorBindingId = createBindingNode()
- addBindsEdge(constructorBindingId, metaTypeDeclId)
+ val constructorBindingId = astNodeBuilder.createBindingNode()
+ astEdgeBuilder.addBindsEdge(constructorBindingId, metaTypeDeclId)
- addRefEdge(constructorId, constructorBindingId)
+ astEdgeBuilder.addRefEdge(constructorId, constructorBindingId)
val memberOrderTracker = new OrderTracker()
classNode.getClassElements.forEach { classElement =>
@@ -507,16 +507,16 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
// identical.
val functionFullName = calcMethodNameAndFullName(function)._2
val dynamicTypeHintFullName = Some(functionFullName)
- createMemberNode(memberName, classElement, dynamicTypeHintFullName)
+ astNodeBuilder.createMemberNode(memberName, classElement, dynamicTypeHintFullName)
case _ =>
- createMemberNode(memberName, classElement, dynamicTypeOption = None)
+ astNodeBuilder.createMemberNode(memberName, classElement, dynamicTypeOption = None)
}
if (classElement.isStatic) {
// Static member belong to the meta class.
- addAstEdge(memberId, metaTypeDeclId, memberOrderTracker)
+ astEdgeBuilder.addAstEdge(memberId, metaTypeDeclId, memberOrderTracker)
} else {
- addAstEdge(memberId, typeDeclId, memberOrderTracker)
+ astEdgeBuilder.addAstEdge(memberId, typeDeclId, memberOrderTracker)
}
}
}
@@ -533,20 +533,20 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
override def visit(ifNode: IfNode): NewNode = {
- val ifNodeId = createControlStructureNode(ifNode, ControlStructureTypes.IF)
+ val ifNodeId = astNodeBuilder.createControlStructureNode(ifNode, ControlStructureTypes.IF)
Option(ifNode.getTest).foreach { testNode =>
val testId = testNode.accept(this)
- addAstEdge(testId, ifNodeId, 1)
- addConditionEdge(testId, ifNodeId)
+ astEdgeBuilder.addAstEdge(testId, ifNodeId, 1)
+ astEdgeBuilder.addConditionEdge(testId, ifNodeId)
}
Option(ifNode.getPass).foreach { passNode =>
val passId = passNode.accept(this)
- addAstEdge(passId, ifNodeId, 2)
+ astEdgeBuilder.addAstEdge(passId, ifNodeId, 2)
}
Option(ifNode.getFail).foreach { failNode =>
val failId = failNode.accept(this)
- addAstEdge(failId, ifNodeId, 3)
+ astEdgeBuilder.addAstEdge(failId, ifNodeId, 3)
}
ifNodeId
@@ -561,29 +561,29 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
): NewCall = {
val argIds = callNode.getArgs.asScala.map(_.accept(this))
- val baseCode = codeOf(functionBaseId)
+ val baseCode = astNodeBuilder.codeOf(functionBaseId)
val propertyCode = functionPropertyId match {
- case Some(id) => "." + codeOf(id)
+ case Some(id) => "." + astNodeBuilder.codeOf(id)
case None => ""
}
- val argsCode = argIds.map(codeOf).mkString("(", ", ", ")")
+ val argsCode = argIds.map(astNodeBuilder.codeOf).mkString("(", ", ", ")")
val code = s"$baseCode$propertyCode$argsCode"
val callId =
- createCallNode(code, "", DispatchTypes.DYNAMIC_DISPATCH, callNode)
+ astNodeBuilder.createCallNode(code, "", DispatchTypes.DYNAMIC_DISPATCH, astNodeBuilder.lineAndColumn(callNode))
val orderTracker = new OrderTracker(0)
val argIndexTracker = new OrderTracker(0)
- addAstEdge(receiverId, callId, orderTracker)
- addReceiverEdge(receiverId, callId)
+ astEdgeBuilder.addAstEdge(receiverId, callId, orderTracker)
+ astEdgeBuilder.addReceiverEdge(receiverId, callId)
- addAstEdge(baseId, callId, orderTracker)
- addArgumentEdge(baseId, callId, argIndexTracker)
+ astEdgeBuilder.addAstEdge(baseId, callId, orderTracker)
+ astEdgeBuilder.addArgumentEdge(baseId, callId, argIndexTracker)
argIds.foreach { argId =>
- addAstEdge(argId, callId, orderTracker)
- addArgumentEdge(argId, callId, argIndexTracker)
+ astEdgeBuilder.addAstEdge(argId, callId, orderTracker)
+ astEdgeBuilder.addArgumentEdge(argId, callId, argIndexTracker)
}
callId
@@ -597,13 +597,18 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
case identNode: IdentNode =>
identNode.getName
}
- val callId = createStaticCallNode(callNode.toString(), methodName, methodFullName, callNode)
+ val callId = astNodeBuilder.createStaticCallNode(
+ callNode.toString(),
+ methodName,
+ methodFullName,
+ astNodeBuilder.lineAndColumn(callNode)
+ )
val orderTracker = new OrderTracker()
val argIndexTracker = new OrderTracker()
callNode.getArgs.forEach { arg =>
val argId = arg.accept(this)
- addAstEdge(argId, callId, orderTracker)
- addArgumentEdge(argId, callId, argIndexTracker)
+ astEdgeBuilder.addAstEdge(argId, callId, orderTracker)
+ astEdgeBuilder.addArgumentEdge(argId, callId, argIndexTracker)
}
callId
@@ -644,12 +649,21 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val baseId = base.accept(this)
- val tmpAssignmentId = createAssignmentNode(baseTmpId, baseId, base, withParenthesis = true)
+ val tmpAssignmentId = astNodeBuilder.createAssignmentNode(
+ baseTmpId,
+ baseId,
+ astNodeBuilder.lineAndColumn(base),
+ withParenthesis = true
+ )
val memberId =
- createFieldIdentifierNode(functionAccessNode.getProperty, functionAccessNode)
+ astNodeBuilder.createFieldIdentifierNode(functionAccessNode.getProperty, functionAccessNode)
- val fieldAccessId = createFieldAccessCallNode(tmpAssignmentId, memberId, functionAccessNode)
+ val fieldAccessId = astNodeBuilder.createFieldAccessNode(
+ tmpAssignmentId,
+ memberId,
+ astNodeBuilder.lineAndColumn(functionAccessNode)
+ )
val thisTmpId = createIdentifierNode(tmpVarName, functionAccessNode)
scope.addVariableReference(tmpVarName, thisTmpId)
@@ -682,45 +696,45 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
} else {
ControlStructureTypes.WHILE
}
- val whileNodeId = createControlStructureNode(whileNode, controlStructureType)
+ val whileNodeId = astNodeBuilder.createControlStructureNode(whileNode, controlStructureType)
if (whileNode.isDoWhile) {
val bodyId = whileNode.getBody.accept(this)
- addAstEdge(bodyId, whileNodeId, 1)
+ astEdgeBuilder.addAstEdge(bodyId, whileNodeId, 1)
val testId = whileNode.getTest.accept(this)
- addAstEdge(testId, whileNodeId, 2)
- addConditionEdge(testId, whileNodeId)
+ astEdgeBuilder.addAstEdge(testId, whileNodeId, 2)
+ astEdgeBuilder.addConditionEdge(testId, whileNodeId)
} else {
val testId = whileNode.getTest.accept(this)
- addAstEdge(testId, whileNodeId, 1)
- addConditionEdge(testId, whileNodeId)
+ astEdgeBuilder.addAstEdge(testId, whileNodeId, 1)
+ astEdgeBuilder.addConditionEdge(testId, whileNodeId)
val bodyId = whileNode.getBody.accept(this)
- addAstEdge(bodyId, whileNodeId, 2)
+ astEdgeBuilder.addAstEdge(bodyId, whileNodeId, 2)
}
whileNodeId
}
private def createForNode(forNode: ForNode): NewControlStructure = {
- val forNodeId = createControlStructureNode(forNode, ControlStructureTypes.FOR)
+ val forNodeId = astNodeBuilder.createControlStructureNode(forNode, ControlStructureTypes.FOR)
Option(forNode.getInit).foreach { initNode =>
val initNodeId = initNode.accept(this)
- addAstEdge(initNodeId, forNodeId, 1)
+ astEdgeBuilder.addAstEdge(initNodeId, forNodeId, 1)
}
val testNodeId = forNode.getTest match {
case null =>
// If the for condition is empty, this ensures that there is always a condition (true) present.
val testNodeId =
- createLiteralNode("true", forNode, Some(Defines.Boolean))
+ astNodeBuilder.createLiteralNode("true", astNodeBuilder.lineAndColumn(forNode), Some(Defines.Boolean))
testNodeId
case testNode if testNode.getExpression == null =>
// If the for condition is empty, this ensures that there is always a condition (true) present.
val testNodeId =
- createLiteralNode("true", forNode, Some(Defines.Boolean))
+ astNodeBuilder.createLiteralNode("true", astNodeBuilder.lineAndColumn(forNode), Some(Defines.Boolean))
testNodeId
// The test of a forNode can be a JoinPredecessorExpression which does not wrap any expression.
// This only happens for "for (x in y)" style loops.
@@ -729,16 +743,16 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
case testNode if testNode.getExpression != null =>
testNode.accept(this)
}
- addAstEdge(testNodeId, forNodeId, 2)
+ astEdgeBuilder.addAstEdge(testNodeId, forNodeId, 2)
Option(forNode.getModify).foreach { modifyNode =>
val modifyNodeId = modifyNode.accept(this)
- addAstEdge(modifyNodeId, forNodeId, 3)
+ astEdgeBuilder.addAstEdge(modifyNodeId, forNodeId, 3)
}
if (forNode.getBody.getStatementCount != 0) {
val bodyId = forNode.getBody.accept(this)
- addAstEdge(bodyId, forNodeId, 4)
+ astEdgeBuilder.addAstEdge(bodyId, forNodeId, 4)
}
forNodeId
@@ -756,7 +770,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
private def createForInOrOfNode(forNode: ForNode): NewBlock = {
// surrounding block:
val blockOrder = new OrderTracker()
- val blockId = createBlockNode(forNode)
+ val blockId = astNodeBuilder.createBlockNode(forNode)
scope.pushNewBlockScope(blockId)
localAstParentStack.push(blockId)
@@ -766,134 +780,143 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
// _iterator assignment:
val iteratorName =
PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_iterator")
- val iteratorLocalId = createLocalNode(iteratorName, Defines.Any)
+ val iteratorLocalId = astNodeBuilder.createLocalNode(iteratorName, Defines.Any)
addLocalToAst(iteratorLocalId)
val iteratorId = createIdentifierNode(iteratorName, forNode)
- val callId = createCallNode(
+ val callId = astNodeBuilder.createCallNode(
"Object.keys(" + collectionName + ")[Symbol.iterator]()",
"",
DispatchTypes.DYNAMIC_DISPATCH,
- forNode
+ astNodeBuilder.lineAndColumn(forNode)
)
val thisId = createIdentifierNode("this", forNode)
- val indexCallId = createCallNode(
+ val indexCallId = astNodeBuilder.createCallNode(
"Object.keys(" + collectionName + ")[Symbol.iterator]",
Operators.indexAccess,
DispatchTypes.STATIC_DISPATCH,
- forNode
+ astNodeBuilder.lineAndColumn(forNode)
)
- val objectKeysCallId =
- createStaticCallNode("Object.keys(" + collectionName + ")", "keys", "Object.keys", forNode)
+ val objectKeysCallId = astNodeBuilder.createStaticCallNode(
+ "Object.keys(" + collectionName + ")",
+ "keys",
+ "Object.keys",
+ astNodeBuilder.lineAndColumn(forNode)
+ )
val argId = collection.accept(this)
- addAstEdge(argId, objectKeysCallId, 1)
- addArgumentEdge(argId, objectKeysCallId, 1)
+ astEdgeBuilder.addAstEdge(argId, objectKeysCallId, 1)
+ astEdgeBuilder.addArgumentEdge(argId, objectKeysCallId, 1)
val indexBaseId = createIdentifierNode("Symbol", forNode)
- val indexMemberId = createFieldIdentifierNode("iterator", forNode)
+ val indexMemberId = astNodeBuilder.createFieldIdentifierNode("iterator", forNode)
val indexAccessId =
- createFieldAccessCallNode(indexBaseId, indexMemberId, forNode)
+ astNodeBuilder.createFieldAccessNode(indexBaseId, indexMemberId, astNodeBuilder.lineAndColumn(forNode))
- addAstEdge(objectKeysCallId, indexCallId, 1)
- addArgumentEdge(objectKeysCallId, indexCallId, 1)
- addAstEdge(indexAccessId, indexCallId, 2)
- addArgumentEdge(indexAccessId, indexCallId, 2)
+ astEdgeBuilder.addAstEdge(objectKeysCallId, indexCallId, 1)
+ astEdgeBuilder.addArgumentEdge(objectKeysCallId, indexCallId, 1)
+ astEdgeBuilder.addAstEdge(indexAccessId, indexCallId, 2)
+ astEdgeBuilder.addArgumentEdge(indexAccessId, indexCallId, 2)
- addAstEdge(indexCallId, callId, 0)
- addReceiverEdge(indexCallId, callId)
+ astEdgeBuilder.addAstEdge(indexCallId, callId, 0)
+ astEdgeBuilder.addReceiverEdge(indexCallId, callId)
- addAstEdge(thisId, callId, 1)
- addArgumentEdge(thisId, callId, 0)
+ astEdgeBuilder.addAstEdge(thisId, callId, 1)
+ astEdgeBuilder.addArgumentEdge(thisId, callId, 0)
val iteratorAssignmentId =
- createCallNode(
+ astNodeBuilder.createCallNode(
iteratorName + " = " + "Object.keys(" + collectionName + ")[Symbol.iterator]()",
Operators.assignment,
DispatchTypes.STATIC_DISPATCH,
- forNode
+ astNodeBuilder.lineAndColumn(forNode)
)
- addAstEdge(iteratorId, iteratorAssignmentId, 1)
- addArgumentEdge(iteratorId, iteratorAssignmentId, 1)
- addAstEdge(callId, iteratorAssignmentId, 2)
- addArgumentEdge(callId, iteratorAssignmentId, 2)
- addAstEdge(iteratorAssignmentId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(iteratorId, iteratorAssignmentId, 1)
+ astEdgeBuilder.addArgumentEdge(iteratorId, iteratorAssignmentId, 1)
+ astEdgeBuilder.addAstEdge(callId, iteratorAssignmentId, 2)
+ astEdgeBuilder.addArgumentEdge(callId, iteratorAssignmentId, 2)
+ astEdgeBuilder.addAstEdge(iteratorAssignmentId, blockId, blockOrder)
// _result:
val resultName =
PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_result")
- val resultLocalId = createLocalNode(resultName, Defines.Any)
+ val resultLocalId = astNodeBuilder.createLocalNode(resultName, Defines.Any)
addLocalToAst(resultLocalId)
val resultId = createIdentifierNode(resultName, forNode)
- addAstEdge(resultId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(resultId, blockId, blockOrder)
// loop variable:
val loopVariableName = forNode.getInit.toString()
- val loopVariableLocalId = createLocalNode(loopVariableName, Defines.Any)
+ val loopVariableLocalId = astNodeBuilder.createLocalNode(loopVariableName, Defines.Any)
addLocalToAst(loopVariableLocalId)
- val loopVariableId = createIdentifierNode(loopVariableName, forNode.getInit)
- addAstEdge(loopVariableId, blockId, blockOrder)
+ val loopVariableId = createIdentifierNode(loopVariableName, forNode)
+ astEdgeBuilder.addAstEdge(loopVariableId, blockId, blockOrder)
// while loop:
val whileLoopId =
- createControlStructureNode(forNode, ControlStructureTypes.WHILE)
- addAstEdge(whileLoopId, blockId, blockOrder)
+ astNodeBuilder.createControlStructureNode(forNode, ControlStructureTypes.WHILE)
+ astEdgeBuilder.addAstEdge(whileLoopId, blockId, blockOrder)
// while loop test:
- val testCallId = createCallNode(
+ val testCallId = astNodeBuilder.createCallNode(
"!(" + resultName + " = " + iteratorName + ".next()).done",
Operators.not,
DispatchTypes.STATIC_DISPATCH,
- forNode
+ astNodeBuilder.lineAndColumn(forNode)
)
- val doneBaseId = createCallNode(
+ val doneBaseId = astNodeBuilder.createCallNode(
"(" + resultName + " = " + iteratorName + ".next())",
Operators.assignment,
DispatchTypes.STATIC_DISPATCH,
- forNode
+ astNodeBuilder.lineAndColumn(forNode)
)
val lhsId = createIdentifierNode(resultName, forNode)
- val rhsId = createCallNode(iteratorName + ".next()", "", DispatchTypes.DYNAMIC_DISPATCH, forNode)
+ val rhsId = astNodeBuilder.createCallNode(
+ iteratorName + ".next()",
+ "",
+ DispatchTypes.DYNAMIC_DISPATCH,
+ astNodeBuilder.lineAndColumn(forNode)
+ )
val nextBaseId = createIdentifierNode(iteratorName, forNode)
- val nextMemberId = createFieldIdentifierNode("next", forNode)
+ val nextMemberId = astNodeBuilder.createFieldIdentifierNode("next", forNode)
val nextReceiverId =
- createFieldAccessCallNode(nextBaseId, nextMemberId, forNode)
+ astNodeBuilder.createFieldAccessNode(nextBaseId, nextMemberId, astNodeBuilder.lineAndColumn(forNode))
val thisNextId = createIdentifierNode(iteratorName, forNode)
- addAstEdge(nextReceiverId, rhsId, 0)
- addReceiverEdge(nextReceiverId, rhsId)
+ astEdgeBuilder.addAstEdge(nextReceiverId, rhsId, 0)
+ astEdgeBuilder.addReceiverEdge(nextReceiverId, rhsId)
- addAstEdge(thisNextId, rhsId, 1)
- addArgumentEdge(thisNextId, rhsId, 0)
+ astEdgeBuilder.addAstEdge(thisNextId, rhsId, 1)
+ astEdgeBuilder.addArgumentEdge(thisNextId, rhsId, 0)
- addAstEdge(lhsId, doneBaseId, 1)
- addArgumentEdge(lhsId, doneBaseId, 1)
- addAstEdge(rhsId, doneBaseId, 2)
- addArgumentEdge(rhsId, doneBaseId, 2)
+ astEdgeBuilder.addAstEdge(lhsId, doneBaseId, 1)
+ astEdgeBuilder.addArgumentEdge(lhsId, doneBaseId, 1)
+ astEdgeBuilder.addAstEdge(rhsId, doneBaseId, 2)
+ astEdgeBuilder.addArgumentEdge(rhsId, doneBaseId, 2)
- val doneMemberId = createFieldIdentifierNode("done", forNode)
+ val doneMemberId = astNodeBuilder.createFieldIdentifierNode("done", forNode)
- val testId = createFieldAccessCallNode(doneBaseId, doneMemberId, forNode)
+ val testId = astNodeBuilder.createFieldAccessNode(doneBaseId, doneMemberId, astNodeBuilder.lineAndColumn(forNode))
- addAstEdge(testId, testCallId, 1)
- addArgumentEdge(testId, testCallId, 1)
+ astEdgeBuilder.addAstEdge(testId, testCallId, 1)
+ astEdgeBuilder.addArgumentEdge(testId, testCallId, 1)
- addAstEdge(testCallId, whileLoopId, 1)
- addConditionEdge(testCallId, whileLoopId)
+ astEdgeBuilder.addAstEdge(testCallId, whileLoopId, 1)
+ astEdgeBuilder.addConditionEdge(testCallId, whileLoopId)
// while loop variable assignment:
val whileLoopVariableId =
@@ -901,37 +924,37 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val baseId = createIdentifierNode(resultName, forNode)
- val memberId = createFieldIdentifierNode("value", forNode)
+ val memberId = astNodeBuilder.createFieldIdentifierNode("value", forNode)
val accessId =
- createFieldAccessCallNode(baseId, memberId, forNode)
+ astNodeBuilder.createFieldAccessNode(baseId, memberId, astNodeBuilder.lineAndColumn(forNode))
- val loopVariableAssignmentId = createCallNode(
+ val loopVariableAssignmentId = astNodeBuilder.createCallNode(
loopVariableName + " = " + resultName + ".value",
Operators.assignment,
DispatchTypes.STATIC_DISPATCH,
- forNode
+ astNodeBuilder.lineAndColumn(forNode)
)
- addAstEdge(whileLoopVariableId, loopVariableAssignmentId, 1)
- addArgumentEdge(whileLoopVariableId, loopVariableAssignmentId, 1)
- addAstEdge(accessId, loopVariableAssignmentId, 2)
- addArgumentEdge(accessId, loopVariableAssignmentId, 2)
+ astEdgeBuilder.addAstEdge(whileLoopVariableId, loopVariableAssignmentId, 1)
+ astEdgeBuilder.addArgumentEdge(whileLoopVariableId, loopVariableAssignmentId, 1)
+ astEdgeBuilder.addAstEdge(accessId, loopVariableAssignmentId, 2)
+ astEdgeBuilder.addArgumentEdge(accessId, loopVariableAssignmentId, 2)
val whileLoopBlockOrder = new OrderTracker()
- val whileLoopBlockId = createBlockNode(forNode)
+ val whileLoopBlockId = astNodeBuilder.createBlockNode(forNode)
scope.pushNewBlockScope(whileLoopBlockId)
localAstParentStack.push(whileLoopBlockId)
- addAstEdge(loopVariableAssignmentId, whileLoopBlockId, whileLoopBlockOrder)
+ astEdgeBuilder.addAstEdge(loopVariableAssignmentId, whileLoopBlockId, whileLoopBlockOrder)
// while loop block:
if (forNode.getBody.getStatementCount != 0) {
val bodyId = forNode.getBody.accept(this)
- addAstEdge(bodyId, whileLoopBlockId, whileLoopBlockOrder)
+ astEdgeBuilder.addAstEdge(bodyId, whileLoopBlockId, whileLoopBlockOrder)
}
- addAstEdge(whileLoopBlockId, whileLoopId, 2)
+ astEdgeBuilder.addAstEdge(whileLoopBlockId, whileLoopId, 2)
scope.popScope()
localAstParentStack.pop()
@@ -954,7 +977,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
private def createRealBlock(block: Block): NewBlock = {
- val blockId = createBlockNode(block)
+ val blockId = astNodeBuilder.createBlockNode(block)
val orderTracker = new OrderTracker()
scope.pushNewBlockScope(blockId)
@@ -963,7 +986,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
visitStatements(
block.getStatements,
statementId => {
- addAstEdge(statementId, blockId, orderTracker)
+ astEdgeBuilder.addAstEdge(statementId, blockId, orderTracker)
}
)
localAstParentStack.pop()
@@ -1023,69 +1046,75 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
*/
private def createArrayLiteralNode(arrayLiteralNode: ArrayLiteralNode): NewNode = {
if (arrayLiteralNode.getElementExpressions.isEmpty) {
- val arrayCallId = createCallNode(
+ val arrayCallId = astNodeBuilder.createCallNode(
EcmaBuiltins.arrayFactory + "()",
EcmaBuiltins.arrayFactory,
DispatchTypes.STATIC_DISPATCH,
- arrayLiteralNode
+ astNodeBuilder.lineAndColumn(arrayLiteralNode)
)
arrayCallId
} else {
- val blockId = createBlockNode(arrayLiteralNode)
+ val blockId = astNodeBuilder.createBlockNode(arrayLiteralNode)
scope.pushNewBlockScope(blockId)
val blockOrder = new OrderTracker()
localAstParentStack.push(blockId)
- val tmpName = PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_tmp")
- val localTmpId = createLocalNode(tmpName, Defines.Any)
+ val tmpName =
+ PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_tmp")
+ val localTmpId = astNodeBuilder.createLocalNode(tmpName, Defines.Any)
addLocalToAst(localTmpId)
val tmpArrayId = createIdentifierNode(tmpName, arrayLiteralNode)
- val arrayCallId = createCallNode(
+ val arrayCallId = astNodeBuilder.createCallNode(
EcmaBuiltins.arrayFactory + "()",
EcmaBuiltins.arrayFactory,
DispatchTypes.STATIC_DISPATCH,
- arrayLiteralNode
+ astNodeBuilder.lineAndColumn(arrayLiteralNode)
)
val assignmentTmpArrayCallId =
- createAssignmentNode(tmpArrayId, arrayCallId, arrayLiteralNode)
+ astNodeBuilder.createAssignmentNode(tmpArrayId, arrayCallId, astNodeBuilder.lineAndColumn(arrayLiteralNode))
- addAstEdge(assignmentTmpArrayCallId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(assignmentTmpArrayCallId, blockId, blockOrder)
arrayLiteralNode.getElementExpressions.forEach {
case element if element != null =>
val elementId = element.accept(this)
val pushCallId =
- createCallNode(tmpName + s".push(${codeOf(elementId)})", "", DispatchTypes.DYNAMIC_DISPATCH, element)
+ astNodeBuilder.createCallNode(
+ tmpName + s".push(${astNodeBuilder.codeOf(elementId)})",
+ "",
+ DispatchTypes.DYNAMIC_DISPATCH,
+ astNodeBuilder.lineAndColumn(element)
+ )
val nextBaseId = createIdentifierNode(tmpName, element)
- val nextMemberId = createFieldIdentifierNode("push", element)
+ val nextMemberId = astNodeBuilder.createFieldIdentifierNode("push", element)
val nextReceiverId =
- createFieldAccessCallNode(nextBaseId, nextMemberId, element)
+ astNodeBuilder.createFieldAccessNode(nextBaseId, nextMemberId, astNodeBuilder.lineAndColumn(element))
val thisPushId = createIdentifierNode(tmpName, element)
- addAstEdge(nextReceiverId, pushCallId, 0)
- addReceiverEdge(nextReceiverId, pushCallId)
+ astEdgeBuilder.addAstEdge(nextReceiverId, pushCallId, 0)
+ astEdgeBuilder.addReceiverEdge(nextReceiverId, pushCallId)
- addAstEdge(thisPushId, pushCallId, 1)
- addArgumentEdge(thisPushId, pushCallId, 0)
+ astEdgeBuilder.addAstEdge(thisPushId, pushCallId, 1)
+ astEdgeBuilder.addArgumentEdge(thisPushId, pushCallId, 0)
- addAstEdge(elementId, pushCallId, 2)
- addArgumentEdge(elementId, pushCallId, 1)
+ astEdgeBuilder.addAstEdge(elementId, pushCallId, 2)
+ astEdgeBuilder.addArgumentEdge(elementId, pushCallId, 1)
- addAstEdge(pushCallId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(pushCallId, blockId, blockOrder)
case _ => // skip
}
val tmpArrayReturnId = createIdentifierNode(tmpName, arrayLiteralNode)
- addAstEdge(tmpArrayReturnId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(tmpArrayReturnId, blockId, blockOrder)
scope.popScope()
localAstParentStack.pop()
@@ -1114,7 +1143,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
case obj =>
(obj.getString, None)
}
- createLiteralNode(code, literalNode, dynamicTypeOption)
+ astNodeBuilder.createLiteralNode(code, astNodeBuilder.lineAndColumn(literalNode), dynamicTypeOption)
}
}
@@ -1126,41 +1155,41 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
override def visit(accessNode: AccessNode): NewNode = {
val baseId = accessNode.getBase.accept(this)
- val memberId = createFieldIdentifierNode(accessNode.getProperty, accessNode)
- val accessId = createFieldAccessCallNode(baseId, memberId, accessNode)
+ val memberId = astNodeBuilder.createFieldIdentifierNode(accessNode.getProperty, accessNode)
+ val accessId = astNodeBuilder.createFieldAccessNode(baseId, memberId, astNodeBuilder.lineAndColumn(accessNode))
accessId
}
private def handleSwitchCase(caseNode: CaseNode, blockId: NewBlock, blockOrder: OrderTracker): Unit = {
- val labelId = createJumpTarget(caseNode)
- addAstEdge(labelId, blockId, blockOrder)
+ val labelId = astNodeBuilder.createJumpTarget(caseNode)
+ astEdgeBuilder.addAstEdge(labelId, blockId, blockOrder)
Option(caseNode.getTest).foreach { testExpr =>
val testId = testExpr.accept(this)
- addAstEdge(testId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(testId, blockId, blockOrder)
}
visitStatements(
caseNode.getStatements,
statementId => {
- addAstEdge(statementId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(statementId, blockId, blockOrder)
}
)
}
override def visit(switchNode: SwitchNode): NewNode = {
val switchNodeId =
- createControlStructureNode(switchNode, ControlStructureTypes.SWITCH)
+ astNodeBuilder.createControlStructureNode(switchNode, ControlStructureTypes.SWITCH)
// We need to get the to be switched upon expression from our switchExpressionStack because
// the compiler generates a synthetic let: 'let :switch = expr' and thus switchNode.getExpression
// just returns an IdentNode to :switch.
val switchExpression = switchExpressionStack.head
val switchExpressionId = switchExpression.accept(this)
- addAstEdge(switchExpressionId, switchNodeId, 1)
- addConditionEdge(switchExpressionId, switchNodeId)
+ astEdgeBuilder.addAstEdge(switchExpressionId, switchNodeId, 1)
+ astEdgeBuilder.addConditionEdge(switchExpressionId, switchNodeId)
- val blockId = createBlockNode(switchNode)
+ val blockId = astNodeBuilder.createBlockNode(switchNode)
scope.pushNewBlockScope(blockId)
localAstParentStack.push(blockId)
@@ -1169,7 +1198,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
handleSwitchCase(caseNode, blockId, blockOrder)
}
- addAstEdge(blockId, switchNodeId, 2)
+ astEdgeBuilder.addAstEdge(blockId, switchNodeId, 2)
scope.popScope()
localAstParentStack.pop()
@@ -1179,10 +1208,14 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
override def visit(parameterNode: ParameterNode): NewNode = {
val parameterNodeId = createIdentifierNode("arguments", parameterNode)
- val indexId = createLiteralNode(parameterNode.getIndex.toString, parameterNode, Some(Defines.Number))
+ val indexId = astNodeBuilder.createLiteralNode(
+ parameterNode.getIndex.toString,
+ astNodeBuilder.lineAndColumn(parameterNode),
+ Some(Defines.Number)
+ )
val accessId =
- createIndexAccessNode(parameterNodeId, indexId, parameterNode)
+ astNodeBuilder.createIndexAccessNode(parameterNodeId, indexId, astNodeBuilder.lineAndColumn(parameterNode))
accessId
}
@@ -1214,7 +1247,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val rhsId = binTestExpr.getRhs.accept(this)
- val testCallId = createEqualsCallNode(lhsId, rhsId, binTestExpr)
+ val testCallId = astNodeBuilder.createEqualsCallNode(lhsId, rhsId, astNodeBuilder.lineAndColumn(binTestExpr))
testCallId
case otherExpr => otherExpr.accept(this)
@@ -1232,13 +1265,13 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
case _ => ternaryNode.getFalseExpression.accept(this)
}
}
- createTernaryNode(testId, trueId, falseId, ternaryNode)
+ astNodeBuilder.createTernaryNode(testId, trueId, falseId, astNodeBuilder.lineAndColumn(ternaryNode))
}
private def createDependencyNodeForRequire(name: String, node: Node): Unit = {
PassHelpers
.getRequire(node)
- .foreach(id => createDependencyNode(name, id, VERSION_REQUIRE))
+ .foreach(id => astNodeBuilder.createDependencyNode(name, id, VERSION_REQUIRE))
}
private def createVarNodeParamNodeInitKindFalse(varNode: VarNode, assignmentSource: Expression): NewNode = {
@@ -1251,7 +1284,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
(Defines.Any, "")
}
- val varId = createLocalNode(varNode.getName.getName, typeFullName)
+ val varId = astNodeBuilder.createLocalNode(varNode.getName.getName, typeFullName)
addLocalToAst(varId)
val scopeType = if (varNode.isLet) {
BlockScope
@@ -1273,7 +1306,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
val assigmentCallId =
- createAssignmentNode(destId, sourceId, varNode, customCode = code)
+ astNodeBuilder.createAssignmentNode(destId, sourceId, astNodeBuilder.lineAndColumn(varNode), customCode = code)
assigmentCallId
} else {
new NewCompositeNode()
@@ -1286,7 +1319,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val destId = varNode.getAssignmentDest.accept(this)
val rhsId = createRhsForConditionalParameterInit(varNode.getInit, varNode)
val assigmentCallId =
- createAssignmentNode(destId, rhsId, varNode)
+ astNodeBuilder.createAssignmentNode(destId, rhsId, astNodeBuilder.lineAndColumn(varNode))
assigmentCallId
case ParamNodeInitKind.PLAIN =>
// We get here for all parameters of functions with at least one default parameter value
@@ -1324,11 +1357,11 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val localTmpName =
PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_tmp")
- val blockId = createBlockNode(assignment)
+ val blockId = astNodeBuilder.createBlockNode(assignment)
scope.pushNewBlockScope(blockId)
localAstParentStack.push(blockId)
- val localId = createLocalNode(localTmpName, Defines.Any)
+ val localId = astNodeBuilder.createLocalNode(localTmpName, Defines.Any)
addLocalToAst(localId)
val tmpId = createIdentifierNode(localTmpName, rhs)
@@ -1353,36 +1386,46 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
val assignmentTmpCallId =
- createAssignmentNode(tmpId, rhsId, rhs)
+ astNodeBuilder.createAssignmentNode(tmpId, rhsId, astNodeBuilder.lineAndColumn(rhs))
- addAstEdge(assignmentTmpCallId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(assignmentTmpCallId, blockId, blockOrder)
def convertDestructingElement(element: Node, index: Int): NewNode = {
element match {
case identNode: IdentNode =>
val elementId = identNode.accept(this)
val fieldAccessTmpId = createIdentifierNode(localTmpName, identNode)
- val indexId = createLiteralNode(index.toString, identNode, Some(Defines.Number))
+ val indexId = astNodeBuilder.createLiteralNode(
+ index.toString,
+ astNodeBuilder.lineAndColumn(identNode),
+ Some(Defines.Number)
+ )
val accessId =
- createIndexAccessNode(fieldAccessTmpId, indexId, identNode)
+ astNodeBuilder.createIndexAccessNode(fieldAccessTmpId, indexId, astNodeBuilder.lineAndColumn(identNode))
val assignmentCallId =
- createAssignmentNode(elementId, accessId, identNode)
+ astNodeBuilder.createAssignmentNode(elementId, accessId, astNodeBuilder.lineAndColumn(identNode))
assignmentCallId
case propertyNode: PropertyNode
if propertyNode.getValue == null && AstHelpers.getUnaryOperation(
propertyNode.getKey.tokenType().toString
) == ".spreadObject" =>
// TODO: how to handle spread objects here?
- logger.debug(s"Using a spread object for object deconstructing is not yet supported! (${code(assignment)})")
- val unknownId = createUnknownNode(propertyNode)
+ logger.debug(
+ s"Using a spread object for object deconstructing is not yet supported! (${source.getCode(assignment)})"
+ )
+ val unknownId = astNodeBuilder.createUnknownNode(propertyNode)
unknownId
case propertyNode: PropertyNode =>
val valueId = propertyNode.getValue.accept(this)
val fieldAccessTmpId = createIdentifierNode(localTmpName, propertyNode)
- val keyId = createPropertyKeyNode(propertyNode)
- val accessId = createFieldAccessCallNode(fieldAccessTmpId, keyId, propertyNode.getKey)
+ val keyId = astNodeBuilder.createPropertyKeyNode(propertyNode)
+ val accessId = astNodeBuilder.createFieldAccessNode(
+ fieldAccessTmpId,
+ keyId,
+ astNodeBuilder.lineAndColumn(propertyNode.getKey)
+ )
val assignmentCallId =
- createAssignmentNode(valueId, accessId, propertyNode)
+ astNodeBuilder.createAssignmentNode(valueId, accessId, astNodeBuilder.lineAndColumn(propertyNode))
assignmentCallId
}
@@ -1397,26 +1440,38 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val fieldAccessTmpId =
createIdentifierNode(localTmpName, binaryNode)
- val indexId = createLiteralNode(index.toString, binaryNode, Some(Defines.Number))
+ val indexId = astNodeBuilder.createLiteralNode(
+ index.toString,
+ astNodeBuilder.lineAndColumn(binaryNode),
+ Some(Defines.Number)
+ )
val accessId =
- createIndexAccessNode(fieldAccessTmpId, indexId, binaryNode)
+ astNodeBuilder.createIndexAccessNode(fieldAccessTmpId, indexId, astNodeBuilder.lineAndColumn(binaryNode))
- val voidCallId =
- createCallNode("void 0", ".void", DispatchTypes.STATIC_DISPATCH, binaryNode.getLhs)
+ val voidCallId = astNodeBuilder.createCallNode(
+ "void 0",
+ ".void",
+ DispatchTypes.STATIC_DISPATCH,
+ astNodeBuilder.lineAndColumn(binaryNode.getLhs)
+ )
val equalsCallId =
- createEqualsCallNode(accessId, voidCallId, binaryNode.getRhs)
+ astNodeBuilder.createEqualsCallNode(accessId, voidCallId, astNodeBuilder.lineAndColumn(binaryNode.getRhs))
equalsCallId
}
val falseId = {
val fieldAccessTmpId = createIdentifierNode(localTmpName, binaryNode)
- val indexId = createLiteralNode(index.toString, binaryNode, Some(Defines.Number))
+ val indexId = astNodeBuilder.createLiteralNode(
+ index.toString,
+ astNodeBuilder.lineAndColumn(binaryNode),
+ Some(Defines.Number)
+ )
val accessId =
- createIndexAccessNode(fieldAccessTmpId, indexId, binaryNode)
+ astNodeBuilder.createIndexAccessNode(fieldAccessTmpId, indexId, astNodeBuilder.lineAndColumn(binaryNode))
accessId
}
(lhsId, testId, rhsId, falseId)
@@ -1425,8 +1480,10 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
propertyNode.getKey.tokenType().toString
) == ".spreadObject" =>
// TODO: how to handle spread objects here?
- logger.debug(s"Using a spread object for object deconstructing is not yet supported! (${code(assignment)})")
- val unknownId = createUnknownNode(propertyNode)
+ logger.debug(
+ s"Using a spread object for object deconstructing is not yet supported! (${source.getCode(assignment)})"
+ )
+ val unknownId = astNodeBuilder.createUnknownNode(propertyNode)
return unknownId
case propertyNode: PropertyNode =>
val valueAsBinaryNode = propertyNode.getValue.asInstanceOf[BinaryNode]
@@ -1435,33 +1492,41 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val testId = {
val fieldAccessTmpId = createIdentifierNode(localTmpName, propertyNode)
- val keyId = createPropertyKeyNode(propertyNode)
+ val keyId = astNodeBuilder.createPropertyKeyNode(propertyNode)
val accessId =
- createFieldAccessCallNode(fieldAccessTmpId, keyId, propertyNode)
+ astNodeBuilder.createFieldAccessNode(fieldAccessTmpId, keyId, astNodeBuilder.lineAndColumn(propertyNode))
- val voidCallId =
- createCallNode("void 0", ".void", DispatchTypes.STATIC_DISPATCH, propertyNode.getKey)
+ val voidCallId = astNodeBuilder.createCallNode(
+ "void 0",
+ ".void",
+ DispatchTypes.STATIC_DISPATCH,
+ astNodeBuilder.lineAndColumn(propertyNode.getKey)
+ )
- val equalsCallId = createEqualsCallNode(accessId, voidCallId, valueAsBinaryNode.getRhs)
+ val equalsCallId = astNodeBuilder.createEqualsCallNode(
+ accessId,
+ voidCallId,
+ astNodeBuilder.lineAndColumn(valueAsBinaryNode.getRhs)
+ )
equalsCallId
}
val falseId = {
val fieldAccessTmpId = createIdentifierNode(localTmpName, propertyNode)
- val keyId = createPropertyKeyNode(propertyNode)
+ val keyId = astNodeBuilder.createPropertyKeyNode(propertyNode)
val accessId =
- createFieldAccessCallNode(fieldAccessTmpId, keyId, propertyNode)
+ astNodeBuilder.createFieldAccessNode(fieldAccessTmpId, keyId, astNodeBuilder.lineAndColumn(propertyNode))
accessId
}
(lhsId, testId, rhsId, falseId)
}
val ternaryNodeId =
- createTernaryNode(testId, trueId, falseId, element)
+ astNodeBuilder.createTernaryNode(testId, trueId, falseId, astNodeBuilder.lineAndColumn(element))
val assignmentCallId =
- createAssignmentNode(lhsId, ternaryNodeId, element)
+ astNodeBuilder.createAssignmentNode(lhsId, ternaryNodeId, astNodeBuilder.lineAndColumn(element))
assignmentCallId
}
@@ -1470,21 +1535,21 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
lhs.getElements.asScala.zipWithIndex.foreach {
case (element: PropertyNode, index: Int) if element.getValue.isInstanceOf[BinaryNode] =>
val subTreeId = convertDestructingElementWithDefault(element, index)
- addAstEdge(subTreeId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(subTreeId, blockId, blockOrder)
createDependencyNodeForRequire(element.getKeyName, assignment.getRhs)
case (element: PropertyNode, index: Int) =>
val subTreeId = convertDestructingElement(element, index)
- addAstEdge(subTreeId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(subTreeId, blockId, blockOrder)
createDependencyNodeForRequire(element.getKeyName, assignment.getRhs)
}
case lhs: ArrayLiteralNode =>
lhs.getElementExpressions.asScala.zipWithIndex.foreach {
case (element: BinaryNode, index: Int) =>
val subTreeId = convertDestructingElementWithDefault(element, index)
- addAstEdge(subTreeId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(subTreeId, blockId, blockOrder)
case (element: IdentNode, index: Int) =>
val subTreeId = convertDestructingElement(element, index)
- addAstEdge(subTreeId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(subTreeId, blockId, blockOrder)
createDependencyNodeForRequire(element.getName, assignment.getRhs)
// Skipped for array destruction assignment with ignores. The JS parser inserts null here.
case (null, _) =>
@@ -1494,21 +1559,21 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
val returnTmpId = createIdentifierNode(localTmpName, rhs)
- addAstEdge(returnTmpId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(returnTmpId, blockId, blockOrder)
scope.popScope()
localAstParentStack.pop()
blockId
}
- private def convertCommaOp(commaOp: BinaryNode): NewBlock = {
- val lhsId = commaOp.getLhs.accept(this)
- val rhsId = commaOp.getRhs.accept(this)
+ private def convertCommaOp(commaop: BinaryNode): NewBlock = {
+ val lhsId = commaop.getLhs.accept(this)
+ val rhsId = commaop.getRhs.accept(this)
// generate the exact same code value that we used to when this was handled through `convertSimpleBinaryOp`
- val code = codeOf(lhsId) + " , " + codeOf(rhsId)
- val blockId = createBlockNode(commaOp, customCode = Some(code))
+ val code = astNodeBuilder.codeOf(lhsId) + " , " + astNodeBuilder.codeOf(rhsId)
+ val blockId = astNodeBuilder.createBlockNode(commaop, customCode = Some(code))
- addAstEdge(lhsId, blockId, 0)
- addAstEdge(rhsId, blockId, 1)
+ astEdgeBuilder.addAstEdge(lhsId, blockId, 0)
+ astEdgeBuilder.addAstEdge(rhsId, blockId, 1)
blockId
}
@@ -1519,17 +1584,17 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val lhsId = binaryNode.getLhs.accept(this)
val rhsId = binaryNode.getRhs.accept(this)
- val callId = createCallNode(
- codeOf(lhsId) + " " + binaryNode.tokenType + " " + codeOf(rhsId),
+ val callId = astNodeBuilder.createCallNode(
+ astNodeBuilder.codeOf(lhsId) + " " + binaryNode.tokenType + " " + astNodeBuilder.codeOf(rhsId),
op,
DispatchTypes.STATIC_DISPATCH,
- binaryNode
+ astNodeBuilder.lineAndColumn(binaryNode)
)
- addAstEdge(lhsId, callId, 1)
- addArgumentEdge(lhsId, callId, 1)
- addAstEdge(rhsId, callId, 2)
- addArgumentEdge(rhsId, callId, 2)
+ astEdgeBuilder.addAstEdge(lhsId, callId, 1)
+ astEdgeBuilder.addArgumentEdge(lhsId, callId, 1)
+ astEdgeBuilder.addAstEdge(rhsId, callId, 2)
+ astEdgeBuilder.addArgumentEdge(rhsId, callId, 2)
callId
}
@@ -1537,7 +1602,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
private def createConstructorBlock(unaryNode: UnaryNode): NewBlock = {
val constructorCall = unaryNode.getExpression.asInstanceOf[CallNode]
- val blockId = createBlockNode(unaryNode)
+ val blockId = astNodeBuilder.createBlockNode(unaryNode)
scope.pushNewBlockScope(blockId)
val blockOrder = new OrderTracker()
@@ -1545,27 +1610,32 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val tmpAllocName =
PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_tmp")
- val localTmpAllocId = createLocalNode(tmpAllocName, Defines.Any)
+ val localTmpAllocId = astNodeBuilder.createLocalNode(tmpAllocName, Defines.Any)
addLocalToAst(localTmpAllocId)
val tmpAllocId1 = createIdentifierNode(tmpAllocName, unaryNode)
- val allocId = createCallNode(".alloc", ".alloc", DispatchTypes.STATIC_DISPATCH, unaryNode)
+ val allocId = astNodeBuilder.createCallNode(
+ ".alloc",
+ ".alloc",
+ DispatchTypes.STATIC_DISPATCH,
+ astNodeBuilder.lineAndColumn(unaryNode)
+ )
val assignmentTmpAllocCallId =
- createAssignmentNode(tmpAllocId1, allocId, unaryNode)
+ astNodeBuilder.createAssignmentNode(tmpAllocId1, allocId, astNodeBuilder.lineAndColumn(unaryNode))
- addAstEdge(assignmentTmpAllocCallId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(assignmentTmpAllocCallId, blockId, blockOrder)
val tmpAllocId2 = createIdentifierNode(tmpAllocName, unaryNode)
val receiverId = constructorCall.getFunction.accept(this)
val callId = handleCallNodeArgs(constructorCall, receiverId, tmpAllocId2, receiverId, None)
- addAstEdge(callId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(callId, blockId, blockOrder)
val tmpAllocReturnId = createIdentifierNode(tmpAllocName, unaryNode)
- addAstEdge(tmpAllocReturnId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(tmpAllocReturnId, blockId, blockOrder)
scope.popScope()
localAstParentStack.pop()
@@ -1576,23 +1646,28 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
private def createUnaryNodeForPrefixOperation(unaryNode: UnaryNode, op: String): NewCall = {
val astChildId = unaryNode.getExpression.accept(this)
- val code = unaryNode.tokenType().toString + " " + codeOf(astChildId)
+ val code = unaryNode.tokenType().toString + " " + astNodeBuilder.codeOf(astChildId)
val callId =
- createCallNode(code, op, DispatchTypes.STATIC_DISPATCH, unaryNode)
+ astNodeBuilder.createCallNode(code, op, DispatchTypes.STATIC_DISPATCH, astNodeBuilder.lineAndColumn(unaryNode))
- addAstEdge(astChildId, callId, 1)
- addArgumentEdge(astChildId, callId, 1)
+ astEdgeBuilder.addAstEdge(astChildId, callId, 1)
+ astEdgeBuilder.addArgumentEdge(astChildId, callId, 1)
callId
}
private def createUnaryNode(unaryNode: UnaryNode, op: String): NewCall = {
- val callId = createCallNode(unaryNode.toString, op, DispatchTypes.STATIC_DISPATCH, unaryNode)
+ val callId = astNodeBuilder.createCallNode(
+ unaryNode.toString,
+ op,
+ DispatchTypes.STATIC_DISPATCH,
+ astNodeBuilder.lineAndColumn(unaryNode)
+ )
val astChildId = unaryNode.getExpression.accept(this)
- addAstEdge(astChildId, callId, 1)
- addArgumentEdge(astChildId, callId, 1)
+ astEdgeBuilder.addAstEdge(astChildId, callId, 1)
+ astEdgeBuilder.addArgumentEdge(astChildId, callId, 1)
callId
}
@@ -1614,37 +1689,37 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
node.getExpressions.asScala
}
- val callId = createCallNode(
+ val callId = astNodeBuilder.createCallNode(
s"__Runtime.TO_STRING(${args.mkString(",")})",
"__Runtime.TO_STRING",
DispatchTypes.STATIC_DISPATCH,
- templateLiteralNode
+ astNodeBuilder.lineAndColumn(templateLiteralNode)
)
val callOrder = new OrderTracker()
val callArgIndex = new OrderTracker()
args.foreach { expression =>
val argId = expression.accept(this)
- addAstEdge(argId, callId, callOrder)
- addArgumentEdge(argId, callId, callArgIndex)
+ astEdgeBuilder.addAstEdge(argId, callId, callOrder)
+ astEdgeBuilder.addArgumentEdge(argId, callId, callArgIndex)
}
callId
}
override def visit(ternaryNode: TernaryNode): NewNode = {
- createTernaryNode(
+ astNodeBuilder.createTernaryNode(
ternaryNode.getTest.accept(this),
ternaryNode.getTrueExpression.accept(this),
ternaryNode.getFalseExpression.accept(this),
- ternaryNode
+ astNodeBuilder.lineAndColumn(ternaryNode)
)
}
override def visit(throwNode: ThrowNode): NewNode = {
- val unknownId = createUnknownNode(throwNode)
+ val unknownId = astNodeBuilder.createUnknownNode(throwNode)
val astChildId = throwNode.getExpression.accept(this)
- addAstEdge(astChildId, unknownId, 1)
+ astEdgeBuilder.addAstEdge(astChildId, unknownId, 1)
unknownId
}
@@ -1653,20 +1728,20 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
// calculated during JS runtime. How to emulate this?
// (see: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/with)
override def visit(withNode: WithNode): NewNode = {
- val unknownId = createUnknownNode(withNode)
+ val unknownId = astNodeBuilder.createUnknownNode(withNode)
val expressionId = withNode.getExpression.accept(this)
- addAstEdge(expressionId, unknownId, 1)
+ astEdgeBuilder.addAstEdge(expressionId, unknownId, 1)
val bodyId = withNode.getBody.accept(this)
- addAstEdge(bodyId, unknownId, 2)
+ astEdgeBuilder.addAstEdge(bodyId, unknownId, 2)
unknownId
}
// TODO: Proper handling of label nodes.
// Currently we lack appropriate handling for GOTOs.
override def visit(labelNode: LabelNode): NewNode = {
- val unknownId = createUnknownNode(labelNode)
+ val unknownId = astNodeBuilder.createUnknownNode(labelNode)
val astChildId = labelNode.getBody.accept(this)
- addAstEdge(astChildId, unknownId, 1)
+ astEdgeBuilder.addAstEdge(astChildId, unknownId, 1)
unknownId
}
@@ -1688,12 +1763,12 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
override def visit(tryNode: TryNode): NewNode = {
- val tryNodeId = createControlStructureNode(tryNode, ControlStructureTypes.TRY)
+ val tryNodeId = astNodeBuilder.createControlStructureNode(tryNode, ControlStructureTypes.TRY)
val bodyId = tryNode.getBody.accept(this)
- addAstEdge(bodyId, tryNodeId, 1)
+ astEdgeBuilder.addAstEdge(bodyId, tryNodeId, 1)
- val blockId = createBlockNode(tryNode)
+ val blockId = astNodeBuilder.createBlockNode(tryNode)
scope.pushNewBlockScope(blockId)
val blockOrder = new OrderTracker()
localAstParentStack.push(blockId)
@@ -1702,18 +1777,18 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
visitStatements(
catchBlock.getStatements,
{ statementId =>
- addAstEdge(statementId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(statementId, blockId, blockOrder)
}
)
}
- addAstEdge(blockId, tryNodeId, 2)
+ astEdgeBuilder.addAstEdge(blockId, tryNodeId, 2)
scope.popScope()
localAstParentStack.pop()
Option(tryNode.getFinallyBody).foreach { finallyBody =>
val finallyBodyId = finallyBody.accept(this)
- addAstEdge(finallyBodyId, tryNodeId, 3)
+ astEdgeBuilder.addAstEdge(finallyBodyId, tryNodeId, 3)
}
tryNodeId
@@ -1722,33 +1797,34 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
override def visit(indexNode: IndexNode): NewNode = {
val baseId = indexNode.getBase.accept(this)
val indexId = indexNode.getIndex.accept(this)
- createIndexAccessNode(baseId, indexId, indexNode)
+ astNodeBuilder.createIndexAccessNode(baseId, indexId, astNodeBuilder.lineAndColumn(indexNode))
}
override def visit(returnNode: ReturnNode): NewNode = {
- val retId = createReturnNode(returnNode)
+ val retId = astNodeBuilder.createReturnNode(returnNode)
Option(returnNode.getExpression).foreach { returnExpression =>
val retExprId = returnExpression.accept(this)
- addAstEdge(retExprId, retId, 1)
- addArgumentEdge(retExprId, retId, 1)
+ astEdgeBuilder.addAstEdge(retExprId, retId, 1)
+ astEdgeBuilder.addArgumentEdge(retExprId, retId, 1)
}
retId
}
override def visit(errorNode: ErrorNode): NewNode = {
- createUnknownNode(errorNode)
+ astNodeBuilder.createUnknownNode(errorNode)
}
override def visit(objectNode: ObjectNode): NewNode = {
- val blockId = createBlockNode(objectNode)
+ val blockId = astNodeBuilder.createBlockNode(objectNode)
scope.pushNewBlockScope(blockId)
val blockOrder = new OrderTracker()
localAstParentStack.push(blockId)
- val tmpName = PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_tmp")
- val localId = createLocalNode(tmpName, Defines.Any)
+ val tmpName =
+ PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_tmp")
+ val localId = astNodeBuilder.createLocalNode(tmpName, Defines.Any)
addLocalToAst(localId)
objectNode.getElements.forEach {
@@ -1762,7 +1838,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
) == ".spreadObject" =>
// TODO: handling of spread objects here
val exprId = element.getKey.asInstanceOf[UnaryNode].getExpression.accept(this)
- addAstEdge(exprId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(exprId, blockId, blockOrder)
case element =>
val rightHandSideId = element.getValue match {
case functionNode: FunctionNode =>
@@ -1772,15 +1848,19 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
val leftHandSideTmpId = createIdentifierNode(tmpName, element)
- val keyId = createPropertyKeyNode(element)
+ val keyId = astNodeBuilder.createPropertyKeyNode(element)
val leftHandSideFieldAccessId =
- createFieldAccessCallNode(leftHandSideTmpId, keyId, element.getKey)
+ astNodeBuilder.createFieldAccessNode(leftHandSideTmpId, keyId, astNodeBuilder.lineAndColumn(element.getKey))
val assignmentCallId =
- createAssignmentNode(leftHandSideFieldAccessId, rightHandSideId, element)
+ astNodeBuilder.createAssignmentNode(
+ leftHandSideFieldAccessId,
+ rightHandSideId,
+ astNodeBuilder.lineAndColumn(element)
+ )
- addAstEdge(assignmentCallId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(assignmentCallId, blockId, blockOrder)
// getter + setter:
Option(element.getGetter).foreach(_.accept(this))
@@ -1788,7 +1868,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
val tmpId = createIdentifierNode(tmpName, objectNode)
- addAstEdge(tmpId, blockId, blockOrder)
+ astEdgeBuilder.addAstEdge(tmpId, blockId, blockOrder)
scope.popScope()
localAstParentStack.pop()
@@ -1827,12 +1907,12 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
case None =>
val methodScopeNodeId = methodScope.scopeNode
val localId =
- createLocalNode(origin.variableName, Defines.Any, Some(closureBindingIdProperty))
- addAstEdge(localId, methodScopeNodeId, 0)
+ astNodeBuilder.createLocalNode(origin.variableName, Defines.Any, Some(closureBindingIdProperty))
+ astEdgeBuilder.addAstEdge(localId, methodScopeNodeId, 0)
val closureBindingId =
- createClosureBindingNode(closureBindingIdProperty, origin.variableName)
+ astNodeBuilder.createClosureBindingNode(closureBindingIdProperty, origin.variableName)
- methodScope.capturingRefId.foreach(addCaptureEdge(closureBindingId, _))
+ methodScope.capturingRefId.foreach(astEdgeBuilder.addCaptureEdge(closureBindingId, _))
nextReferenceId = closureBindingId
@@ -1849,7 +1929,7 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
}
localOrCapturedLocalIdOption.foreach { localOrCapturedLocalId =>
- addRefEdge(localOrCapturedLocalId, currentReferenceId)
+ astEdgeBuilder.addRefEdge(localOrCapturedLocalId, currentReferenceId)
currentReferenceId = nextReferenceId
}
@@ -1863,8 +1943,8 @@ class AstCreator(val diffGraph: DiffGraphBuilder, val source: JsSource, val used
variableName: String
): (NewNode, ScopeType) = {
val varId =
- createLocalNode(variableName, Defines.Any)
- addAstEdge(varId, methodScopeNodeId, 0)
+ astNodeBuilder.createLocalNode(variableName, Defines.Any)
+ astEdgeBuilder.addAstEdge(varId, methodScopeNodeId, 0)
(varId, MethodScope)
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala
index fda3d4129..41e179c4b 100644
--- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala
@@ -6,7 +6,7 @@ import io.shiftleft.js2cpg.datastructures.OrderTracker
import overflowdb.BatchedUpdate.DiffGraphBuilder
import org.slf4j.LoggerFactory
-trait AstEdgeBuilder { this: AstCreator =>
+class AstEdgeBuilder(private val diffGraph: DiffGraphBuilder) {
private val logger = LoggerFactory.getLogger(getClass)
diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala
index 9a79af905..6ccbdb632 100644
--- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala
@@ -1,115 +1,39 @@
package io.shiftleft.js2cpg.astcreation
-import com.oracle.js.parser.ir.*
-import io.shiftleft.codepropertygraph.generated.nodes.*
+import com.oracle.js.parser.ir._
+import io.shiftleft.codepropertygraph.generated.nodes._
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EvaluationStrategies, Operators}
-import io.shiftleft.js2cpg.datastructures.OrderTracker
-import io.shiftleft.js2cpg.datastructures.scope.MethodScope
+import io.shiftleft.js2cpg.datastructures.{LineAndColumn, OrderTracker}
+import io.shiftleft.js2cpg.datastructures.scope.{MethodScope, Scope}
import io.shiftleft.js2cpg.passes.Defines
import io.shiftleft.js2cpg.parser.JsSource
-import io.joern.x2cpg.utils.NodeBuilders
-import io.shiftleft.js2cpg.parser.JsSource.SourceMapOrigin
-import org.apache.commons.lang3.StringUtils
+import io.shiftleft.js2cpg.parser.JsSource.shortenCode
+import overflowdb.BatchedUpdate.DiffGraphBuilder
-trait AstNodeBuilder extends io.joern.x2cpg.AstNodeBuilder[Node, AstNodeBuilder] { this: AstCreator =>
+class AstNodeBuilder(
+ private val diffGraph: DiffGraphBuilder,
+ private val astEdgeBuilder: AstEdgeBuilder,
+ private val source: JsSource,
+ private val scope: Scope
+) {
- private val MinCodeLength: Int = 50
- private val DefaultMaxCodeLength: Int = 1000
+ implicit def int2IntegerOpt(x: Option[Int]): Option[Integer] = x.map(java.lang.Integer.valueOf)
+ implicit def int2Integer(x: Int): Integer = java.lang.Integer.valueOf(x)
def codeOf(node: NewNode): String = node match {
case node: AstNodeNew => node.code
case _ => ""
}
- override protected def line(node: Node): Option[Integer] =
- source.lineFromSourceMap(node).map(java.lang.Integer.valueOf)
-
- override protected def lineEnd(node: Node): Option[Integer] = None // impossible with transpilation / source maps
-
- override protected def column(node: Node): Option[Integer] =
- source.columnFromSourceMap(node).map(java.lang.Integer.valueOf)
-
- override protected def columnEnd(node: Node): Option[Integer] = None // impossible with transpilation / source maps
-
- /** @return
- * the code of a node in the parsed file. If this file is the result of transpilation the original code is
- * calculated from the corresponding sourcemap. Note: in this case, only the re-mapped starting line/column number
- * are available. Hence, we extract only a fixed number of characters (max. until the end of the file).
- */
- override protected def code(node: Node): String = codeFromSourceMap(node)
-
- private def codeFromSourceMap(node: Node): String = {
- source.getSourceMap match {
- case Some(SourceMapOrigin(_, Some(sourceMap), sourceWithLineNumbers)) =>
- val line = source.getLineOfSource(node.getStart) - 1
- val column = source.getColumnOfSource(node.getStart)
- sourceMap.getMapping(line, column) match {
- case null =>
- source.source.getString(node.getStart, node.getFinish - node.getStart)
- case mapping =>
- val originLine = mapping.getSourceLine
- val originColumn = mapping.getSourceColumn
- val transpiledCodeLength = node.getFinish - node.getStart match {
- // for some transpiled nodes the start and finish indices are wrong:
- case 0 => node.toString.length
- case other => other
- }
- sourceWithLineNumbers.get(originLine) match {
- case Some(startingCodeLine) =>
- // Code from the origin source file was found.
- val maxCodeLength = math.min(transpiledCodeLength, DefaultMaxCodeLength)
- // We are extra careful: we do not want to generate empty lines.
- // That can happen e.g., for synthetic return statements.
- // Hence, we back up 1 char.
- val startingCode =
- startingCodeLine.substring(math.min(math.max(startingCodeLine.length - 1, 0), originColumn))
- calculateCode(sourceWithLineNumbers, startingCode, originLine, maxCodeLength)
- case None =>
- // It has an actual mapping, but it is synthetic code not found in the source file.
- // We return the synthetic code.
- source.source.getString(node.getStart, node.getFinish - node.getStart)
- }
- }
- case _ =>
- // No mapping at all. We return the node code.
- source.source.getString(node.getStart, node.getFinish - node.getStart)
- }
- }
-
- /** Code field calculation:
- * - We start with the re-mapped line/column number.
- * - We always read at the length of the transpiled node (except if the original file ends earlier) capped at
- * MAX_CODE_LENGTH.
- * - If there would be more content we append ' [...]'.
- */
- @scala.annotation.tailrec
- private def calculateCode(
- sourceWithLineNumbers: Map[Int, String],
- currentLine: String,
- currentLineNumber: Int,
- transpiledCodeLength: Int
- ): String = {
- currentLine match {
- case line if line.length >= transpiledCodeLength =>
- StringUtils.abbreviate(line, math.max(MinCodeLength, transpiledCodeLength - 1))
- case line if line.length < transpiledCodeLength && sourceWithLineNumbers.contains(currentLineNumber + 1) =>
- calculateCode(
- sourceWithLineNumbers,
- line + "\n" + sourceWithLineNumbers(currentLineNumber + 1),
- currentLineNumber + 1,
- transpiledCodeLength
- )
- case line =>
- line.stripLineEnd
- }
+ def lineAndColumn(node: Node): LineAndColumn = {
+ LineAndColumn(source.getLine(node), source.getColumn(node))
}
def createDependencyNode(name: String, groupId: String, version: String): NewDependency = {
- val dependency = NodeBuilders.newDependencyNode(
- Option(name).getOrElse(""),
- Option(groupId).getOrElse(""),
- Option(version).getOrElse("")
- )
+ val dependency = NewDependency()
+ .name(Option(name).getOrElse(""))
+ .dependencyGroupId(Option(groupId).getOrElse(""))
+ .version(Option(version).getOrElse(""))
diffGraph.addNode(dependency)
dependency
}
@@ -128,69 +52,89 @@ trait AstNodeBuilder extends io.joern.x2cpg.AstNodeBuilder[Node, AstNodeBuilder]
lineAndColumnProvider: Node,
orderTracker: OrderTracker
): NewMethodParameterIn = {
- val param = parameterInNode(
- lineAndColumnProvider,
- name,
- shortenCode(code),
- orderTracker.order,
- false,
- EvaluationStrategies.BY_VALUE,
- Defines.Any
- )
+ val lineColumn = lineAndColumn(lineAndColumnProvider)
+ val line = lineColumn.line
+ val column = lineColumn.column
+ val param = NewMethodParameterIn()
+ .name(name)
+ .code(shortenCode(code))
+ .evaluationStrategy(EvaluationStrategies.BY_VALUE)
+ .lineNumber(line)
+ .columnNumber(column)
+ .order(orderTracker.order)
+ .typeFullName(Defines.Any)
+
diffGraph.addNode(param)
orderTracker.inc()
- addAstEdge(param, methodNode)
+ astEdgeBuilder.addAstEdge(param, methodNode)
scope.addVariable(name, param, MethodScope)
param
}
def createImportNode(importNode: ImportNode): NewImport = {
- val importedEntity = groupIdFromImportNode(importNode)
- val code_ = importNode.toString().stripSuffix(";")
- val node = newImportNode(code_, importedEntity, "", importNode)
+ val lineColumn = lineAndColumn(importNode)
+ val line = lineColumn.line
+ val column = lineColumn.column
+
+ val importedEntity = groupIdFromImportNode(importNode) match {
+ case "" => None
+ case x => Some(x)
+ }
+
+ val node = NewImport()
+ .importedEntity(importedEntity)
+ .code(importNode.toString().stripSuffix(";"))
+ .lineNumber(line)
+ .columnNumber(column)
+
diffGraph.addNode(node)
node
}
private def sanitizeCode(node: Node): String = node match {
case _: ReturnNode =>
- code(node).stripSuffix(";")
+ source.getCode(node).stripSuffix(";")
case _: BreakNode =>
- code(node).stripSuffix(";")
+ source.getCode(node).stripSuffix(";")
case _: ContinueNode =>
- code(node).stripSuffix(";")
+ source.getCode(node).stripSuffix(";")
case _: ErrorNode =>
// ErrorNode represents a runtime call; does not have a code representation
""
case _ =>
- code(node)
+ source.getCode(node)
}
def createUnknownNode(parserNode: Node): NewUnknown = {
- val code = sanitizeCode(parserNode)
- val unknown = unknownNode(parserNode, shortenCode(code))
+ val code = sanitizeCode(parserNode)
+ val lineColumn = lineAndColumn(parserNode)
+ val unknown = NewUnknown()
+ .parserTypeName(parserNode.getClass.getSimpleName)
+ .lineNumber(lineColumn.line)
+ .columnNumber(lineColumn.column)
+ .code(shortenCode(code))
+ .typeFullName(Defines.Any)
+
diffGraph.addNode(unknown)
unknown
}
def createTypeDeclNode(
- node: Node,
name: String,
fullName: String,
astParentType: String,
astParentFullName: String,
inheritsFrom: Option[String]
): NewTypeDecl = {
- val typeDecl = typeDeclNode(
- node,
- name,
- fullName,
- source.filePath,
- code(node),
- astParentType,
- astParentFullName,
- inheritsFrom.toList
- )
+ val typeDecl = NewTypeDecl()
+ .name(name)
+ .fullName(fullName)
+ .astParentType(astParentType)
+ .astParentFullName(astParentFullName)
+ .isExternal(false)
+ .inheritsFromTypeFullName(inheritsFrom.toList)
+ .filename(source.filePath)
+ diffGraph.addNode(typeDecl)
typeDecl
}
@@ -199,94 +143,145 @@ trait AstNodeBuilder extends io.joern.x2cpg.AstNodeBuilder[Node, AstNodeBuilder]
lineAndColumnProvider: Node,
dynamicTypeOption: Option[String]
): NewIdentifier = {
- val identifier =
- identifierNode(lineAndColumnProvider, name, shortenCode(name), Defines.Any, dynamicTypeOption.toList)
+ val lineColumn = lineAndColumn(lineAndColumnProvider)
+ val line = lineColumn.line
+ val column = lineColumn.column
+
+ val identifier = NewIdentifier()
+ .name(name)
+ .code(shortenCode(name))
+ .lineNumber(line)
+ .columnNumber(column)
+ .typeFullName(Defines.Any)
+ .dynamicTypeHintFullName(dynamicTypeOption.toList)
diffGraph.addNode(identifier)
identifier
}
def createFieldIdentifierNode(name: String, lineAndColumnProvider: Node): NewFieldIdentifier = {
- val fieldIdentifier = fieldIdentifierNode(lineAndColumnProvider, name, shortenCode(name))
+ val lineColumn = lineAndColumn(lineAndColumnProvider)
+ val line = lineColumn.line
+ val column = lineColumn.column
+
+ val fieldIdentifier = NewFieldIdentifier()
+ .code(shortenCode(name))
+ .canonicalName(name)
+ .lineNumber(line)
+ .columnNumber(column)
diffGraph.addNode(fieldIdentifier)
fieldIdentifier
}
- def createFieldAccessCallNode(baseId: NewNode, partId: NewNode, node: Node): NewCall = {
- val code = codeOf(baseId) + "." + codeOf(partId)
- val call = callNode(node, code, Operators.fieldAccess, Operators.fieldAccess, DispatchTypes.STATIC_DISPATCH)
- addAstEdge(baseId, call, 1)
- addArgumentEdge(baseId, call, 1)
- addAstEdge(partId, call, 2)
- addArgumentEdge(partId, call, 2)
+ def createFieldAccessNode(baseId: NewNode, partId: NewNode, lineAndColumn: LineAndColumn): NewCall = {
+ val call = createCallNode(
+ codeOf(baseId) + "." + codeOf(partId),
+ Operators.fieldAccess,
+ DispatchTypes.STATIC_DISPATCH,
+ lineAndColumn
+ )
+
+ astEdgeBuilder.addAstEdge(baseId, call, 1)
+ astEdgeBuilder.addArgumentEdge(baseId, call, 1)
+
+ astEdgeBuilder.addAstEdge(partId, call, 2)
+ astEdgeBuilder.addArgumentEdge(partId, call, 2)
+
call
}
- def createStaticCallNode(code: String, methodName: String, fullName: String, node: Node): NewCall = {
- val call = callNode(
- node,
- shortenCode(code),
- methodName,
- fullName,
- DispatchTypes.STATIC_DISPATCH,
- signature = Some(""),
- typeFullName = Some(Defines.Any)
- )
+ def createStaticCallNode(
+ code: String,
+ methodName: String,
+ fullName: String,
+ lineAndColumn: LineAndColumn
+ ): NewCall = {
+ val line = lineAndColumn.line
+ val column = lineAndColumn.column
+ val call = NewCall()
+ .code(shortenCode(code))
+ .name(methodName)
+ .methodFullName(fullName)
+ .dispatchType(DispatchTypes.STATIC_DISPATCH)
+ .signature("")
+ .lineNumber(line)
+ .columnNumber(column)
+ .typeFullName(Defines.Any)
+
diffGraph.addNode(call)
call
}
- def createEqualsCallNode(lhsId: NewNode, rhsId: NewNode, node: Node): NewCall = {
- val call = callNode(
- node,
+ def createEqualsCallNode(lhsId: NewNode, rhsId: NewNode, lineAndColumn: LineAndColumn): NewCall = {
+ val call = createCallNode(
codeOf(lhsId) + " === " + codeOf(rhsId),
Operators.equals,
- Operators.equals,
- DispatchTypes.STATIC_DISPATCH
+ DispatchTypes.STATIC_DISPATCH,
+ lineAndColumn
)
- addAstEdge(lhsId, call, 1)
- addArgumentEdge(lhsId, call, 1)
- addAstEdge(rhsId, call, 2)
- addArgumentEdge(rhsId, call, 2)
+
+ astEdgeBuilder.addAstEdge(lhsId, call, 1)
+ astEdgeBuilder.addArgumentEdge(lhsId, call, 1)
+
+ astEdgeBuilder.addAstEdge(rhsId, call, 2)
+ astEdgeBuilder.addArgumentEdge(rhsId, call, 2)
+
call
}
- def createIndexAccessNode(baseId: NewNode, indexId: NewNode, node: Node): NewCall = {
- val call = callNode(
- node,
+ def createIndexAccessNode(baseId: NewNode, indexId: NewNode, lineAndColumn: LineAndColumn): NewCall = {
+ val call = createCallNode(
codeOf(baseId) + "[" + codeOf(indexId) + "]",
Operators.indexAccess,
- Operators.indexAccess,
- DispatchTypes.STATIC_DISPATCH
+ DispatchTypes.STATIC_DISPATCH,
+ lineAndColumn
)
- addAstEdge(baseId, call, 1)
- addArgumentEdge(baseId, call, 1)
- addAstEdge(indexId, call, 2)
- addArgumentEdge(indexId, call, 2)
+
+ astEdgeBuilder.addAstEdge(baseId, call, 1)
+ astEdgeBuilder.addArgumentEdge(baseId, call, 1)
+
+ astEdgeBuilder.addAstEdge(indexId, call, 2)
+ astEdgeBuilder.addArgumentEdge(indexId, call, 2)
call
}
def createAssignmentNode(
destId: NewNode,
sourceId: NewNode,
- node: Node,
+ lineAndColumn: LineAndColumn,
withParenthesis: Boolean = false,
customCode: String = ""
): NewCall = {
val code = if (customCode.isEmpty) {
- if (withParenthesis) { s"(${codeOf(destId)} = ${codeOf(sourceId)})" }
- else { s"${codeOf(destId)} = ${codeOf(sourceId)}" }
- } else { customCode }
- val call = callNode(node, code, Operators.assignment, Operators.assignment, DispatchTypes.STATIC_DISPATCH)
- addAstEdge(destId, call, 1)
- addArgumentEdge(destId, call, 1)
- addAstEdge(sourceId, call, 2)
- addArgumentEdge(sourceId, call, 2)
+ if (withParenthesis) {
+ s"(${codeOf(destId)} = ${codeOf(sourceId)})"
+ } else {
+ s"${codeOf(destId)} = ${codeOf(sourceId)}"
+ }
+ } else {
+ customCode
+ }
+
+ val call =
+ createCallNode(code, Operators.assignment, DispatchTypes.STATIC_DISPATCH, lineAndColumn)
+
+ astEdgeBuilder.addAstEdge(destId, call, 1)
+ astEdgeBuilder.addArgumentEdge(destId, call, 1)
+
+ astEdgeBuilder.addAstEdge(sourceId, call, 2)
+ astEdgeBuilder.addArgumentEdge(sourceId, call, 2)
call
}
- def createLiteralNode(code: String, node: Node, dynamicTypeOption: Option[String]): NewLiteral = {
- val literal = literalNode(node, shortenCode(code), Defines.Any, dynamicTypeOption.toList)
+ def createLiteralNode(code: String, lineAndColumn: LineAndColumn, dynamicTypeOption: Option[String]): NewLiteral = {
+ val line = lineAndColumn.line
+ val column = lineAndColumn.column
+ val literal = NewLiteral()
+ .code(shortenCode(code))
+ .typeFullName(Defines.Any)
+ .lineNumber(line)
+ .columnNumber(column)
+ .dynamicTypeHintFullName(dynamicTypeOption.toList)
diffGraph.addNode(literal)
literal
}
@@ -304,30 +299,44 @@ trait AstNodeBuilder extends io.joern.x2cpg.AstNodeBuilder[Node, AstNodeBuilder]
createFieldIdentifierNode(literalNode.getValue.toString, propertyNode.getKey)
case _ =>
// TODO: handle other kinds of possible nodes (e.g., computed property name)
- createFieldIdentifierNode(code(propertyNode.getKey), propertyNode.getKey)
+ createFieldIdentifierNode(source.getCode(propertyNode.getKey), propertyNode.getKey)
}
}
- def createTernaryNode(testId: NewNode, trueId: NewNode, falseId: NewNode, node: Node): NewCall = {
+ def createTernaryNode(testId: NewNode, trueId: NewNode, falseId: NewNode, lineAndColumn: LineAndColumn): NewCall = {
val code = codeOf(testId) + " ? " + codeOf(trueId) + " : " + codeOf(falseId)
- val call = callNode(node, code, Operators.conditional, Operators.conditional, DispatchTypes.STATIC_DISPATCH)
- addAstEdge(testId, call, 1)
- addArgumentEdge(testId, call, 1)
- addAstEdge(trueId, call, 2)
- addArgumentEdge(trueId, call, 2)
- addAstEdge(falseId, call, 3)
- addArgumentEdge(falseId, call, 3)
- call
- }
+ val callId =
+ createCallNode(code, Operators.conditional, DispatchTypes.STATIC_DISPATCH, lineAndColumn)
+
+ astEdgeBuilder.addAstEdge(testId, callId, 1)
+ astEdgeBuilder.addArgumentEdge(testId, callId, 1)
+ astEdgeBuilder.addAstEdge(trueId, callId, 2)
+ astEdgeBuilder.addArgumentEdge(trueId, callId, 2)
+ astEdgeBuilder.addAstEdge(falseId, callId, 3)
+ astEdgeBuilder.addArgumentEdge(falseId, callId, 3)
+
+ callId
+ }
+
+ def createCallNode(code: String, callName: String, dispatchType: String, lineAndColumn: LineAndColumn): NewCall = {
+ val line = lineAndColumn.line
+ val column = lineAndColumn.column
+ val call = NewCall()
+ .code(shortenCode(code))
+ .name(callName)
+ .methodFullName(callName)
+ .dispatchType(dispatchType)
+ .lineNumber(line)
+ .columnNumber(column)
+ .typeFullName(Defines.Any)
- def createCallNode(code: String, callName: String, dispatchType: String, node: Node): NewCall = {
- val call = callNode(node, shortenCode(code), callName, callName, dispatchType, None, Some(Defines.Any))
diffGraph.addNode(call)
call
}
def createFileNode(fileName: String): NewFile = {
- val fileNode = NewFile().name(fileName)
+ val fileNode = NewFile()
+ .name(fileName)
diffGraph.addNode(fileNode)
fileNode
}
@@ -353,20 +362,46 @@ trait AstNodeBuilder extends io.joern.x2cpg.AstNodeBuilder[Node, AstNodeBuilder]
}
def createMethodRefNode(code: String, methodFullName: String, functionNode: FunctionNode): NewMethodRef = {
- val methodRef = methodRefNode(functionNode, shortenCode(code), methodFullName, methodFullName)
+ val lineColumn = lineAndColumn(functionNode)
+ val line = lineColumn.line
+ val column = lineColumn.column
+ val methodRef = NewMethodRef()
+ .code(shortenCode(code))
+ .methodFullName(methodFullName)
+ .typeFullName(methodFullName)
+ .lineNumber(line)
+ .columnNumber(column)
diffGraph.addNode(methodRef)
methodRef
}
def createTypeRefNode(code: String, typeFullName: String, classNode: ClassNode): NewTypeRef = {
- val typeRef = typeRefNode(classNode, shortenCode(code), typeFullName)
+ val lineColumn = lineAndColumn(classNode)
+ val line = lineColumn.line
+ val column = lineColumn.column
+ val typeRef = NewTypeRef()
+ .code(shortenCode(code))
+ .typeFullName(typeFullName)
+ .lineNumber(line)
+ .columnNumber(column)
diffGraph.addNode(typeRef)
typeRef
}
def createMethodNode(methodName: String, methodFullName: String, functionNode: FunctionNode): NewMethod = {
- val code = shortenCode(sanitizeCode(functionNode))
- val method = methodNode(functionNode, methodName, code, methodFullName, None, source.filePath)
+ val lineColumn = lineAndColumn(functionNode)
+ val line = lineColumn.line
+ val column = lineColumn.column
+ val code = shortenCode(sanitizeCode(functionNode))
+
+ val method = NewMethod()
+ .name(methodName)
+ .filename(source.filePath)
+ .code(code)
+ .fullName(methodFullName)
+ .isExternal(false)
+ .lineNumber(line)
+ .columnNumber(column)
diffGraph.addNode(method)
method
}
@@ -379,60 +414,102 @@ trait AstNodeBuilder extends io.joern.x2cpg.AstNodeBuilder[Node, AstNodeBuilder]
}
def createBlockNode(node: Node, keepWholeCode: Boolean = false, customCode: Option[String] = None): NewBlock = {
- val code = if (keepWholeCode) { customCode.getOrElse(sanitizeCode(node)) }
- else { shortenCode(customCode.getOrElse(sanitizeCode(node))) }
- val block = blockNode(node, code, Defines.Any)
+ val lineColumn = lineAndColumn(node)
+ val line = lineColumn.line
+ val column = lineColumn.column
+ val code = if (keepWholeCode) {
+ customCode.getOrElse(sanitizeCode(node))
+ } else {
+ shortenCode(customCode.getOrElse(sanitizeCode(node)))
+ }
+ val block = NewBlock()
+ .typeFullName(Defines.Any)
+ .code(code)
+ .lineNumber(line)
+ .columnNumber(column)
diffGraph.addNode(block)
block
}
- def createMethodReturnNode(node: Node): NewMethodReturn = {
- val ret = methodReturnNode(node, Defines.Any)
+ def createMethodReturnNode(lineAndColumn: LineAndColumn): NewMethodReturn = {
+ val line = lineAndColumn.line
+ val column = lineAndColumn.column
+ val code = "RET"
+
+ val ret = NewMethodReturn()
+ .code(shortenCode(code))
+ .evaluationStrategy(EvaluationStrategies.BY_VALUE)
+ .typeFullName(Defines.Any)
+ .lineNumber(line)
+ .columnNumber(column)
diffGraph.addNode(ret)
ret
}
def createTypeNode(name: String, fullName: String): NewType = {
- val typ = NewType().name(name).fullName(fullName).typeDeclFullName(fullName)
+ val typ = NewType()
+ .name(name)
+ .fullName(fullName)
+ .typeDeclFullName(fullName)
diffGraph.addNode(typ)
typ
}
def createBindingNode(): NewBinding = {
- val binding = NewBinding().name("").signature("")
+ val binding = NewBinding()
+ .name("")
+ .signature("")
diffGraph.addNode(binding)
binding
}
def createJumpTarget(caseNode: CaseNode): NewJumpTarget = {
- val name = if (caseNode.toString().startsWith("case")) "case" else "default"
- val code = shortenCode(caseNode.toString())
- val jumpTarget = jumpTargetNode(caseNode, name, code, Some(caseNode.getClass.getSimpleName))
+ val jumpTarget = NewJumpTarget()
+ .parserTypeName(caseNode.getClass.getSimpleName)
+ .name(if (caseNode.toString().startsWith("case")) "case" else "default")
+ .code(shortenCode(caseNode.toString()))
diffGraph.addNode(jumpTarget)
jumpTarget
}
def createControlStructureNode(node: Node, controlStructureType: String): NewControlStructure = {
- val controlStructure = controlStructureNode(node, controlStructureType, shortenCode(source.getString(node)))
+ val controlStructure = NewControlStructure()
+ .controlStructureType(controlStructureType)
+ .code(shortenCode(source.getString(node)))
diffGraph.addNode(controlStructure)
controlStructure
}
def createMemberNode(name: String, node: Node, dynamicTypeOption: Option[String]): NewMember = {
- val member = memberNode(node, name, shortenCode(source.getString(node)), Defines.Any, dynamicTypeOption.toList)
+ val member = NewMember()
+ .code(shortenCode(source.getString(node)))
+ .name(name)
+ .typeFullName(Defines.Any)
+ .dynamicTypeHintFullName(dynamicTypeOption.toList)
diffGraph.addNode(member)
member
}
def createLocalNode(name: String, typeFullName: String, closureBindingId: Option[String] = None): NewLocal = {
- val local = NodeBuilders.newLocalNode(shortenCode(name), typeFullName, closureBindingId)
+ val code = "N/A"
+ val local = NewLocal()
+ .code(shortenCode(code))
+ .name(name)
+ .typeFullName(typeFullName)
+ .closureBindingId(closureBindingId)
diffGraph.addNode(local)
local
}
def createReturnNode(node: Node): NewReturn = {
- val code = sanitizeCode(node)
- val ret = returnNode(node, shortenCode(code))
+ val lineColumn = lineAndColumn(node)
+ val line = lineColumn.line
+ val column = lineColumn.column
+ val code = sanitizeCode(node)
+ val ret = NewReturn()
+ .code(shortenCode(code))
+ .lineNumber(line)
+ .columnNumber(column)
diffGraph.addNode(ret)
ret
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Config.scala b/src/main/scala/io/shiftleft/js2cpg/core/Config.scala
index 1c199757f..240e0f1a5 100644
--- a/src/main/scala/io/shiftleft/js2cpg/core/Config.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/core/Config.scala
@@ -1,6 +1,5 @@
package io.shiftleft.js2cpg.core
-import io.joern.x2cpg.X2CpgConfig
import io.shiftleft.js2cpg.io.FileDefaults
import io.shiftleft.js2cpg.io.FileDefaults.VSIX_SUFFIX
@@ -10,8 +9,10 @@ import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler
import io.shiftleft.utils.IOUtils
import scala.util.{Failure, Success, Try}
+import scala.util.matching.Regex
object Config {
+
val SL_IGNORE_FILE: String = ".slignore"
val DEFAULT_TS_TYPES: Boolean = false
val DEFAULT_TS_TRANSPILING: Boolean = true
@@ -19,10 +20,14 @@ object Config {
val DEFAULT_VUE_TRANSPILING: Boolean = true
val DEFAULT_NUXT_TRANSPILING: Boolean = true
val DEFAULT_TEMPLATE_TRANSPILING: Boolean = true
+ val DEFAULT_CPG_OUT_FILE: String = "cpg.bin.zip"
+ val DEFAULT_IGNORED_FILES_REGEX: Regex = "".r
+ val DEFAULT_IGNORED_FILES: Seq[Path] = Seq.empty
val DEFAULT_IGNORE_MINIFIED: Boolean = true
val DEFAULT_IGNORE_TESTS: Boolean = true
val DEFAULT_IGNORE_PRIVATE_DEPS: Boolean = false
val DEFAULT_PRIVATE_DEPS: Seq[String] = Seq.empty
+ val DEFAULT_INCLUDE_CONFIGS: Boolean = true
val DEFAULT_INCLUDE_HTML: Boolean = true
val DEFAULT_JVM_METRICS: Option[Int] = None
val DEFAULT_MODULE_MODE: Option[String] = None
@@ -32,116 +37,59 @@ object Config {
}
-final case class Config(
+case class Config(
+ srcDir: String = "",
tsTranspiling: Boolean = Config.DEFAULT_TS_TRANSPILING,
babelTranspiling: Boolean = Config.DEFAULT_BABEL_TRANSPILING,
vueTranspiling: Boolean = Config.DEFAULT_VUE_TRANSPILING,
nuxtTranspiling: Boolean = Config.DEFAULT_NUXT_TRANSPILING,
templateTranspiling: Boolean = Config.DEFAULT_TEMPLATE_TRANSPILING,
packageJsonLocation: String = FileDefaults.PACKAGE_JSON_FILENAME,
+ outputFile: String = Config.DEFAULT_CPG_OUT_FILE,
withTsTypes: Boolean = Config.DEFAULT_TS_TYPES,
+ ignoredFilesRegex: Regex = Config.DEFAULT_IGNORED_FILES_REGEX,
+ ignoredFiles: Seq[Path] = Config.DEFAULT_IGNORED_FILES,
ignoreMinified: Boolean = Config.DEFAULT_IGNORE_MINIFIED,
ignoreTests: Boolean = Config.DEFAULT_IGNORE_TESTS,
ignorePrivateDeps: Boolean = Config.DEFAULT_IGNORE_PRIVATE_DEPS,
privateDeps: Seq[String] = Config.DEFAULT_PRIVATE_DEPS,
+ includeConfigs: Boolean = Config.DEFAULT_INCLUDE_CONFIGS,
includeHtml: Boolean = Config.DEFAULT_INCLUDE_HTML,
jvmMetrics: Option[Int] = Config.DEFAULT_JVM_METRICS,
moduleMode: Option[String] = Config.DEFAULT_MODULE_MODE,
withNodeModuleFolder: Boolean = Config.DEFAULT_WITH_NODE_MODULES_FOLDER,
optimizeDependencies: Boolean = Config.DEFAULT_OPTIMIZE_DEPENDENCIES,
fixedTranspilationDependencies: Boolean = Config.DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES
-) extends X2CpgConfig[Config] {
+) {
def createPathForPackageJson(): Path = Paths.get(packageJsonLocation) match {
case path if path.isAbsolute => path
- case _ if inputPath.endsWith(VSIX_SUFFIX) =>
- Paths.get(inputPath, "extension", packageJsonLocation).toAbsolutePath.normalize()
- case _ => Paths.get(inputPath, packageJsonLocation).toAbsolutePath.normalize()
- }
-
- def withTsTranspiling(value: Boolean): Config = {
- copy(tsTranspiling = value).withInheritedFields(this)
- }
-
- def withBabelTranspiling(value: Boolean): Config = {
- copy(babelTranspiling = value).withInheritedFields(this)
- }
-
- def withVueTranspiling(value: Boolean): Config = {
- copy(vueTranspiling = value).withInheritedFields(this)
- }
-
- def withNuxtTranspiling(value: Boolean): Config = {
- copy(nuxtTranspiling = value).withInheritedFields(this)
- }
-
- def withTemplateTranspiling(value: Boolean): Config = {
- copy(templateTranspiling = value).withInheritedFields(this)
- }
-
- def withPackageJsonLocation(value: String): Config = {
- copy(packageJsonLocation = value).withInheritedFields(this)
- }
-
- def withTsTypes(value: Boolean): Config = {
- copy(withTsTypes = value).withInheritedFields(this)
+ case _ if srcDir.endsWith(VSIX_SUFFIX) =>
+ Paths.get(srcDir, "extension", packageJsonLocation).toAbsolutePath.normalize()
+ case _ => Paths.get(srcDir, packageJsonLocation).toAbsolutePath.normalize()
}
- def withIgnoreMinified(value: Boolean): Config = {
- copy(ignoreMinified = value).withInheritedFields(this)
- }
-
- def withIgnoreTests(value: Boolean): Config = {
- copy(ignoreTests = value).withInheritedFields(this)
- }
-
- def withIgnorePrivateDeps(value: Boolean): Config = {
- copy(ignorePrivateDeps = value).withInheritedFields(this)
- }
-
- def withPrivateDeps(value: Seq[String]): Config = {
- copy(privateDeps = value).withInheritedFields(this)
- }
-
- def withIncludeHtml(value: Boolean): Config = {
- copy(includeHtml = value).withInheritedFields(this)
- }
-
- def withJvmMetrics(value: Option[Int]): Config = {
- copy(jvmMetrics = value).withInheritedFields(this)
- }
-
- def withModuleMode(value: Option[String]): Config = {
- copy(moduleMode = value).withInheritedFields(this)
- }
-
- def withNodeModuleFolder(value: Boolean): Config = {
- copy(withNodeModuleFolder = value).withInheritedFields(this)
- }
-
- def withOptimizeDependencies(value: Boolean): Config = {
- copy(optimizeDependencies = value).withInheritedFields(this)
- }
-
- def withFixedTranspilationDependencies(value: Boolean): Config = {
- copy(fixedTranspilationDependencies = value).withInheritedFields(this)
- }
-
- override def withInputPath(inputPath: String): Config = {
- super.withInputPath(inputPath).withLoadedIgnores().withInheritedFields(this)
+ def createPathForIgnore(ignore: String): Path = {
+ val path = Paths.get(ignore)
+ if (path.isAbsolute) {
+ path
+ } else {
+ Paths.get(srcDir, ignore).toAbsolutePath.normalize()
+ }
}
- private def withLoadedIgnores(): Config = {
- val slIngoreFilePath = Paths.get(inputPath, Config.SL_IGNORE_FILE)
+ def withLoadedIgnores(): Config = {
+ val slIngoreFilePath = Paths.get(srcDir, Config.SL_IGNORE_FILE)
Try(IOUtils.readLinesInFile(slIngoreFilePath)) match {
- case Failure(_) => this
- case Success(lines) => this.withIgnoredFiles(this.ignoredFiles ++ lines).withInheritedFields(this)
+ case Failure(_) => this
+ case Success(lines) =>
+ this.copy(ignoredFiles = ignoredFiles ++ lines.map(createPathForIgnore))
}
}
override def toString: String =
s"""
- |\t- Source project: '$inputPath'
+ |\t- Source project: '$srcDir'
|\t- package.json location: '${createPathForPackageJson()}'
|\t- Module mode: '${moduleMode.getOrElse(TypescriptTranspiler.DefaultModule)}'
|\t- Optimize dependencies: $optimizeDependencies
@@ -151,14 +99,10 @@ final case class Config(
|\t- Vue.js transpiling: $vueTranspiling
|\t- Nuxt.js transpiling: $nuxtTranspiling
|\t- Template transpiling: $templateTranspiling
- |\t- Ignored files: ${ignoredFiles
- .filter(f => new File(f).isFile)
- .map(f => s"${System.lineSeparator()}\t\t'$f'")
- .mkString}
|\t- Ignored files regex: '$ignoredFilesRegex'
|\t- Ignored folders: ${ignoredFiles
- .filter(f => new File(f).isDirectory)
- .map(f => s"${System.lineSeparator()}\t\t'$f'")
+ .filter(f => new File(f.toString).isDirectory)
+ .map(f => s"${System.lineSeparator()}\t\t'${f.toString}'")
.mkString}
|\t- Ignore minified files: $ignoreMinified
|\t- Ignore test files: $ignoreTests
@@ -166,7 +110,8 @@ final case class Config(
|\t- Additional private dependencies: ${privateDeps
.map(f => s"${System.lineSeparator()}\t\t'$f'")
.mkString}
+ |\t- Include configuration files: $includeConfigs
|\t- Include HTML files: $includeHtml
- |\t- Output file: '$outputPath'
+ |\t- Output file: '$outputFile'
|""".stripMargin
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Js2Cpg.scala b/src/main/scala/io/shiftleft/js2cpg/core/Js2Cpg.scala
index a5e1b71b8..d867d2b3c 100644
--- a/src/main/scala/io/shiftleft/js2cpg/core/Js2Cpg.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/core/Js2Cpg.scala
@@ -3,23 +3,20 @@ package io.shiftleft.js2cpg.core
import java.nio.file.{Path, StandardCopyOption}
import better.files.File
import better.files.File.LinkOptions
-import io.shiftleft.js2cpg.passes.*
-import io.shiftleft.js2cpg.io.FileDefaults.*
+import io.shiftleft.js2cpg.passes._
+import io.shiftleft.js2cpg.io.FileDefaults._
import io.shiftleft.js2cpg.io.FileUtils
import io.shiftleft.js2cpg.parser.PackageJsonParser
import io.shiftleft.js2cpg.preprocessing.NuxtTranspiler
import io.shiftleft.js2cpg.preprocessing.TranspilationRunner
import io.shiftleft.js2cpg.utils.MemoryMetrics
-import io.joern.x2cpg.X2Cpg.withNewEmptyCpg
+import io.joern.x2cpg.X2Cpg.newEmptyCpg
import io.joern.x2cpg.utils.HashUtil
-import io.joern.x2cpg.utils.Report
-import io.joern.x2cpg.X2CpgFrontend
-import io.shiftleft.codepropertygraph.Cpg
import org.slf4j.LoggerFactory
import scala.util.{Failure, Success, Try}
-class Js2Cpg extends X2CpgFrontend[Config] {
+class Js2Cpg {
private val logger = LoggerFactory.getLogger(getClass)
@@ -27,7 +24,7 @@ class Js2Cpg extends X2CpgFrontend[Config] {
private def checkCpgGenInputFiles(jsFiles: List[(Path, Path)], config: Config): Unit = {
if (jsFiles.isEmpty) {
- val project = File(config.inputPath)
+ val project = File(config.srcDir)
logger.warn(s"'$project' contains no *.js files. No CPG was generated.")
if (config.babelTranspiling) {
logger.warn("\t- Babel transpilation did not yield any *.js files")
@@ -104,7 +101,7 @@ class Js2Cpg extends X2CpgFrontend[Config] {
} ++ transpiledJsFiles
}
- private def prepareAndGenerateCpg(project: File, tmpProjectDir: File, config: Config): Try[Cpg] = {
+ private def prepareAndGenerateCpg(project: File, tmpProjectDir: File, config: Config): Unit = {
val newTmpProjectDir = if (project.extension.contains(VSIX_SUFFIX)) {
handleVsixProject(project, tmpProjectDir)
} else {
@@ -117,9 +114,7 @@ class Js2Cpg extends X2CpgFrontend[Config] {
.getFileTree(newTmpProjectDir.path, config, List(JS_SUFFIX, MJS_SUFFIX))
.map(f => (f, newTmpProjectDir.path))
- val result = for {
- tmpTranspileDir <- File.temporaryDirectory("js2cpgTranspileOut")
- } yield {
+ File.usingTemporaryDirectory("js2cpgTranspileOut") { tmpTranspileDir =>
findProjects(newTmpProjectDir, config)
.foreach { p =>
val subDir =
@@ -149,55 +144,57 @@ class Js2Cpg extends X2CpgFrontend[Config] {
// as we do not have any control of the transpilers themselves
// (also they very much depend on the speed of npm).
MemoryMetrics.withMemoryMetrics(config) {
- generateCPG(config.withInputPath(newTmpProjectDir.toString), jsFiles)
+ generateCPG(config.copy(srcDir = newTmpProjectDir.toString), jsFiles)
}
}
- result.get()
}
- def createCpg(config: Config): Try[Cpg] = {
- val project = File(config.inputPath)
+ def run(config: Config): Unit = {
+ val project = File(config.srcDir)
// We need to get the absolut project path here otherwise user configured
// excludes based on either absolut or relative paths can not be matched.
// We intentionally use .canonicalFile as this is using java.io.File#getCanonicalPath
// under the hood that will resolve . or ../ and symbolic links in any combination.
val absoluteProjectPath = project.canonicalFile.pathAsString
- val configWithAbsolutProjectPath = config.withInputPath(absoluteProjectPath)
+ val configWithAbsolutProjectPath = config.copy(srcDir = absoluteProjectPath)
logger.info(s"Generating CPG from Javascript sources in: '$absoluteProjectPath'")
logger.debug(s"Configuration:$configWithAbsolutProjectPath")
- val result = for {
- tmpProjectDir <- File.temporaryDirectory(project.name)
- } yield prepareAndGenerateCpg(project, tmpProjectDir, configWithAbsolutProjectPath)
+ File.usingTemporaryDirectory(project.name) { tmpProjectDir =>
+ prepareAndGenerateCpg(project, tmpProjectDir, configWithAbsolutProjectPath)
+ }
logger.info("Generation of CPG is complete.")
report.print()
-
- result.get()
}
private def configFiles(config: Config, extensions: List[String]): List[(Path, Path)] =
FileUtils
- .getFileTree(File(config.inputPath).path, config, extensions, filterIgnoredFiles = false)
- .map(f => (f, File(config.inputPath).path))
-
- private def generateCPG(config: Config, jsFilesWithRoot: List[(Path, Path)]): Try[Cpg] = {
- withNewEmptyCpg(config.outputPath, config) { (cpg, config) =>
- val hash = HashUtil.sha256(jsFilesWithRoot.map(_._1))
-
- new AstCreationPass(cpg, jsFilesWithRoot, config, report).createAndApply()
- new JsMetaDataPass(cpg, hash, config.inputPath).createAndApply()
- new BuiltinTypesPass(cpg).createAndApply()
- new DependenciesPass(cpg, config).createAndApply()
- new ConfigPass(configFiles(config, List(VUE_SUFFIX)), cpg, report).createAndApply()
- new PrivateKeyFilePass(configFiles(config, List(KEY_SUFFIX)), cpg, report).createAndApply()
- if (config.includeHtml) {
- new ConfigPass(configFiles(config, List(HTML_SUFFIX)), cpg, report).createAndApply()
- }
+ .getFileTree(File(config.srcDir).path, config, extensions, filterIgnoredFiles = false)
+ .map(f => (f, File(config.srcDir).path))
+
+ private def generateCPG(config: Config, jsFilesWithRoot: List[(Path, Path)]): Unit = {
+ val cpg = newEmptyCpg(Some(config.outputFile))
+ val hash = HashUtil.sha256(jsFilesWithRoot.map(_._1))
+
+ new AstCreationPass(File(config.srcDir), jsFilesWithRoot, cpg, report).createAndApply()
+ new JsMetaDataPass(cpg, hash, config.srcDir).createAndApply()
+ new BuiltinTypesPass(cpg).createAndApply()
+ new DependenciesPass(cpg, config).createAndApply()
+ new ConfigPass(configFiles(config, List(VUE_SUFFIX)), cpg, report).createAndApply()
+ new PrivateKeyFilePass(configFiles(config, List(KEY_SUFFIX)), cpg, report).createAndApply()
+
+ if (config.includeHtml) {
+ new ConfigPass(configFiles(config, List(HTML_SUFFIX)), cpg, report).createAndApply()
+ }
+
+ if (config.includeConfigs) {
new ConfigPass(configFiles(config, CONFIG_FILES), cpg, report).createAndApply()
}
+
+ cpg.close()
}
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Js2CpgMain.scala b/src/main/scala/io/shiftleft/js2cpg/core/Js2CpgMain.scala
index d95105169..24f957d14 100644
--- a/src/main/scala/io/shiftleft/js2cpg/core/Js2CpgMain.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/core/Js2CpgMain.scala
@@ -1,18 +1,16 @@
package io.shiftleft.js2cpg.core
-import Js2cpgArgumentsParser.*
-import io.joern.x2cpg.X2CpgMain
-import io.joern.x2cpg.utils.Environment
+object Js2CpgMain {
+ def main(args: Array[String]): Unit = {
+ val argumentsParser: Js2cpgArgumentsParser = new Js2cpgArgumentsParser()
-import java.nio.file.Paths
-
-object Js2CpgMain extends X2CpgMain(parser, new Js2Cpg()) {
- def run(config: Config, js2cpg: Js2Cpg): Unit = {
- val absPath = Paths.get(config.inputPath).toAbsolutePath.toString
- if (Environment.pathExists(absPath)) {
- js2cpg.run(config.withInputPath(absPath))
- } else {
- System.exit(1)
+ argumentsParser.parse(args) match {
+ case Some(config) =>
+ new Js2Cpg().run(config)
+ case None =>
+ argumentsParser.showUsage()
+ System.exit(1)
}
+
}
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala b/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala
index 297679f3b..a685bec66 100644
--- a/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala
@@ -1,14 +1,21 @@
package io.shiftleft.js2cpg.core
import io.shiftleft.js2cpg.io.FileDefaults
+import io.shiftleft.js2cpg.io.FileDefaults.VSIX_SUFFIX
+
import java.io.File
+import io.shiftleft.js2cpg.parser.PackageJsonParser
import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler
-import scopt.OParser
+import scopt.OptionParser
object Js2cpgArgumentsParser {
- implicit val defaultConfig: Config = Config()
+ val HELP: String = "help"
val VERSION: String = "version"
+ val SRCDIR: String = ""
+ val OUTPUT: String = "output"
val WITH_TS_TYPES: String = "with-typescript-types"
+ val EXCLUDE: String = "exclude"
+ val EXCLUDE_REGEX: String = "exclude-regex"
val PACKAGE_JSON: String = "package-json"
val NO_TS: String = "no-ts"
val TS: String = "ts"
@@ -35,6 +42,11 @@ object Js2cpgArgumentsParser {
val OPTIMIZE_DEPENDENCIES: String = "optimize-dependencies"
val ALL_DEPENDENCIES: String = "all-dependencies"
val FIXED_TRANSPILATION_DEPENDENCIES: String = "fixed-transpilation-dependencies"
+}
+
+class Js2cpgArgumentsParser {
+
+ import Js2cpgArgumentsParser._
private lazy val banner: String =
"""
@@ -46,130 +58,156 @@ object Js2cpgArgumentsParser {
| ╚════╝ ╚══════╝╚══════╝ ╚═════╝╚═╝ ╚═════╝
""".stripMargin
- val parser: OParser[Unit, Config] = {
- val builder = OParser.builder[Config]
- import builder.*
- OParser.sequence(
- programName("js2cpg"),
- head(s"""
+ private val parser: OptionParser[Config] = new OptionParser[Config]("js2cpg.sh") {
+ help(HELP).text("prints this usage text")
+ head(s"""
|$banner
|js2cpg version "${io.shiftleft.js2cpg.core.BuildInfo.version}"
- |""".stripMargin),
- version(VERSION)
- .text("print js2cpg version and exit"),
- opt[String](PACKAGE_JSON)
- .text(
- s"path to the projects package.json (path relative to or absolute path; defaults to '${java.io.File.separator}${FileDefaults.PACKAGE_JSON_FILENAME}')"
- )
- .action((x, c) => c.withPackageJsonLocation(x))
- .validate(path => {
- val f = new File(path)
- if (f.exists() && !f.isDirectory) success
- else failure(s"File '$path' does not exist or is a directory")
- }),
- opt[Unit](NO_TS)
- .text("disables transpiling Typescript files to Javascript")
- .action((_, c) => c.withTsTranspiling(false)),
- opt[Unit](NO_BABEL)
- .text("disables transpiling Javascript files with Babel")
- .action((_, c) => c.withBabelTranspiling(false)),
- opt[Unit](NO_VUE)
- .text("disables transpiling Vue.js files")
- .action((_, c) => c.withVueTranspiling(false)),
- opt[Unit](NO_NUXT)
- .text("disables Nuxt.js transpiling")
- .action((_, c) => c.withNuxtTranspiling(false)),
- opt[Unit](NO_TEMPLATES)
- .text("disables transpiling EJS or Pug template files")
- .action((_, c) => c.withTemplateTranspiling(false)),
- // for backwards compatibility - has no effect:
- opt[Unit](TRANSPILING)
- .text("enables transpiling Typescript files to Javascript")
- .hidden(), // deprecated
- // for backwards compatibility - has no effect:
- opt[Unit](BABEL)
- .text("enables transpiling Javascript files with Babel")
- .hidden(),
- // for backwards compatibility - has no effect:
- opt[Unit](TS)
- .text("enables transpiling Typescript files to Javascript")
- .hidden(),
- opt[Unit](WITH_NODE_MODULES_FOLDER)
- .text(s"include the node_module folder (defaults to '${Config.DEFAULT_WITH_NODE_MODULES_FOLDER}')")
- .action((_, c) => c.withNodeModuleFolder(true))
- .hidden(),
- opt[Unit](WITH_TS_TYPES)
- .text(s"query types via Typescript; needs a `package.json` (defaults to '${Config.DEFAULT_TS_TYPES}')")
- .action((_, c) => c.withTsTypes(true))
- .hidden(), // deprecated
- // for backwards compatibility - has no effect:
- opt[Unit](IGNORE_MINIFIED)
- .text("ignore minified Javascript files (filename ending with '-min.js', '.min.js', or 'bundle.js')")
- .hidden(), // deprecated
- opt[Unit](WITH_MINIFIED)
- .action((_, c) => c.withIgnoreMinified(false))
- .text("include minified Javascript files (filename ending with '-min.js', '.min.js', or 'bundle.js')")
- .hidden(), // deprecated
- opt[Unit](INCLUDE_MINIFIED)
- .action((_, c) => c.withIgnoreMinified(false))
- .text("include minified Javascript files (filename ending with '-min.js', '.min.js', or 'bundle.js')"),
- opt[Unit](WITH_TESTS)
- .action((_, c) => c.withIgnoreTests(false))
- .text("include test files")
- .hidden(), // deprecated
- opt[Unit](INCLUDE_TESTS)
- .action((_, c) => c.withIgnoreTests(false))
- .text("include test files"),
- opt[Unit](IGNORE_PRIVATE_DEPS)
- .text(
- s"ignores private modules/dependencies in 'node_modules/' (defaults to '${Config.DEFAULT_IGNORE_PRIVATE_DEPS}')"
- )
- .action((_, c) => c.withIgnorePrivateDeps(true))
- .hidden(),
- opt[Unit](EXCLUDE_PRIVATE_DEPS)
- .text(
- s"excludes private modules/dependencies in 'node_modules/' (defaults to '${Config.DEFAULT_IGNORE_PRIVATE_DEPS}')"
- )
- .action((_, c) => c.withIgnorePrivateDeps(true)),
- opt[Seq[String]](PRIVATE_DEPS)
- .valueName(",,...")
- .action((x, c) => c.withPrivateDeps(c.privateDeps ++ x.flatMap(d => Seq(d, s"@$d"))))
- .text(s"additional private dependencies to be analyzed from '${FileDefaults.NODE_MODULES_DIR_NAME}/'"),
- opt[Unit](INCLUDE_CONFIGS)
- .text("include configuration files (*.conf.js, *.config.js, *.json)")
- .hidden(), // deprecated, it is the default
- opt[Unit](INCLUDE_HTML)
- .text("include HTML files (*.html)")
- .hidden(), // deprecated, it is the default
- opt[Unit](EXCLUDE_HTML)
- .text("excludes HTML files (*.html)")
- .action((_, c) => c.withIncludeHtml(false)),
- opt[Unit](OPTIMIZE_DEPENDENCIES)
- .text(
- s"optimize project dependencies during transpilation (defaults to '${Config.DEFAULT_OPTIMIZE_DEPENDENCIES}')"
- )
- .hidden(), // deprecated, it is the default
- opt[Unit](ALL_DEPENDENCIES)
- .text(
- s"install all project dependencies during transpilation (defaults to '${!Config.DEFAULT_OPTIMIZE_DEPENDENCIES}')"
- )
- .action((_, c) => c.withOptimizeDependencies(false)),
- opt[Unit](FIXED_TRANSPILATION_DEPENDENCIES)
- .text(
- s"install fixed versions of transpilation dependencies during transpilation (defaults to '${!Config.DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES}')"
- )
- .action((_, c) => c.withFixedTranspilationDependencies(true)),
- opt[Int](JVM_MONITOR)
- .text("enable JVM metrics logging (requires JMX port number)")
- .action((jmxPortNumber, c) => c.withJvmMetrics(Some(jmxPortNumber)))
- .hidden(),
- opt[String](MODULE_MODE)
- .text(
- s"set the module mode for transpiling (default is '${TypescriptTranspiler.DefaultModule}', alternatives are e.g., esnext or es2015)"
- )
- .action((module, c) => c.withModuleMode(Some(module)))
- .hidden()
- )
+ |""".stripMargin)
+ version(VERSION)
+ .text("print js2cpg version and exit")
+ arg[String](SRCDIR)
+ .required()
+ .text("directory containing Javascript code or the path to a *.vsix file")
+ .action((x, c) => c.copy(srcDir = x).withLoadedIgnores())
+ .validate(path => {
+ val f = new File(path)
+ if (f.exists() && (f.isDirectory || f.toString.endsWith(VSIX_SUFFIX))) success
+ else failure(s"Invalid $SRCDIR path: '$path'")
+ })
+ opt[String](PACKAGE_JSON)
+ .text(
+ s"path to the projects package.json (path relative to $SRCDIR or absolute path; defaults to '$SRCDIR${java.io.File.separator}${FileDefaults.PACKAGE_JSON_FILENAME}')"
+ )
+ .action((x, c) => c.copy(packageJsonLocation = x))
+ .validate(path => {
+ val f = new File(path)
+ if (f.exists() && !f.isDirectory) success
+ else failure(s"File '$path' does not exist or is a directory")
+ })
+ opt[String](OUTPUT)
+ .text(s"CPG output file name (defaults to '${Config.DEFAULT_CPG_OUT_FILE}')")
+ .action((x, c) => c.copy(outputFile = x))
+ .validate(x =>
+ if (x.isEmpty) {
+ failure("Output file cannot be empty")
+ } else if (!new File(x).getAbsoluteFile.getParentFile.exists()) {
+ failure("Directory of the output file does not exist")
+ } else success
+ )
+ opt[Unit](NO_TS)
+ .text("disables transpiling Typescript files to Javascript")
+ .action((_, c) => c.copy(tsTranspiling = false))
+ opt[Unit](NO_BABEL)
+ .text("disables transpiling Javascript files with Babel")
+ .action((_, c) => c.copy(babelTranspiling = false))
+ opt[Unit](NO_VUE)
+ .text("disables transpiling Vue.js files")
+ .action((_, c) => c.copy(vueTranspiling = false))
+ opt[Unit](NO_NUXT)
+ .text("disables Nuxt.js transpiling")
+ .action((_, c) => c.copy(nuxtTranspiling = false))
+ opt[Unit](NO_TEMPLATES)
+ .text("disables transpiling EJS or Pug template files")
+ .action((_, c) => c.copy(templateTranspiling = false))
+ // for backwards compatibility - has no effect:
+ opt[Unit](TRANSPILING)
+ .text("enables transpiling Typescript files to Javascript")
+ .hidden() // deprecated
+ // for backwards compatibility - has no effect:
+ opt[Unit](BABEL)
+ .text("enables transpiling Javascript files with Babel")
+ .hidden()
+ // for backwards compatibility - has no effect:
+ opt[Unit](TS)
+ .text("enables transpiling Typescript files to Javascript")
+ .hidden()
+ opt[Unit](WITH_NODE_MODULES_FOLDER)
+ .text(s"include the node_module folder (defaults to '${Config.DEFAULT_WITH_NODE_MODULES_FOLDER}')")
+ .action((_, c) => c.copy(withNodeModuleFolder = true))
+ .hidden()
+ opt[Unit](WITH_TS_TYPES)
+ .text(s"query types via Typescript; needs a `package.json` (defaults to '${Config.DEFAULT_TS_TYPES}')")
+ .action((_, c) => c.copy(withTsTypes = true))
+ .hidden() // deprecated
+ opt[Seq[String]](EXCLUDE)
+ .valueName(",,...")
+ .action((x, c) => c.copy(ignoredFiles = c.ignoredFiles ++ x.map(c.createPathForIgnore)))
+ .text("files to exclude during CPG generation (paths relative to or absolute paths)")
+ opt[String](EXCLUDE_REGEX)
+ .action((x, c) => c.copy(ignoredFilesRegex = x.r))
+ .text("a regex specifying files to exclude during CPG generation (the absolute file path is matched)")
+ // for backwards compatibility - has no effect:
+ opt[Unit](IGNORE_MINIFIED)
+ .text("ignore minified Javascript files (filename ending with '-min.js', '.min.js', or 'bundle.js')")
+ .hidden() // deprecated
+ opt[Unit](WITH_MINIFIED)
+ .action((_, c) => c.copy(ignoreMinified = false))
+ .hidden() // deprecated
+ .text("include minified Javascript files (filename ending with '-min.js', '.min.js', or 'bundle.js')")
+ opt[Unit](INCLUDE_MINIFIED)
+ .action((_, c) => c.copy(ignoreMinified = false))
+ .text("include minified Javascript files (filename ending with '-min.js', '.min.js', or 'bundle.js')")
+ opt[Unit](WITH_TESTS)
+ .action((_, c) => c.copy(ignoreTests = false))
+ .hidden() // deprecated
+ .text("include test files")
+ opt[Unit](INCLUDE_TESTS)
+ .action((_, c) => c.copy(ignoreTests = false))
+ .text("include test files")
+ opt[Unit](IGNORE_PRIVATE_DEPS)
+ .text(
+ s"ignores private modules/dependencies in 'node_modules/' (defaults to '${Config.DEFAULT_IGNORE_PRIVATE_DEPS}')"
+ )
+ .action((_, c) => c.copy(ignorePrivateDeps = true))
+ .hidden()
+ opt[Unit](EXCLUDE_PRIVATE_DEPS)
+ .text(
+ s"excludes private modules/dependencies in 'node_modules/' (defaults to '${Config.DEFAULT_IGNORE_PRIVATE_DEPS}')"
+ )
+ .action((_, c) => c.copy(ignorePrivateDeps = true))
+ opt[Seq[String]](PRIVATE_DEPS)
+ .valueName(",,...")
+ .action((x, c) => c.copy(privateDeps = c.privateDeps ++ x.flatMap(d => Seq(d, s"@$d"))))
+ .text(s"additional private dependencies to be analyzed from '${FileDefaults.NODE_MODULES_DIR_NAME}/'")
+ opt[Unit](INCLUDE_CONFIGS)
+ .text("include configuration files (*.conf.js, *.config.js, *.json)")
+ .hidden() // deprecated, it is the default
+ opt[Unit](INCLUDE_HTML)
+ .text("include HTML files (*.html)")
+ .hidden() // deprecated, it is the default
+ opt[Unit](EXCLUDE_HTML)
+ .text("excludes HTML files (*.html)")
+ .action((_, c) => c.copy(includeHtml = false))
+ opt[Unit](OPTIMIZE_DEPENDENCIES)
+ .text(
+ s"optimize project dependencies during transpilation (defaults to '${Config.DEFAULT_OPTIMIZE_DEPENDENCIES}')"
+ )
+ .hidden() // deprecated, it is the default
+ opt[Unit](ALL_DEPENDENCIES)
+ .text(
+ s"install all project dependencies during transpilation (defaults to '${!Config.DEFAULT_OPTIMIZE_DEPENDENCIES}')"
+ )
+ .action((_, c) => c.copy(optimizeDependencies = false))
+ opt[Unit](FIXED_TRANSPILATION_DEPENDENCIES)
+ .text(
+ s"install fixed versions of transpilation dependencies during transpilation (defaults to '${!Config.DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES}')"
+ )
+ .action((_, c) => c.copy(fixedTranspilationDependencies = true))
+ opt[Int](JVM_MONITOR)
+ .text("enable JVM metrics logging (requires JMX port number)")
+ .action((jmxPortNumber, c) => c.copy(jvmMetrics = Some(jmxPortNumber)))
+ .hidden()
+ opt[String](MODULE_MODE)
+ .text(
+ s"set the module mode for transpiling (default is '${TypescriptTranspiler.DefaultModule}', alternatives are e.g., esnext or es2015)"
+ )
+ .action((module, c) => c.copy(moduleMode = Some(module)))
+ .hidden()
}
+ def parse(args: Array[String]): Option[Config] = parser.parse(args, Config())
+
+ def showUsage(): Unit = println(parser.usage)
+
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Report.scala b/src/main/scala/io/shiftleft/js2cpg/core/Report.scala
new file mode 100644
index 000000000..ab967ec20
--- /dev/null
+++ b/src/main/scala/io/shiftleft/js2cpg/core/Report.scala
@@ -0,0 +1,105 @@
+package io.shiftleft.js2cpg.core
+
+import io.shiftleft.js2cpg.utils.TimeUtils
+import org.slf4j.LoggerFactory
+
+import scala.collection.concurrent.TrieMap
+import scala.concurrent.duration.Duration
+
+object Report {
+
+ private val logger = LoggerFactory.getLogger(Report.getClass)
+
+ private type FileName = String
+
+ private type Reports = TrieMap[FileName, ReportEntry]
+
+ private case class ReportEntry(
+ loc: Long,
+ parsed: Boolean,
+ cpgGen: Boolean,
+ duration: Long,
+ isConfig: Boolean = false
+ ) {
+ def toSeq: Seq[String] = {
+ val lines = loc.toString
+ val dur = if (duration == 0) "-" else TimeUtils.pretty(Duration.fromNanos(duration))
+ val es6 = if (parsed) "yes" else "no"
+ val cpg = if (cpgGen) "yes" else "no"
+ Seq(lines, es6, cpg, dur)
+ }
+ }
+
+}
+
+class Report {
+
+ import Report._
+
+ private val reports: Reports = TrieMap.empty
+
+ def print(): Unit = {
+
+ def formatTable(table: Seq[Seq[String]]): String = {
+ if (table.isEmpty) ""
+ else {
+ // Get column widths based on the maximum cell width in each column (+2 for a one character padding on each side)
+ val colWidths =
+ table.transpose.map(_.map(cell => if (cell == null) 0 else cell.length).max + 2)
+ // Format each row
+ val rows = table.map(
+ _.zip(colWidths)
+ .map { case (item, size) => (" %-" + (size - 1) + "s").format(item) }
+ .mkString("|", "|", "|")
+ )
+ // Formatted separator row, used to separate the header and draw table borders
+ val separator = colWidths.map("-" * _).mkString("+", "+", "+")
+ // Put the table together and return
+ val header = rows.head
+ val content = rows.tail.take(rows.tail.size - 1)
+ val footer = rows.tail.last
+ (separator +: header +: separator +: content :+ separator :+ footer :+ separator)
+ .mkString("\n")
+ }
+ }
+
+ val rows = reports.toSeq
+ .sortBy(_._1)
+ .zipWithIndex
+ .view
+ .map {
+ case ((file, sum), index) if sum.isConfig =>
+ s"${index + 1}" +: s"$file (config file)" +: sum.toSeq
+ case ((file, sum), index) => s"${index + 1}" +: file +: sum.toSeq
+ }
+ .toSeq
+ val numOfReports = reports.size
+ val header = Seq(Seq("#", "File", "LOC", "Parsed", "Got a CPG", "CPG Gen. Duration"))
+ val footer = Seq(
+ Seq(
+ "Total",
+ "",
+ s"${reports.map(_._2.loc).sum}",
+ s"${reports.count(_._2.parsed)}/$numOfReports",
+ s"${reports.count(_._2.cpgGen)}/$numOfReports",
+ ""
+ )
+ )
+ val table = header ++ rows ++ footer
+ logger.info(s"Report:${System.lineSeparator()}" + formatTable(table))
+ }
+
+ def addReportInfo(
+ fileName: FileName,
+ loc: Long,
+ parsed: Boolean = false,
+ cpgGen: Boolean = false,
+ duration: Long = 0,
+ isConfig: Boolean = false
+ ): Unit =
+ reports(fileName) = ReportEntry(loc, parsed, cpgGen, duration, isConfig)
+
+ def updateReportDuration(fileName: FileName, duration: Long): Unit =
+ reports.updateWith(fileName)(_.map(_.copy(cpgGen = true, duration = duration)))
+
+}
diff --git a/src/main/scala/io/shiftleft/js2cpg/datastructures/LineAndColumn.scala b/src/main/scala/io/shiftleft/js2cpg/datastructures/LineAndColumn.scala
new file mode 100644
index 000000000..ef66b86d0
--- /dev/null
+++ b/src/main/scala/io/shiftleft/js2cpg/datastructures/LineAndColumn.scala
@@ -0,0 +1,3 @@
+package io.shiftleft.js2cpg.datastructures
+
+case class LineAndColumn(line: Option[Int], column: Option[Int])
diff --git a/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala b/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala
index dd7ab1d72..bc747d8c4 100644
--- a/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala
@@ -165,7 +165,7 @@ object FileUtils {
(positionToLineNumber, positionToFirstPositionInLine)
}
- final case class FileStatistics(linesOfCode: Int, longestLineLength: Int, containsMarker: Boolean)
+ final case class FileStatistics(linesOfCode: Long, longestLineLength: Int, containsMarker: Boolean)
private def createDecoder(): CharsetDecoder =
Codec.UTF8.decoder
@@ -173,7 +173,7 @@ object FileUtils {
.onUnmappableCharacter(CodingErrorAction.REPLACE)
def fileStatistics(lines: Iterator[String]): FileStatistics = {
- var linesOfCode = 0
+ var linesOfCode = 0L
var longestLineLength = 0
var containsMarker = false
diff --git a/src/main/scala/io/shiftleft/js2cpg/io/JsFileChecks.scala b/src/main/scala/io/shiftleft/js2cpg/io/JsFileChecks.scala
index d670197a6..1130ee02f 100644
--- a/src/main/scala/io/shiftleft/js2cpg/io/JsFileChecks.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/io/JsFileChecks.scala
@@ -1,7 +1,9 @@
package io.shiftleft.js2cpg.io
+import io.shiftleft.js2cpg.core.Js2cpgArgumentsParser
import io.shiftleft.js2cpg.io.FileDefaults._
import io.shiftleft.js2cpg.io.FileUtils.FileStatistics
+import io.shiftleft.utils.IOUtils
import org.slf4j.LoggerFactory
import java.nio.file.Path
@@ -12,13 +14,15 @@ object JsFileChecks {
private val logger = LoggerFactory.getLogger(JsFileChecks.getClass)
private def printPerformanceHints(relPath: String, reasons: Seq[String]): Unit = {
- logger.debug(s"""The file '$relPath' may have negative impact on the analyzing performance!
+ logger.debug(
+ s"""The file '$relPath' may have negative impact on the analyzing performance!
| ${if (reasons.length > 1) "Reasons:" else "Reason:"}
| ${reasons.mkString(System.lineSeparator())}
| Please check if:
| \t- this file is the result of your build process
| \t- this file is the result of applying transpilation tools (e.g., Typescript, Emscripten)
- | You might want to exclude this file when running js2cpg by adding it to '--exclude'.""".stripMargin)
+ | You might want to exclude this file when running js2cpg by adding it to '--${Js2cpgArgumentsParser.EXCLUDE}'.""".stripMargin
+ )
}
def isMinifiedFile(path: String, fileStatistics: FileStatistics): Boolean = {
diff --git a/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala b/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala
index 944e7d5f5..3c45694d3 100644
--- a/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala
@@ -18,10 +18,10 @@ case class PathFilter(
private val logger = LoggerFactory.getLogger(PathFilter.getClass)
- private val projectDir: String = Paths.get(config.inputPath).toAbsolutePath.toString
+ private val projectDir: String = Paths.get(config.srcDir).toAbsolutePath.toString
private def shouldBeIgnoredByUserConfig(filePath: Path, config: Config): Boolean =
- config.ignoredFiles.contains(filePath.toString) || config.ignoredFilesRegex.matches(filePath.toString)
+ config.ignoredFiles.contains(filePath) || config.ignoredFilesRegex.matches(filePath.toString)
private def acceptFromNodeModulesFolder(path: Path): Boolean =
withNodeModuleFolder && (".*" + NODE_MODULES_DIR_NAME + ".*").r.matches(path.toString)
@@ -34,7 +34,7 @@ case class PathFilter(
if IGNORED_FOLDERS_REGEX.exists(_.matches(File(dirPath).name)) &&
!acceptFromNodeModulesFolder(dirPath) =>
Rejected(relDir, "folder ignored by default")
- case dirPath if config.ignoredFiles.exists(i => dirPath.toString.startsWith(i)) =>
+ case dirPath if config.ignoredFiles.exists(i => dirPath.toString.startsWith(i.toString)) =>
Rejected(relDir, "folder ignored by user configuration")
case _ =>
Accepted()
diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/FreshJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/FreshJsonParser.scala
index f5705b795..dfb8d0f6c 100644
--- a/src/main/scala/io/shiftleft/js2cpg/parser/FreshJsonParser.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/parser/FreshJsonParser.scala
@@ -33,7 +33,7 @@ object FreshJsonParser {
def findImportMapPaths(config: Config): Set[Path] = {
val objectMapper = new ObjectMapper
FileUtils
- .getFileTree(Paths.get(config.inputPath), config, List(".json"))
+ .getFileTree(Paths.get(config.srcDir), config, List(".json"))
.filter(_.endsWith(TypescriptTranspiler.DenoConfig))
.flatMap { file =>
val packageJson = objectMapper.readTree(IOUtils.readLinesInFile(file).mkString)
diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala
index 534d6b286..e6e1a65ba 100644
--- a/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala
@@ -74,7 +74,7 @@ object JavaScriptParser {
}
private def nodeJsFix(jsSource: JsSource): JsSource = {
- val lines = jsSource.source.getContent.linesIterator.toSeq
+ val lines = jsSource.source.getContent.toString.linesIterator.toSeq
val replaceIndex = lines.lastIndexWhere(l => importRegex.matches(l.trim())) + 1
val (head, rest) = lines.splitAt(replaceIndex)
val fixedCode = (head ++ nodeJsFixWrapper(rest)).mkString("\n")
diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala
index 1a30329f5..7c6afada9 100644
--- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala
@@ -5,29 +5,30 @@ import better.files.File
import com.atlassian.sourcemap.{ReadableSourceMap, ReadableSourceMapImpl}
import com.oracle.js.parser.Source
import com.oracle.js.parser.ir.Node
-import io.shiftleft.js2cpg.io.FileDefaults.*
+import io.shiftleft.js2cpg.io.FileDefaults._
import io.shiftleft.js2cpg.io.FileUtils
import io.shiftleft.js2cpg.preprocessing.NuxtTranspiler
import io.shiftleft.utils.IOUtils
-
+import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
-import scala.jdk.CollectionConverters.*
+import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}
object JsSource {
private val logger = LoggerFactory.getLogger(getClass)
- case class SourceMapOrigin(
- sourceFilePath: Path,
- sourceMap: Option[ReadableSourceMap],
- sourceWithLineNumbers: Map[Int, String]
- )
+ // maximum length of re-mapped code fields after transpilation in number of characters
+ private val MAX_CODE_LENGTH: Int = 1000
+ private val MIN_CODE_LENGTH: Int = 50
+
+ def shortenCode(code: String, length: Int = MAX_CODE_LENGTH): String =
+ StringUtils.abbreviate(code, math.max(MIN_CODE_LENGTH, length))
}
-case class JsSource(srcDir: File, projectDir: Path, source: Source) {
+class JsSource(val srcDir: File, val projectDir: Path, val source: Source) {
import JsSource._
@@ -35,17 +36,40 @@ case class JsSource(srcDir: File, projectDir: Path, source: Source) {
private val mapFilePath = absoluteFilePath + ".map"
private val sourceMap = sourceMapOrigin()
- def getSourceMap: Option[SourceMapOrigin] = sourceMap
-
private val (positionToLineNumberMapping, positionToFirstPositionInLineMapping) =
FileUtils.positionLookupTables(source.getContent)
+ private case class SourceMapOrigin(
+ sourceFilePath: Path,
+ sourceMap: Option[ReadableSourceMap],
+ sourceWithLineNumbers: Map[Int, String]
+ )
+
/** @return
* the file path of the parsed file. If this file is the result of transpilation the original source file path is
* calculated from the corresponding sourcemap.
*/
def filePath: String = filePathFromSourceMap
+ /** @return
+ * the line number of a node in the parsed file. If this file is the result of transpilation the original line
+ * number is calculated from the corresponding sourcemap.
+ */
+ def getLine(node: Node): Option[Int] = lineFromSourceMap(node)
+
+ /** @return
+ * the column number of a node in the parsed file. If this file is the result of transpilation the original column
+ * number is calculated from the corresponding sourcemap.
+ */
+ def getColumn(node: Node): Option[Int] = columnFromSourceMap(node)
+
+ /** @return
+ * the code of a node in the parsed file. If this file is the result of transpilation the original code is
+ * calculated from the corresponding sourcemap. Note: in this case, only the re-mapped starting line/column number
+ * are available. Hence, we extract only a fixed number of characters (max. until the end of the file).
+ */
+ def getCode(node: Node): String = codeFromSourceMap(node)
+
def getString(node: Node): String = source.getString(node.getToken)
/** @return
@@ -141,7 +165,72 @@ case class JsSource(srcDir: File, projectDir: Path, source: Source) {
}
}
- def lineFromSourceMap(node: Node): Option[Int] = {
+ private def codeFromSourceMap(node: Node): String = {
+ sourceMap match {
+ case Some(SourceMapOrigin(_, Some(sourceMap), sourceWithLineNumbers)) =>
+ val line = getLineOfSource(node.getStart) - 1
+ val column = getColumnOfSource(node.getStart)
+ sourceMap.getMapping(line, column) match {
+ case null =>
+ source.getString(node.getStart, node.getFinish - node.getStart)
+ case mapping =>
+ val originLine = mapping.getSourceLine
+ val originColumn = mapping.getSourceColumn
+ val transpiledCodeLength = node.getFinish - node.getStart match {
+ // for some transpiled nodes the start and finish indices are wrong:
+ case 0 => node.toString.length
+ case other => other
+ }
+ sourceWithLineNumbers.get(originLine) match {
+ case Some(startingCodeLine) =>
+ // Code from the origin source file was found.
+ val maxCodeLength = math.min(transpiledCodeLength, MAX_CODE_LENGTH)
+ // We are extra careful: we do not want to generate empty lines.
+ // That can happen e.g., for synthetic return statements.
+ // Hence, we back up 1 char.
+ val startingCode =
+ startingCodeLine.substring(math.min(math.max(startingCodeLine.length - 1, 0), originColumn))
+ calculateCode(sourceWithLineNumbers, startingCode, originLine, maxCodeLength)
+ case None =>
+ // It has an actual mapping, but it is synthetic code not found in the source file.
+ // We return the synthetic code.
+ source.getString(node.getStart, node.getFinish - node.getStart)
+ }
+ }
+ case _ =>
+ // No mapping at all. We return the node code.
+ source.getString(node.getStart, node.getFinish - node.getStart)
+ }
+ }
+
+ /** Code field calculation:
+ * - We start with the re-mapped line/column number.
+ * - We always read at the length of the transpiled node (except if the original file ends earlier) capped at
+ * MAX_CODE_LENGTH.
+ * - If there would be more content we append ' [...]'.
+ */
+ @scala.annotation.tailrec
+ private def calculateCode(
+ sourceWithLineNumbers: Map[Int, String],
+ currentLine: String,
+ currentLineNumber: Int,
+ transpiledCodeLength: Int
+ ): String =
+ currentLine match {
+ case line if line.length >= transpiledCodeLength =>
+ shortenCode(line, transpiledCodeLength - 1)
+ case line if line.length < transpiledCodeLength && sourceWithLineNumbers.contains(currentLineNumber + 1) =>
+ calculateCode(
+ sourceWithLineNumbers,
+ line + "\n" + sourceWithLineNumbers(currentLineNumber + 1),
+ currentLineNumber + 1,
+ transpiledCodeLength
+ )
+ case line =>
+ line.stripLineEnd
+ }
+
+ private def lineFromSourceMap(node: Node): Option[Int] = {
sourceMap match {
case Some(SourceMapOrigin(_, Some(sourceMap), _)) =>
val line = getLineOfSource(node.getStart) - 1
@@ -152,7 +241,7 @@ case class JsSource(srcDir: File, projectDir: Path, source: Source) {
}
}
- def columnFromSourceMap(node: Node): Option[Int] = {
+ private def columnFromSourceMap(node: Node): Option[Int] = {
sourceMap match {
case Some(SourceMapOrigin(_, Some(sourceMap), _)) =>
val line = getLineOfSource(node.getStart) - 1
@@ -178,14 +267,14 @@ case class JsSource(srcDir: File, projectDir: Path, source: Source) {
// Returns the line number for a given position in the source.
// We use this method instead of source.getLine for performance reasons.
- def getLineOfSource(position: Int): Int = {
+ private def getLineOfSource(position: Int): Int = {
val (_, lineNumber) = positionToLineNumberMapping.minAfter(position).get
lineNumber
}
// Returns the column number for a given position in the source.
// We use this method instead of source.getColumn for performance reasons.
- def getColumnOfSource(position: Int): Int = {
+ private def getColumnOfSource(position: Int): Int = {
val (_, firstPositionInLine) = positionToFirstPositionInLineMapping.minAfter(position).get
position - firstPositionInLine
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala
index 9a8432280..0b77da5fa 100644
--- a/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala
@@ -4,23 +4,22 @@ import java.nio.file.Path
import better.files.File
import com.oracle.js.parser.Source
import com.oracle.js.parser.ir.FunctionNode
-import io.joern.x2cpg.utils.Report
-import io.joern.x2cpg.utils.TimeUtils
import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.js2cpg.core.Report
import io.shiftleft.js2cpg.astcreation.AstCreator
-import io.shiftleft.js2cpg.core.Config
import io.shiftleft.js2cpg.io.{FileUtils, JsFileChecks}
import io.shiftleft.js2cpg.parser.{JavaScriptParser, JsSource}
import io.shiftleft.passes.ConcurrentWriterCpgPass
import org.slf4j.LoggerFactory
-import io.shiftleft.js2cpg.utils.SourceWrapper.*
+import io.shiftleft.js2cpg.utils.SourceWrapper._
+import io.shiftleft.js2cpg.utils.TimeUtils
import scala.util.{Failure, Success, Try}
/** Given a list of filenames, this pass creates the abstract syntax tree and CPG AST for each file. Files are processed
* in parallel.
*/
-class AstCreationPass(cpg: Cpg, filenames: List[(Path, Path)], config: Config, report: Report)
+class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, report: Report)
extends ConcurrentWriterCpgPass[(Path, Path)](cpg) {
private val logger = LoggerFactory.getLogger(getClass)
@@ -50,7 +49,7 @@ class AstCreationPass(cpg: Cpg, filenames: List[(Path, Path)], config: Config, r
logger.warn(s"Failed to generate CPG for '$path'!", exception)
case Success(localDiff) =>
logger.info(s"Processed file '$path'")
- report.updateReport(path, true, duration)
+ report.updateReportDuration(path, duration)
diffGraph.absorb(localDiff)
}
}
@@ -85,7 +84,7 @@ class AstCreationPass(cpg: Cpg, filenames: List[(Path, Path)], config: Config, r
val fileStatistics = JsFileChecks.check(relPath, lines)
val source = Source.sourceFor(relPath, lines.mkString("\n"))
- val jsSource = source.toJsSource(File(config.inputPath), rootDir)
+ val jsSource = source.toJsSource(srcDir, rootDir)
logger.debug(s"Parsing file '$relPath'.")
Try(JavaScriptParser.parseFromSource(jsSource)) match {
diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPass.scala
index f4511a70f..b68ec4751 100644
--- a/src/main/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPass.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPass.scala
@@ -21,7 +21,7 @@ class BuiltinTypesPass(cpg: Cpg) extends CpgPass(cpg) {
diffGraph.addNode(namespaceBlock)
val orderTracker = new OrderTracker()
- Defines.JsTypes.foreach { typeName =>
+ Defines.JsTypes.foreach { case typeName: String =>
val tpe = NewType()
.name(typeName)
.fullName(typeName)
diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala
index 2e369e973..31d07a23c 100644
--- a/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala
@@ -1,11 +1,11 @@
package io.shiftleft.js2cpg.passes
-import io.joern.x2cpg.utils.Report
-import io.joern.x2cpg.utils.TimeUtils
import io.shiftleft.codepropertygraph.Cpg
import io.shiftleft.codepropertygraph.generated.nodes.NewConfigFile
+import io.shiftleft.js2cpg.core.Report
import io.shiftleft.js2cpg.io.FileDefaults
import io.shiftleft.js2cpg.io.FileUtils
+import io.shiftleft.js2cpg.utils.TimeUtils
import io.shiftleft.passes.ConcurrentWriterCpgPass
import io.shiftleft.utils.IOUtils
import org.slf4j.{Logger, LoggerFactory}
@@ -40,12 +40,12 @@ class ConfigPass(filenames: List[(Path, Path)], cpg: Cpg, report: Report)
val localDiff = new DiffGraphBuilder
logger.debug(s"Adding file '$relativeFile' as config file.")
val configNode = NewConfigFile().name(fileName).content(content.mkString("\n"))
- report.addReportInfo(fileName, fileStatistics.linesOfCode, parsed = true, cpgGen = true)
+ report.addReportInfo(fileName, fileStatistics.linesOfCode, parsed = true, cpgGen = true, isConfig = true)
localDiff.addNode(configNode)
localDiff
}
diffGraph.absorb(result)
- report.updateReport(fileName, true, time)
+ report.updateReportDuration(fileName, time)
}
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/DependenciesPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/DependenciesPass.scala
index 6ae8d6c78..8931dda7d 100644
--- a/src/main/scala/io/shiftleft/js2cpg/passes/DependenciesPass.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/passes/DependenciesPass.scala
@@ -15,7 +15,7 @@ class DependenciesPass(cpg: Cpg, config: Config) extends CpgPass(cpg) {
private def dependenciesForPackageJsons(): Map[String, String] = {
val packagesJsons =
(FileUtils
- .getFileTree(Paths.get(config.inputPath), config, List(".json"))
+ .getFileTree(Paths.get(config.srcDir), config, List(".json"))
.filter(_.toString.endsWith(FileDefaults.PACKAGE_JSON_FILENAME)) :+
config.createPathForPackageJson()).toSet
packagesJsons.flatMap(p => PackageJsonParser.dependencies(p)).toMap
diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/PrivateKeyFilePass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/PrivateKeyFilePass.scala
index 00372bb21..96fe83288 100644
--- a/src/main/scala/io/shiftleft/js2cpg/passes/PrivateKeyFilePass.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/passes/PrivateKeyFilePass.scala
@@ -1,7 +1,7 @@
package io.shiftleft.js2cpg.passes
-import io.joern.x2cpg.utils.Report
import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.js2cpg.core.Report
import io.shiftleft.utils.IOUtils
import java.nio.file.Path
diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala
index bd8e946ff..a0755a1eb 100644
--- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala
@@ -46,7 +46,7 @@ class NuxtTranspiler(override val config: Config, override val projectPath: Path
private def isNuxtProject: Boolean =
PackageJsonParser
- .dependencies((File(config.inputPath) / FileDefaults.PACKAGE_JSON_FILENAME).path)
+ .dependencies((File(config.srcDir) / FileDefaults.PACKAGE_JSON_FILENAME).path)
.contains("nuxt")
override def shouldRun(): Boolean = config.nuxtTranspiling && isNuxtProject
diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala
index c0a91ab27..e4f655630 100644
--- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala
@@ -67,7 +67,7 @@ class TranspilationRunner(projectPath: Path, tmpTranspileDir: Path, config: Conf
}
def handlePrivateModules(): List[(Path, Path)] = {
- val project = File(config.inputPath)
+ val project = File(config.srcDir)
val nodeModulesFolder = project / NODE_MODULES_DIR_NAME
if (!nodeModulesFolder.exists) {
List.empty
diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala
index f4f51b334..92f670bd9 100644
--- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala
@@ -160,7 +160,7 @@ trait TranspilingEnvironment {
isYarnAvailable.get
}
- private def npmAvailable(): Boolean = isNpmAvailable match {
+ protected def npmAvailable(): Boolean = isNpmAvailable match {
case Some(value) =>
value
case None =>
diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala
index 30d6adee4..dad399398 100644
--- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala
@@ -41,14 +41,14 @@ object VueTranspiler {
def isVueProject(config: Config, projectPath: Path): Boolean = {
val hasVueDep =
PackageJsonParser
- .dependencies((File(config.inputPath) / FileDefaults.PACKAGE_JSON_FILENAME).path)
+ .dependencies((File(config.srcDir) / FileDefaults.PACKAGE_JSON_FILENAME).path)
.exists(_._1.startsWith("vue"))
hasVueDep && hasVueFiles(config, projectPath)
}
private def vueVersion(config: Config): Int = {
val versionString =
- PackageJsonParser.dependencies((File(config.inputPath) / FileDefaults.PACKAGE_JSON_FILENAME).path)("vue")
+ PackageJsonParser.dependencies((File(config.srcDir) / FileDefaults.PACKAGE_JSON_FILENAME).path)("vue")
// ignore ~, ^, and more from semver; see: https://stackoverflow.com/a/25861938
val c = versionString.collectFirst { case c if Try(c.toString.toInt).isSuccess => c.toString.toInt }
c.getOrElse(3) // 3 is the latest version; we default to that
diff --git a/src/main/scala/io/shiftleft/js2cpg/utils/MemoryMetrics.scala b/src/main/scala/io/shiftleft/js2cpg/utils/MemoryMetrics.scala
index 64d1f0c38..aae334b7e 100644
--- a/src/main/scala/io/shiftleft/js2cpg/utils/MemoryMetrics.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/utils/MemoryMetrics.scala
@@ -3,7 +3,6 @@ package io.shiftleft.js2cpg.utils
import io.shiftleft.js2cpg.core.Config
import org.slf4j.LoggerFactory
-import scala.util.Try
import scala.util.Using
object MemoryMetrics {
@@ -55,14 +54,14 @@ object MemoryMetrics {
}
}
- def withMemoryMetrics[T](config: Config)(work: => T): T = config.jvmMetrics match {
+ def withMemoryMetrics(config: Config)(work: => Unit): Unit = config.jvmMetrics match {
case Some(port) =>
Using(new JmxRunnable(port, 5000)) { monitor =>
val t = new Thread(monitor)
t.setName("js2cpg-jvm-monitor")
t.start()
work
- }.get
+ }
case None => work
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/utils/SourceWrapper.scala b/src/main/scala/io/shiftleft/js2cpg/utils/SourceWrapper.scala
index 304194f16..80c63d03b 100644
--- a/src/main/scala/io/shiftleft/js2cpg/utils/SourceWrapper.scala
+++ b/src/main/scala/io/shiftleft/js2cpg/utils/SourceWrapper.scala
@@ -1,13 +1,13 @@
package io.shiftleft.js2cpg.utils
import better.files.File
-import com.oracle.js.parser.Source
+
import java.nio.file.{Path, Paths}
import io.shiftleft.js2cpg.parser.JsSource
object SourceWrapper {
- implicit class SourceWrapper(val source: Source) extends AnyVal {
- def toJsSource(srcDir: File = File(""), projectDir: Path = Paths.get("")): JsSource =
- JsSource(srcDir, projectDir, source)
+ implicit class SourceWrapper(val source: com.oracle.js.parser.Source) extends AnyVal {
+ def toJsSource(srcDir: File = File(""), projectDir: Path = Paths.get("")) =
+ new JsSource(srcDir, projectDir, source)
}
}
diff --git a/src/main/scala/io/shiftleft/js2cpg/utils/TimeUtils.scala b/src/main/scala/io/shiftleft/js2cpg/utils/TimeUtils.scala
new file mode 100644
index 000000000..647acf6c6
--- /dev/null
+++ b/src/main/scala/io/shiftleft/js2cpg/utils/TimeUtils.scala
@@ -0,0 +1,55 @@
+package io.shiftleft.js2cpg.utils
+
+import java.util.Locale
+import scala.concurrent.duration._
+
+object TimeUtils {
+
+ /** Measures elapsed time for executing a block in nanoseconds */
+ def time[R](block: => R): (R, Long) = {
+ val t0 = System.nanoTime()
+ val result = block
+ val t1 = System.nanoTime()
+ val elapsed = t1 - t0
+ (result, elapsed)
+ }
+
+ /** Selects most appropriate TimeUnit for given duration and formats it accordingly */
+ def pretty(duration: Duration): String = {
+ duration match {
+ case d: FiniteDuration =>
+ val nanos = d.toNanos
+ val unit = chooseUnit(nanos)
+ val value = nanos.toDouble / NANOSECONDS.convert(1, unit)
+
+ s"%.4g %s".formatLocal(Locale.ROOT, value, abbreviate(unit))
+
+ case Duration.MinusInf => s"-∞ (minus infinity)"
+ case Duration.Inf => s"∞ (infinity)"
+ case _ => "undefined"
+ }
+ }
+
+ private def chooseUnit(nanos: Long): TimeUnit = {
+ val d = nanos.nanos
+
+ if (d.toDays > 0) DAYS
+ else if (d.toHours > 0) HOURS
+ else if (d.toMinutes > 0) MINUTES
+ else if (d.toSeconds > 0) SECONDS
+ else if (d.toMillis > 0) MILLISECONDS
+ else if (d.toMicros > 0) MICROSECONDS
+ else NANOSECONDS
+ }
+
+ private def abbreviate(unit: TimeUnit): String = unit match {
+ case NANOSECONDS => "ns"
+ case MICROSECONDS => "μs"
+ case MILLISECONDS => "ms"
+ case SECONDS => "s"
+ case MINUTES => "min"
+ case HOURS => "h"
+ case DAYS => "d"
+ }
+
+}
diff --git a/src/test/resources/excludes/a.js b/src/test/resources/excludes/a.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/resources/excludes/folder/b.js b/src/test/resources/excludes/folder/b.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/resources/excludes/folder/c.js b/src/test/resources/excludes/folder/c.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/resources/excludes/foo.bar/d.js b/src/test/resources/excludes/foo.bar/d.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/resources/excludes/index.js b/src/test/resources/excludes/index.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/resources/excludes/tests/a.spec.js b/src/test/resources/excludes/tests/a.spec.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/resources/excludes/tests/b.mock.js b/src/test/resources/excludes/tests/b.mock.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/resources/excludes/tests/c.e2e.js b/src/test/resources/excludes/tests/c.e2e.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/resources/excludes/tests/d.test.js b/src/test/resources/excludes/tests/d.test.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala
index 33e42e38f..f7b7aa889 100644
--- a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala
@@ -1,67 +1,50 @@
package io.shiftleft.js2cpg.io
import better.files.File
-import io.joern.x2cpg.X2Cpg.newEmptyCpg
-import io.joern.x2cpg.utils.Report
-import io.shiftleft.js2cpg.core.Config
-import io.shiftleft.js2cpg.io.FileDefaults.JS_SUFFIX
-import io.shiftleft.js2cpg.passes.AstCreationPass
-import io.shiftleft.semanticcpg.language.*
+import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.js2cpg.core.{Js2cpgArgumentsParser, Js2CpgMain}
+import io.shiftleft.semanticcpg.language._
+
import org.scalatest.matchers.should.Matchers
import org.scalatest.prop.TableDrivenPropertyChecks
import org.scalatest.wordspec.AnyWordSpec
-import org.scalatest.BeforeAndAfterAll
import java.util.regex.Pattern
-class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChecks with BeforeAndAfterAll {
-
- private val testFiles: List[String] = List(
- ".sub/e.js",
- "folder/b.js",
- "folder/c.js",
- "foo.bar/d.js",
- "tests/a.spec.js",
- "tests/b.mock.js",
- "tests/c.e2e.js",
- "tests/d.test.js",
- "a.js",
- "b-min.js",
- "c.spec.js",
- "d.chunk.js",
- "index.js"
- )
-
- private val projectUnderTest: File = {
- val dir = File.newTemporaryDirectory("jssrc2cpgTestsExcludeTest")
- testFiles.foreach { testFile =>
- val file = dir / testFile
- file.createIfNotExists(createParents = true)
- }
- dir
+object ExcludeTest {
+ private implicit class ToArg(val arg: String) extends AnyVal {
+ def toArg: String = s"--$arg"
}
+}
+
+class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChecks {
+
+ import ExcludeTest._
+ import Js2cpgArgumentsParser._
- override def afterAll(): Unit = projectUnderTest.delete(swallowIOExceptions = true)
+ private val projectUnderTestPath = File(getClass.getResource("/excludes").toURI).pathAsString
+
+ private def fileNames(cpg: Cpg): List[String] = cpg.file.name.l
private def testWithArguments(
- exclude: Seq[String],
- excludeRegex: String,
- ignoreTests: Boolean,
- expectedFiles: Set[String]
+ args: Seq[String],
+ expectedFiles: Set[String],
+ defaultArgs: Set[String] = Set(NO_TS, NO_BABEL)
): Unit = {
- File.usingTemporaryDirectory("js2cpgTests") { tmpDir =>
- val cpg = newEmptyCpg()
- val config = Config()
- .withInputPath(projectUnderTest.toString)
- .withOutputPath(tmpDir.toString)
- .withIgnoredFiles(exclude)
- .withIgnoredFilesRegex(excludeRegex)
- .withIgnoreTests(ignoreTests)
- val files = FileUtils
- .getFileTree(projectUnderTest.path, config, List(JS_SUFFIX))
- .map(f => (f, projectUnderTest.path))
- new AstCreationPass(cpg, files, config, new Report()).createAndApply()
- cpg.file.name.l should contain theSameElementsAs expectedFiles.map(_.replace("/", java.io.File.separator))
+ File.usingTemporaryDirectory("js2cpgTest") { tmpDir =>
+ val cpgPath = tmpDir / "cpg.bin.zip"
+
+ Js2CpgMain.main(
+ Array(projectUnderTestPath, "--output", cpgPath.pathAsString) ++
+ args.toArray ++
+ defaultArgs.map(_.toArg).toArray
+ )
+
+ val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
+
+ fileNames(cpg) should contain theSameElementsAs expectedFiles.map(_.replace("/", java.io.File.separator))
+ cpg.close()
+ cpgPath.deleteOnExit()
}
}
@@ -69,106 +52,83 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
val testInput = Table(
// -- Header for naming all test parameters
- ("statement", "--exclude", "--exclude-regex", "with-tests", "expectedResult"),
+ ("statement", "arguments", "expectedResult"),
// --
// Test for default:
(
"exclude nothing if no excludes are given",
Seq.empty[String],
- "",
- true,
Set("index.js", "a.js", "folder/b.js", "folder/c.js", "foo.bar/d.js")
),
// --
// Tests for --exclude only:
(
- "exclude a file with --exclude with relative path",
- Seq("index.js"),
- "",
- true,
+ s"exclude a file with ${EXCLUDE.toArg} with relative path",
+ Seq(EXCLUDE.toArg, "index.js"),
Set("a.js", "folder/b.js", "folder/c.js", "foo.bar/d.js")
),
(
- "exclude files with --exclude with relative paths",
- Seq("index.js", "folder/b.js"),
- "",
- true,
+ s"exclude files with ${EXCLUDE.toArg} with relative paths",
+ Seq(EXCLUDE.toArg, "index.js,folder/b.js"),
Set("a.js", "folder/c.js", "foo.bar/d.js")
),
(
- "exclude a file with --exclude with absolute path",
- Seq(s"$projectUnderTest/index.js"),
- "",
- true,
+ s"exclude a file with ${EXCLUDE.toArg} with absolute path",
+ Seq(EXCLUDE.toArg, s"$projectUnderTestPath/index.js"),
Set("a.js", "folder/b.js", "folder/c.js", "foo.bar/d.js")
),
(
- "exclude files with --exclude with absolute paths",
- Seq(s"$projectUnderTest/index.js", s"$projectUnderTest/folder/b.js"),
- "",
- true,
+ s"exclude files with ${EXCLUDE.toArg} with absolute paths",
+ Seq(EXCLUDE.toArg, s"$projectUnderTestPath/index.js,$projectUnderTestPath/folder/b.js"),
Set("a.js", "folder/c.js", "foo.bar/d.js")
),
(
- "exclude files with --exclude with mixed paths",
- Seq("index.js", s"$projectUnderTest/folder/b.js"),
- "",
- true,
+ s"exclude files with ${EXCLUDE.toArg} with mixed paths",
+ Seq(EXCLUDE.toArg, s"index.js,$projectUnderTestPath/folder/b.js"),
Set("a.js", "folder/c.js", "foo.bar/d.js")
),
(
- "exclude a folder with --exclude with absolute path",
- Seq(s"$projectUnderTest/folder/"),
- "",
- true,
+ s"exclude a folder with ${EXCLUDE.toArg} with absolute path",
+ Seq(EXCLUDE.toArg, s"$projectUnderTestPath/folder/"),
Set("a.js", "index.js", "foo.bar/d.js")
),
(
- "exclude a folder with --exclude with relative path",
- Seq("folder/"),
- "",
- true,
+ s"exclude a folder with ${EXCLUDE.toArg} with relative path",
+ Seq(EXCLUDE.toArg, s"folder/"),
Set("a.js", "index.js", "foo.bar/d.js")
),
// --
// Tests for --exclude-regex only:
(
- "exclude a file with --exclude-regex",
- Seq.empty,
- ".*index\\..*",
- true,
+ s"exclude a file with ${EXCLUDE_REGEX.toArg}",
+ Seq(EXCLUDE_REGEX.toArg, ".*index\\..*"),
Set("a.js", "folder/b.js", "folder/c.js", "foo.bar/d.js")
),
(
- "exclude files with --exclude-regex",
- Seq.empty,
- ".*(index|b)\\..*",
- true,
+ s"exclude files with ${EXCLUDE_REGEX.toArg}",
+ Seq(EXCLUDE_REGEX.toArg, ".*(index|b)\\..*"),
Set("a.js", "folder/c.js", "foo.bar/d.js")
),
(
- "exclude a complete folder with --exclude-regex",
- Seq.empty,
- s".*${Pattern.quote(java.io.File.separator)}?folder${Pattern.quote(java.io.File.separator)}.*",
- true,
+ s"exclude a complete folder with ${EXCLUDE_REGEX.toArg}",
+ Seq(
+ EXCLUDE_REGEX.toArg,
+ s".*${Pattern.quote(java.io.File.separator)}folder${Pattern.quote(java.io.File.separator)}.*"
+ ),
Set("index.js", "a.js", "foo.bar/d.js")
),
// --
// Tests for mixed arguments
(
- "exclude files with --exclude and --exclude-regex",
- Seq("a.js"),
- ".*(index|b)\\..*",
- true,
+ s"exclude files with ${EXCLUDE.toArg} and ${EXCLUDE_REGEX.toArg}",
+ Seq(EXCLUDE.toArg, "a.js", EXCLUDE_REGEX.toArg, ".*(index|b)\\..*"),
Set("folder/c.js", "foo.bar/d.js")
),
// --
// Tests for including test files
(
- "include test files with --with-tests",
- Seq.empty,
- "",
- false,
+ s"include test files with ${WITH_TESTS.toArg}",
+ Seq(WITH_TESTS.toArg),
Set(
"index.js",
"a.js",
@@ -178,14 +138,13 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
"tests/a.spec.js",
"tests/b.mock.js",
"tests/c.e2e.js",
- "tests/d.test.js",
- "c.spec.js"
+ "tests/d.test.js"
)
)
)
- forAll(testInput) { (statement, exclude, excludeRegex, withTests, result) =>
- s"$statement" in testWithArguments(exclude, excludeRegex, withTests, result)
+ forAll(testInput) { (statement, arguments, result) =>
+ s"$statement" in testWithArguments(arguments, result)
}
}
diff --git a/src/test/scala/io/shiftleft/js2cpg/io/ExternalCommandTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/ExternalCommandTest.scala
index 683625a5a..152c545b0 100644
--- a/src/test/scala/io/shiftleft/js2cpg/io/ExternalCommandTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/io/ExternalCommandTest.scala
@@ -5,22 +5,17 @@ import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
class ExternalCommandTest extends AnyWordSpec with Matchers {
-
- private val command = if (scala.util.Properties.isWin) "cmd /C dir" else "ls"
-
"ExternalCommand" should {
"run an external command with ProcessBuilder and no spaces in the directory name" in {
File.usingTemporaryDirectory("js2cpgTest") { sourceDir =>
- val cmd = s"$command ${sourceDir.pathAsString}"
- (sourceDir / "Main.js").createFileIfNotExists().write("console.log('Foo');")
+ val cmd = "ls " + sourceDir.pathAsString
ExternalCommand.run(cmd, sourceDir.pathAsString) should be a Symbol("success")
}
}
"run an external command with ProcessBuilder and spaces in the directory name" in {
File.usingTemporaryDirectory("js2cpg Test") { sourceDir =>
- val cmd = s"$command ${sourceDir.pathAsString}"
- (sourceDir / "Main.js").createFileIfNotExists().write("console.log('Foo');")
+ val cmd = "ls " + sourceDir.pathAsString
ExternalCommand.run(cmd, sourceDir.pathAsString) should be a Symbol("success")
}
}
diff --git a/src/test/scala/io/shiftleft/js2cpg/io/FileUtilsTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/FileUtilsTest.scala
index 01f1ee4f2..eba6f2ec1 100644
--- a/src/test/scala/io/shiftleft/js2cpg/io/FileUtilsTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/io/FileUtilsTest.scala
@@ -19,7 +19,7 @@ class FileUtilsTest extends AnyWordSpec with Matchers {
(sourceDir / ".folder" / "e.js").createIfNotExists(createParents = true)
File.usingTemporaryDirectory("js2cpgTest") { targetDir =>
- val config = Config().withInputPath(sourceDir.pathAsString)
+ val config = Config(srcDir = sourceDir.pathAsString)
val copiedDir = FileUtils.copyToDirectory(sourceDir, targetDir, config)
val dirContent = FileUtils.getFileTree(copiedDir.path, config, List(JS_SUFFIX))
dirContent shouldBe empty
@@ -34,7 +34,7 @@ class FileUtilsTest extends AnyWordSpec with Matchers {
(sourceDir / "b-min.js").createFile()
(sourceDir / "b-min.23472420.js").createFile()
- val config = Config().withInputPath(sourceDir.pathAsString)
+ val config = Config(srcDir = sourceDir.pathAsString)
val minFile = (sourceDir / "something.js").createFile()
minFile.write(s"console.log('${"x" * FileDefaults.LINE_LENGTH_THRESHOLD}');")
val dirContent = FileUtils.getFileTree(sourceDir.path, config, List(JS_SUFFIX))
diff --git a/src/test/scala/io/shiftleft/js2cpg/parser/ParserTest.scala b/src/test/scala/io/shiftleft/js2cpg/parser/ParserTest.scala
index ccd297ec6..3c9a7728c 100644
--- a/src/test/scala/io/shiftleft/js2cpg/parser/ParserTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/parser/ParserTest.scala
@@ -109,7 +109,7 @@ class ParserTest extends AnyWordSpec with Matchers {
val expected =
"(function (exports, require, module, __filename, __dirname) { if(true) { return 1; } });"
val jsSource = JavaScriptParser.parse(jsfunction)._2
- jsSource.source.getContent shouldBe expected
+ jsSource.source.getContent.toString shouldBe expected
}
"fix NodeJS invalid return with import like code line" in {
@@ -124,7 +124,7 @@ class ParserTest extends AnyWordSpec with Matchers {
|// fooimportbar
|console.log('fizzbuzz'); });""".stripMargin
val jsSource = JavaScriptParser.parse(jsfunction)._2
- jsSource.source.getContent shouldBe expected
+ jsSource.source.getContent.toString shouldBe expected
}
"fix NodeJS invalid return with simple import" in {
@@ -137,7 +137,7 @@ class ParserTest extends AnyWordSpec with Matchers {
|import foo from 'dep.js';
|(function (exports, require, module, __filename, __dirname) { if(true) { return 1; } });""".stripMargin
val jsSource = JavaScriptParser.parse(jsfunction)._2
- jsSource.source.getContent shouldBe expected
+ jsSource.source.getContent.toString shouldBe expected
}
"fix NodeJS invalid return with multiple imports" in {
@@ -168,7 +168,7 @@ class ParserTest extends AnyWordSpec with Matchers {
|import "module-name";
|(function (exports, require, module, __filename, __dirname) { if(true) { return 1; } });""".stripMargin
val jsSource = JavaScriptParser.parse(jsfunction)._2
- jsSource.source.getContent shouldBe expected
+ jsSource.source.getContent.toString shouldBe expected
}
}
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala
index 55872e9bd..054762224 100644
--- a/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala
@@ -2,14 +2,18 @@ package io.shiftleft.js2cpg.passes
import better.files.File
import io.joern.x2cpg.X2Cpg.newEmptyCpg
-import io.joern.x2cpg.utils.Report
+import io.shiftleft.codepropertygraph.generated.EdgeTypes
import io.shiftleft.codepropertygraph.generated.nodes.Dependency
import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.js2cpg.core.Config
+import io.shiftleft.js2cpg.core.Report
import io.shiftleft.semanticcpg.language.*
+import overflowdb.Node
+import overflowdb.traversal.*
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
+import scala.jdk.CollectionConverters.*
+
abstract class AbstractPassTest extends AnyWordSpec with Matchers {
protected abstract class Fixture
@@ -23,7 +27,7 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers {
file.write(code)
val cpg = newEmptyCpg()
val filenames = List((file.path, file.parent.path))
- new AstCreationPass(cpg, filenames, Config().withInputPath(dir.toString), new Report()).createAndApply()
+ new AstCreationPass(dir, filenames, cpg, new Report()).createAndApply()
f(cpg)
file.delete()
}
@@ -33,7 +37,7 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers {
val file = testFile
val cpg = newEmptyCpg()
val filenames = List((file.path, file.parent.path))
- new AstCreationPass(cpg, filenames, Config().withInputPath(file.parent.toString), new Report()).createAndApply()
+ new AstCreationPass(file.parent, filenames, cpg, new Report()).createAndApply()
f(cpg)
}
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala
index cd01d122a..353047375 100644
--- a/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala
@@ -14,7 +14,7 @@ class BuiltinTypesPassTest extends AbstractPassTest {
}
"create types and type decls correctly" in {
- Defines.JsTypes.foreach { typeName =>
+ Defines.JsTypes.foreach { case typeName: String =>
val typeDeclNodes = cpg.typeDecl(typeName).l
typeDeclNodes should have length 1
val typeDeclNode = typeDeclNodes.head
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala
index 568dee7ef..e7b61f299 100644
--- a/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala
@@ -2,14 +2,13 @@ package io.shiftleft.js2cpg.passes
import better.files.File
import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.*
-import io.shiftleft.semanticcpg.language.*
+import io.shiftleft.codepropertygraph.generated._
+import io.shiftleft.js2cpg.core.Report
+import io.shiftleft.semanticcpg.language._
import io.joern.x2cpg.passes.controlflow.CfgCreationPass
-import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg.*
-import io.joern.x2cpg.utils.Report
+import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg._
import io.shiftleft.codepropertygraph.generated.nodes.AstNodeBase
import io.shiftleft.codepropertygraph.generated.nodes.Method
-import io.shiftleft.js2cpg.core.Config
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import overflowdb.Node
@@ -1297,7 +1296,7 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers {
val file = workspace / "test.js"
file.write(code)
val filenames = List((file.path, file.parent.path))
- new AstCreationPass(cpg, filenames, Config().withInputPath(workspace.toString), new Report()).createAndApply()
+ new AstCreationPass(workspace, filenames, cpg, new Report()).createAndApply()
new CfgCreationPass(cpg).createAndApply()
}
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala
index 5cbc3b7e8..1cc004acb 100644
--- a/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala
@@ -1,9 +1,9 @@
package io.shiftleft.js2cpg.passes
+import io.shiftleft.js2cpg.core.Report
import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.semanticcpg.language.*
+import io.shiftleft.semanticcpg.language._
import better.files.File
-import io.joern.x2cpg.utils.Report
import io.shiftleft.js2cpg.io.FileDefaults
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/DependenciesPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/DependenciesPassTest.scala
index 0347baf50..ec08cd93e 100644
--- a/src/test/scala/io/shiftleft/js2cpg/passes/DependenciesPassTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/passes/DependenciesPassTest.scala
@@ -2,13 +2,12 @@ package io.shiftleft.js2cpg.passes
import better.files.File
import io.joern.x2cpg.X2Cpg.newEmptyCpg
-import io.joern.x2cpg.utils.Report
import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.js2cpg.core.Config
+import io.shiftleft.js2cpg.core.{Config, Report}
import io.shiftleft.js2cpg.io.FileDefaults
import io.shiftleft.js2cpg.parser.PackageJsonParser
import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler
-import io.shiftleft.semanticcpg.language.*
+import io.shiftleft.semanticcpg.language._
class DependenciesPassTest extends AbstractPassTest {
@@ -181,9 +180,8 @@ class DependenciesPassTest extends AbstractPassTest {
val filenames = List((file.path, file.parent.path))
val cpg = newEmptyCpg()
- val config = Config().withInputPath(dir.toString).withPackageJsonLocation(packageJson)
- new AstCreationPass(cpg, filenames, config, new Report()).createAndApply()
- new DependenciesPass(cpg, config).createAndApply()
+ new AstCreationPass(dir, filenames, cpg, new Report()).createAndApply()
+ new DependenciesPass(cpg, Config(srcDir = dir.toString, packageJsonLocation = packageJson)).createAndApply()
f(cpg)
jsonTmpFiles.foreach(_.delete())
diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/SimpleAstCreationPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/SimpleAstCreationPassTest.scala
index 155e841a7..beab49894 100644
--- a/src/test/scala/io/shiftleft/js2cpg/passes/SimpleAstCreationPassTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/passes/SimpleAstCreationPassTest.scala
@@ -1,7 +1,11 @@
package io.shiftleft.js2cpg.passes
+import io.shiftleft.codepropertygraph.Cpg
import io.shiftleft.codepropertygraph.generated.*
import io.shiftleft.codepropertygraph.generated.nodes.*
+import io.shiftleft.js2cpg.core.Config
+import io.shiftleft.js2cpg.io.FileDefaults.JS_SUFFIX
+import io.shiftleft.js2cpg.io.FileUtils
import io.shiftleft.semanticcpg.language.*
class SimpleAstCreationPassTest extends AbstractPassTest {
diff --git a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala
index 48d151392..80f686843 100644
--- a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala
+++ b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala
@@ -32,7 +32,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers {
"generate js files correctly for a simple Babel project" in
TranspilationFixture("babel") { tmpDir =>
File.usingTemporaryDirectory("js2cpgTest") { transpileOutDir =>
- val config = Config().withInputPath(transpileOutDir.pathAsString).withTsTranspiling(false)
+ val config = Config(srcDir = transpileOutDir.pathAsString, tsTranspiling = false)
new TranspilationRunner(tmpDir.path, transpileOutDir.path, config).execute()
@@ -52,7 +52,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers {
"generate js files correctly for a simple Babel project in folder with whitespace" in
TranspilationFixture("babel") { tmpDir =>
File.usingTemporaryDirectory("js2cpgTest folder") { transpileOutDir =>
- val config = Config().withInputPath(transpileOutDir.pathAsString).withTsTranspiling(false)
+ val config = Config(srcDir = transpileOutDir.pathAsString, tsTranspiling = false)
new TranspilationRunner(tmpDir.path, transpileOutDir.path, config).execute()
@@ -113,10 +113,10 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers {
"generate js files correctly for a simple Typescript project" in
TranspilationFixture("typescript") { tmpDir =>
File.usingTemporaryDirectory("js2cpgTest") { transpileOutDir =>
- val config = Config().withInputPath(transpileOutDir.pathAsString).withTsTranspiling(false)
+ val config = Config(srcDir = transpileOutDir.pathAsString, tsTranspiling = false)
val jsFiles = FileUtils
- .getFileTree(tmpDir.path, Config().withInputPath(tmpDir.pathAsString), List(JS_SUFFIX))
+ .getFileTree(tmpDir.path, Config(srcDir = tmpDir.pathAsString), List(JS_SUFFIX))
.map(f => (f, tmpDir.path))
val expectedJsFiles =
@@ -294,14 +294,10 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers {
new TranspilationRunner(
tmpDir.path,
transpileOutDir.path,
- Config().withInputPath(tmpDir.pathAsString).withBabelTranspiling(false).withOptimizeDependencies(false)
+ Config(srcDir = tmpDir.pathAsString, babelTranspiling = false, optimizeDependencies = false)
).execute()
val transpiledJsFiles =
- FileUtils.getFileTree(
- transpileOutDir.path,
- Config().withInputPath(transpileOutDir.pathAsString),
- List(JS_SUFFIX)
- )
+ FileUtils.getFileTree(transpileOutDir.path, Config(srcDir = transpileOutDir.pathAsString), List(JS_SUFFIX))
transpiledJsFiles shouldBe empty
}
}
@@ -313,10 +309,10 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers {
new TranspilationRunner(
tmpDir.path,
transpileOutDir.path,
- Config().withInputPath(tmpDir.pathAsString).withBabelTranspiling(false).withOptimizeDependencies(true)
+ Config(srcDir = tmpDir.pathAsString, babelTranspiling = false, optimizeDependencies = true)
).execute()
val transpiledJsFiles = FileUtils
- .getFileTree(transpileOutDir.path, Config().withInputPath(transpileOutDir.pathAsString), List(JS_SUFFIX))
+ .getFileTree(transpileOutDir.path, Config(srcDir = transpileOutDir.pathAsString), List(JS_SUFFIX))
.map(_.getFileName.toString)
transpiledJsFiles shouldBe List("index.js")
}