Merge pull request #10 from ShiftLeftSecurity/max/workflowOSs

Configured different OS for PR tests

.github/workflows/pr.yml

@@ -2,21 +2,24 @@ name: PR
 on: pull_request
 jobs:
   pr:
-    runs-on: ubuntu-20.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, windows-2019, macos-10.15, macos-11]
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Set up JDK 11
-        uses: actions/setup-java@v2
+      - name: Set up JDK 8 and SBT
+        uses: olafurpg/setup-scala@v11
         with:
-          distribution: 'adopt'
-          java-version: 8
+          java-version: [email protected]
       - uses: actions/cache@v2
         with:
           path: |
             ~/.sbt
             ~/.coursier
           key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }}
       - name: Compile and run tests
-        run: sbt clean test
+        run: sbt -v -Dfile.encoding=UTF-8 "clean; +test;"
+        shell: bash

build.sbt

@@ -4,8 +4,6 @@ val gitCommitString = SettingKey[String]("gitSha")
 
 enablePlugins(JavaAppPackaging, GitVersioning, BuildInfoPlugin)
 
-Test / test := ((Test / test) dependsOn (cfgIntegrationTests / Test / test)).value
-
 lazy val Fast = config("fast").extend(Test)
 configs(Fast)
 inConfig(Fast)(Defaults.testTasks)
@@ -122,11 +120,5 @@ lazy val js2cpg = (project in file(".")).settings(
   buildInfoPackage := "io.shiftleft.js2cpg.core"
 )
 
-lazy val cfgIntegrationTests = project.settings(
-  commonSettings,
-  publish / skip := true,
-  Test / test := ((Test / test) dependsOn (js2cpg / stage)).value
-)
-
 Universal / packageName := name.value
 Universal / topLevelDirectory := None

src/main/scala/io/shiftleft/js2cpg/core/Config.scala

@@ -74,7 +74,7 @@ case class Config(srcDir: String = "",
     val slIngoreFilePath = Paths.get(srcDir, Config.SL_IGNORE_FILE)
     val result = Using(FileUtils.bufferedSourceFromFile(slIngoreFilePath)) { bufferedSource =>
       val content = FileUtils.contentFromBufferedSource(bufferedSource)
-      content.split("\n").toSeq.map(createPathForIgnore)
+      content.split(System.lineSeparator()).toSeq.map(createPathForIgnore)
     }
 
     result match {

src/main/scala/io/shiftleft/js2cpg/io/EmScriptenCleaner.scala

@@ -14,7 +14,7 @@ object EmScriptenCleaner {
     val endIndex   = lines.indexWhere(EMSCRIPTEN_END_FUNCS.matches)
     if (startIndex != -1 && endIndex != -1 && endIndex > startIndex) {
       (lines.slice(0, startIndex) ++
-        Seq.fill(endIndex - startIndex - 1)("\n") ++
+        Seq.fill(endIndex - startIndex - 1)(System.lineSeparator()) ++
         lines.slice(endIndex + 1, lines.length)).iterator
     } else {
       lines.iterator

src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala

@@ -5,14 +5,14 @@ import better.files.File
 import java.io.Reader
 import java.math.BigInteger
 import java.nio.charset.{CharsetDecoder, CodingErrorAction}
-import java.nio.file.{FileVisitResult, Files, Path, SimpleFileVisitor}
+import java.nio.file.{Files, FileVisitResult, Path, SimpleFileVisitor}
 import io.shiftleft.js2cpg.core.Config
 import io.shiftleft.js2cpg.io.FileDefaults._
 import org.slf4j.LoggerFactory
 
 import java.nio.file.attribute.BasicFileAttributes
 import scala.collection.concurrent.TrieMap
-import scala.collection.{SortedMap, mutable}
+import scala.collection.{mutable, SortedMap}
 import scala.io.{BufferedSource, Codec, Source}
 import scala.jdk.CollectionConverters._
 
@@ -33,7 +33,7 @@ object FileUtils {
 
   /**
     * Cleans the given path as String and removes unwanted elements that
-    * occure during transpilation on the Windows platform and/or CI environments.
+    * occur during transpilation on the Windows platform and/or CI environments.
     */
   def cleanPath(sourceFileName: String): String = {
     val replaceDots = sourceFileName.replace("../", "")

src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala

@@ -81,7 +81,7 @@ object JavaScriptParser {
     val lines        = jsSource.source.getContent.toString.linesIterator.toSeq
     val replaceIndex = lines.lastIndexWhere(l => importRegex.matches(l.trim())) + 1
     val (head, rest) = lines.splitAt(replaceIndex)
-    val fixedCode    = (head ++ nodeJsFixWrapper(rest)).mkString("\n")
+    val fixedCode    = (head ++ nodeJsFixWrapper(rest)).mkString(System.lineSeparator())
     val source       = Source.sourceFor(jsSource.filePath, fixedCode)
     source.toJsSource(jsSource.srcDir, jsSource.projectDir)
   }

src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala

@@ -95,9 +95,12 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) {
         // on MacOS the path to the tmp dir is already there
         File("/") / replacedName
       } else {
-        // on all other OS we have to prepend it
-        if (srcDir.pathAsString.startsWith(tmpDir)) File(tmpDir) / replacedName
-        else File("/") / replacedName
+        // on all other OS we have to prepend it with special handling for Windows:
+        if (replacedName.contains("AppData/Local/Temp")) {
+          srcDir.root / "Users" / replacedName
+        } else {
+          File(tmpDir) / replacedName
+        }
       }
       srcFilePath
   }

src/main/scala/io/shiftleft/js2cpg/preprocessing/EjsTranspiler.scala

@@ -81,7 +81,7 @@ class EjsTranspiler(override val config: Config, override val projectPath: Path)
         result.append(extractedJsCode)
       }
     }
-    (result.mkString("\n"), sourceMap)
+    (result.mkString(System.lineSeparator()), sourceMap)
   }
 
   override def shouldRun(): Boolean = config.templateTranspiling && ejsFiles.nonEmpty

cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala → src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgCreationPassTest.scala

@@ -3,24 +3,20 @@ package io.shiftleft.js2cpg.cpg.passes
 import better.files.File
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated._
+import io.shiftleft.js2cpg.core.Report
+import io.shiftleft.passes.IntervalKeyPool
 import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.passes.CfgCreationPass
-import io.shiftleft.semanticcpg.passes.cfgcreation.Cfg.{
-  AlwaysEdge,
-  CaseEdge,
-  CfgEdgeType,
-  FalseEdge,
-  TrueEdge
-}
+import io.shiftleft.semanticcpg.passes.cfgcreation.Cfg._
 import overflowdb.traversal._
 import overflowdb._
+
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 
 import scala.jdk.CollectionConverters._
-import scala.util.{Failure, Success}
 
-class CfgIntegrationTest extends AnyWordSpec with Matchers {
+class CfgCreationPassTest extends AnyWordSpec with Matchers {
 
   "CFG generation for simple fragments" should {
     "have correct structure for block expression" in {
@@ -1290,17 +1286,17 @@ class CfgIntegrationTest extends AnyWordSpec with Matchers {
     }
   }
 
-  private class CfgFixture(code: String) extends IntegrationTestFixture {
+  private class CfgFixture(code: String) {
 
-    private var cpg: Cpg = Cpg.emptyCpg
+    private val cpg: Cpg = Cpg.emptyCpg
 
     File.usingTemporaryDirectory("js2cpgCfgIntegrationTest") { workspace =>
-      cpg = callFrontend(workspace, code) match {
-        case Failure(exception) => fail(exception)
-        case Success(cpg) =>
-          new CfgCreationPass(cpg).createAndApply()
-          cpg
-      }
+      val file = workspace / "test.js"
+      file.write(code)
+      val keyPool   = new IntervalKeyPool(1001, 2000)
+      val filenames = List((file.path, file.parent.path))
+      new AstCreationPass(workspace, filenames, cpg, keyPool, new Report()).createAndApply()
+      new CfgCreationPass(cpg).createAndApply()
     }
 
     private def matchCode(node: Node, code: String): Boolean = {

src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala

@@ -11,7 +11,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers {
         """
         |console.log("Hello");
         |console.log("World!");""".stripMargin
-      val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n")
+      val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator())
       result shouldBe code
     }
 
@@ -21,7 +21,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers {
           |// EMSCRIPTEN_START_FUNCS
           |console.log("Hello");
           |console.log("World!");""".stripMargin
-      val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n")
+      val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator())
       result shouldBe code
     }
 
@@ -31,7 +31,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers {
            |console.log("Hello");
            |console.log("World!");
            |// EMSCRIPTEN_END_FUNCS""".stripMargin
-      val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n")
+      val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator())
       result shouldBe code
     }
 
@@ -42,7 +42,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers {
            |console.log("Hello");
            |console.log("World!");
            |// EMSCRIPTEN_START_FUNCS""".stripMargin
-      val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n")
+      val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator())
       result shouldBe code
     }
 
@@ -66,7 +66,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers {
           |
           |otherCode();
           |foobar();""".stripMargin
-      val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n")
+      val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator())
       result shouldBe expected
     }
   }

src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala

@@ -3,14 +3,16 @@ package io.shiftleft.js2cpg.io
 import better.files.File
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}
-import io.shiftleft.codepropertygraph.generated.{PropertyNames, NodeTypes}
-import io.shiftleft.js2cpg.core.{Js2CpgMain, Js2cpgArgumentsParser}
+import io.shiftleft.codepropertygraph.generated.{NodeTypes, PropertyNames}
+import io.shiftleft.js2cpg.core.{Js2cpgArgumentsParser, Js2CpgMain}
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.wordspec.AnyWordSpec
 import overflowdb._
 import overflowdb.traversal.TraversalSource
 
+import java.util.regex.Pattern
+
 object ExcludeTest {
   private implicit class ToArg(val arg: String) extends AnyVal {
     def toArg: String = s"--$arg"
@@ -22,7 +24,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
   import ExcludeTest._
   import Js2cpgArgumentsParser._
 
-  private val projectUnderTestPath: String = getClass.getResource("/excludes").getPath
+  private val projectUnderTestPath = File(getClass.getResource("/excludes").toURI).pathAsString
 
   private def fileNames(cpg: Cpg): List[String] =
     TraversalSource(cpg.graph).label(NodeTypes.FILE).property(PropertyNames.NAME).toList
@@ -44,7 +46,8 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
             CpgLoaderConfig.withDefaults.withOverflowConfig(
               Config.withDefaults.withStorageLocation(cpgPath)))
 
-      fileNames(cpg) should contain theSameElementsAs expectedFiles
+      fileNames(cpg) should contain theSameElementsAs expectedFiles.map(
+        _.replace("/", java.io.File.separator))
     }
   }
 
@@ -90,7 +93,9 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
        Seq(EXCLUDE_REGEX.toArg, ".*(index|b)\\..*"),
        Set("a.js", "folder/c.js", "foo.bar/d.js")),
       (s"exclude a complete folder with ${EXCLUDE_REGEX.toArg}",
-       Seq(EXCLUDE_REGEX.toArg, ".*/folder/.*"),
+       Seq(
+         EXCLUDE_REGEX.toArg,
+         s".*${Pattern.quote(java.io.File.separator)}folder${Pattern.quote(java.io.File.separator)}.*"),
        Set("index.js", "a.js", "foo.bar/d.js")),
       // --
       // Tests for mixed arguments