[nix] migrate to new CI

.github/workflows/pr.yml

@@ -42,7 +42,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: "Build verilator emulator"
         run: |
-          nix build '.#t1.${{ matrix.config }}.ip.emu' -L --no-link --cores 64
+          nix build '.#t1.${{ matrix.config }}.ip.difftest' -L --no-link --cores 64
       - name: "Build all testcases"
         run: |
           # Build testcases with vlen 1024 and vlen 4096
@@ -81,7 +81,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: "Build verilator emulator with trace"
-        run: nix build '.#t1.${{ matrix.config }}.ip.emu-trace' -L --no-link --cores 64
+        run: nix build '.#t1.${{ matrix.config }}.ip.difftest-trace' -L --no-link --cores 64
 
   test-emit:
     if: '! github.event.pull_request.draft'
@@ -163,10 +163,10 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: "Run testcases"
         run: |
-          nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}"
+          nix run ".#ci-helper" -- runTests --jobs "${{ matrix.jobs }}"
       - name: "Run OM tests"
         run: |
-          nix develop -c t1-helper runOMTests --jobs "${{ matrix.jobs }}"
+          nix run ".#ci-helper" -- runOMTests --jobs "${{ matrix.jobs }}"
 
   report:
     name: "Report CI result"
@@ -183,9 +183,8 @@ jobs:
           ref: ${{ github.head_ref }}
       - name: "Print step summary"
         run: |
-          nix run ".#ci-helper" -- postCI --failed-test-file-path ./failed-test.md --cycle-update-file-path ./cycle-update.md
-          cat ./failed-test.md >> $GITHUB_STEP_SUMMARY
-          echo >> $GITHUB_STEP_SUMMARY
+          nix run ".#ci-helper" -- postCI --failed-tests-file-path ./failed-tests.md --cycle-update-file-path ./cycle-update.md
+          cat ./failed-tests.md >> $GITHUB_STEP_SUMMARY
           cat ./cycle-update.md >> $GITHUB_STEP_SUMMARY
       - name: "Commit cycle updates"
         run: |

nix/overlay.nix

@@ -24,8 +24,6 @@ rec {
   circt-full = final.callPackage ./pkgs/circt-full.nix { };
   rvv-codegen = final.callPackage ./pkgs/rvv-codegen.nix { };
   add-determinism = final.callPackage ./pkgs/add-determinism { }; # faster strip-undetereminism
-  # difftest simulator
-  t1-simulator = final.callPackage ../difftest/t1-simulator { };
 
   mill = let jre = final.jdk21; in
     (prev.mill.override { inherit jre; }).overrideAttrs (_: {

nix/t1/default.nix

@@ -51,7 +51,7 @@ lib.makeScope newScope
         elaborateConfigJson = configPath;
         elaborateConfig = builtins.fromJSON (lib.readFile configPath);
 
-        cases = innerSelf.callPackage ../../tests { ip-emu = ip.emu; };
+        cases = innerSelf.callPackage ../../tests { difftest = ip.difftest; difftest-trace = ip.difftest-trace; };
 
         # for the convenience to use x86 cases on non-x86 machines, avoiding the extra build time
         cases-x86 =
@@ -75,10 +75,14 @@ lib.makeScope newScope
           emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; };
           emu-rtl = innerSelf.callPackage ./rtl.nix { mlirbc = emu-mlirbc; };
 
+          verilated = innerSelf.callPackage ./verilated.nix { rtl = emu-rtl; };
+          verilated-trace = innerSelf.callPackage ./verilated.nix { rtl = emu-rtl; enable-trace = true; };
+
           emu = innerSelf.callPackage ./ipemu.nix { rtl = ip.emu-rtl; stdenv = moldStdenv; };
           emu-trace = innerSelf.callPackage ./ipemu.nix { rtl = emu-rtl; stdenv = moldStdenv; do-trace = true; };
 
-          t1-simulator = innerSelf.callPackage ../../difftest/t1-simulator/default.nix { rtl = innerSelf.ip.emu-rtl; };
+          difftest = innerSelf.callPackage ../../difftest/default.nix { inherit verilated; };
+          difftest-trace = innerSelf.callPackage ../../difftest/default.nix { verilated = verilated-trace; };
         };
 
         subsystem = rec {

nix/t1/verilated.nix

@@ -0,0 +1,61 @@
+{ lib
+, stdenv
+, configName
+, rtl
+, verilator
+, enable-trace ? false
+, zlib
+}:
+stdenv.mkDerivation {
+  name = "${configName}-verilated";
+
+  src = rtl;
+
+  nativeBuildInputs = [ verilator ];
+
+  # zlib is required for Rust to link against
+  propagatedBuildInputs = [ zlib ];
+
+  buildPhase = ''
+    runHook preBuild
+
+    echo "[nix] running verilator"
+    verilator \
+      ${lib.optionalString enable-trace "--trace-fst"} \
+      --timing \
+      --threads 8 \
+      -O1 \
+      --cc TestBench
+
+    echo "[nix] building verilated C lib"
+
+    # backup srcs
+    mkdir -p $out/share
+    cp -r obj_dir $out/share/verilated_src
+
+    # We can't use -C here because VTestBench.mk is generated with relative path
+    cd obj_dir
+    make -j "$NIX_BUILD_CORES" -f VTestBench.mk libVTestBench
+
+    runHook postBuild
+  '';
+
+  passthru = {
+    inherit enable-trace;
+  };
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir -p $out/include $out/lib
+    cp *.h $out/include
+    cp *.a $out/lib
+
+    runHook postInstall
+  '';
+
+  # nix fortify hardening add `-O2` gcc flag,
+  # we'd like verilator to controll optimization flags, so disable it.
+  # `-O2` will make gcc build time in verilating extremely long
+  hardeningDisable = [ "fortify" ];
+}

script/ci/src/Main.scala

@@ -161,85 +161,102 @@ object Main:
         )
 
         val testResultPath =
-          os.Path(nixResolvePath(s".#t1.$config.cases.$caseName.emu-result"))
+          try
+            os.Path(
+              nixResolvePath(
+                s".#t1.$config.cases.$caseName.emu-result.with-offline"
+              )
+            )
+          catch
+            case _ =>
+              Logger.error(s"Emulation for config $config, case $caseName fail")
+              println("-" * 50)
+              println(
+                os.proc(
+                  "nix",
+                  "log",
+                  s".#t1.$config.cases.$caseName.emu-result"
+                ).call()
+                  .out
+              )
+              println("-" * 50)
+              Logger.fatal("Got error from emulation, exiting CI")
+
+        Logger.info("Checking RTL event with offline difftest")
         val testSuccess =
-          os.read(testResultPath / "emu-success").trim().toInt == 1
-
+          os.read(testResultPath / "offline-check-status").trim() == "0"
         if !testSuccess then
-          Logger.error(s"Test case $testName failed")
-          val err = os.read(testResultPath / "emu.log")
-          Logger.error(s"Detail error: $err")
-
-        Logger.info("Running difftest")
-        val handle = os.proc(
-          "nix",
-          "run",
-          ".#t1-helper",
-          "--",
-          "difftest",
-          "--config",
-          config,
-          "--case-attr",
-          caseName,
-          "--log-level",
-          "ERROR"
-        ).call(stdout = os.Inherit, stderr = os.Inherit, check = false)
-        val diffTestSuccess = handle.exitCode == 0
-        if !diffTestSuccess then
-          Logger.error("difftest run failed")
-
-        if diffTestSuccess != testSuccess then
-          Logger.fatal(
-            "Got different online and offline difftest result, please check this test manually. CI aborted."
-          )
-
-        if !testSuccess then allFailedTest :+ s"t1.$config.cases.$caseName"
+          Logger.error(s"Offline check for $caseName ($config) failed")
+          allFailedTest :+ s"t1.$config.cases.$caseName"
         else allFailedTest
     end findFailedTests
 
     val failedTests = findFailedTests()
     if failedTests.isEmpty then Logger.info(s"All tests passed")
+    else if dontBail.value then
+      Logger.error(
+        s"${BOLD}${failedTests.length} tests failed${RESET}"
+      )
     else
-      val listOfFailJobs =
-        failedTests.map(job => s"* $job").appended("").mkString("\n")
-      val failedJobsWithError = failedTests
-        .map(testName =>
-          val testResult = os.Path(nixResolvePath(s".#$testName.emu-result"))
-          val emuLog = os.read(testResult / "emu.log")
-          if emuLog.nonEmpty then
-            s"* $testName\n     >>> ERROR SUMMARY <<<\n${emuLog}"
-          else
-            s"* $testName\n     >>> OTHER ERROR   <<<\n${os.read(testResult / "emu.journal")}"
-        )
-        .appended("")
-        .mkString("\n")
+      Logger.fatal(
+        s"${BOLD}${failedTests.length} tests failed${RESET}"
+      )
+  end runTests
 
-      if dontBail.value then
-        Logger.error(
-          s"${BOLD}${failedTests.length} tests failed${RESET}:\n${failedJobsWithError}"
+  @main
+  def runOMTests(
+      jobs: String
+  ): Unit =
+    val configs = jobs.split(";").map(_.split(",")(0))
+    configs.distinct.foreach: config =>
+      Seq("omreader", "emu-omreader").foreach: target =>
+        val command = Seq(
+          "nix",
+          "run",
+          s".#t1.$config.ip.$target",
+          "--",
+          "run",
+          "--dump-methods"
         )
-      else
-        Logger.fatal(
-          s"${BOLD}${failedTests.length} tests failed${RESET}:\n${failedJobsWithError}"
+        println("\n")
+        Logger.info(
+          s"Running OM test with command $BOLD'${command.mkString(" ")}'$RESET"
         )
-  end runTests
+        val outputs = os.proc(command).call().out.trim()
+        Logger.trace(s"Outputs:\n${outputs}")
+
+        Seq("vlen =", "dlen =").foreach: keyword =>
+          if outputs.contains(keyword) then
+            Logger.info(
+              s"Keyword $BOLD'$keyword'$RESET found - ${GREEN}Pass!$RESET"
+            )
+          else
+            Logger.fatal(
+              s"Keyword $BOLD'$keyword'$RESET not found - ${RED}Fail!$RESET"
+            )
+  end runOMTests
 
   // PostCI do the below four things:
   //   * read default.json at .github/cases/$config/default.json
   //   * generate case information for each entry in default.json (cycle, run success)
-  //   * collect and report failed tests
   //   * collect and report cycle update
   @main
   def postCI(
       @arg(
-        name = "failed-test-file-path",
-        doc = "specify the failed test markdown file output path"
+        name = "failed-tests-file-path",
+        doc = "specify the failed tests markdown file output path"
       ) failedTestsFilePath: String,
       @arg(
         name = "cycle-update-file-path",
         doc = "specify the cycle update markdown file output path"
       ) cycleUpdateFilePath: String
   ) =
+    val failedTestsFile = os.Path(failedTestsFilePath, os.pwd)
+    os.write.over(failedTestsFile, "## Failed Tests\n")
+
+    val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd)
+    os.write.over(cycleUpdateRecordFile, "## Cycle Update\n")
+
     os.walk(os.pwd / ".github" / "cases")
       .filter(_.last == "default.json")
       .foreach: file =>
@@ -250,20 +267,18 @@ object Main:
         val emuResultPath =
           os.Path(nixResolvePath(s".#t1.$config.cases._allEmuResult"))
 
-        Logger.info("Collecting failed cases")
-        val failedCases = os
-          .walk(emuResultPath)
-          .filter(path => path.last == "emu-success")
-          .filter(path => os.read(path) == "0")
-          .map(path => path.segments.toSeq.reverse.drop(1).head)
-          .map(caseName => s"* `.#t1.${config}.cases.${caseName}`")
-        val failedTestsRecordFile = os.Path(failedTestsFilePath, os.pwd)
-        os.write.over(failedTestsRecordFile, "## Failed tests\n")
-        os.write.append(failedTestsRecordFile, failedCases)
+        Logger.info("Collecting failed tests")
+        os.walk(emuResultPath)
+          .filter(path => path.last == "offline-check-status")
+          .filter(path => os.read(path).trim() != "0")
+          .map(path => {
+            val caseName = path.segments.toSeq.reverse.drop(1).head
+            val journal = os.read(path / os.up / "offline-check-journal")
+            os.write.append(failedTestsFile, s"* ${config} - ${caseName}\n")
+            os.write.append(failedTestsFile, s"```text\n${journal}\n```\n")
+          })
 
         Logger.info("Collecting cycle update info")
-        val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd)
-        os.write.over(cycleUpdateRecordFile, "## Cycle Update\n")
         val perfCycleRegex = raw"total_cycles:\s(\d+)".r
         val allCycleUpdates = os
           .walk(emuResultPath)
@@ -280,12 +295,13 @@ object Main:
           .map:
             case (caseName, newCycle, oldCycle) =>
               cycleRecord(caseName) = newCycle
-              if oldCycle == -1 then s"* 🆕 ${caseName}: NaN -> ${newCycle}"
+              if oldCycle == -1 then s"* 🆕 ${caseName}($config): NaN -> ${newCycle}"
               else if oldCycle > newCycle then
-                s"* 🚀 $caseName: $oldCycle -> $newCycle"
-              else s"* 🐢 $caseName: $oldCycle -> $newCycle"
+                s"* 🚀 $caseName($config): $oldCycle -> $newCycle"
+              else s"* 🐢 $caseName($config): $oldCycle -> $newCycle"
 
         os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n"))
+        os.write.append(cycleUpdateRecordFile, "\n")
 
         os.write.over(file, ujson.write(cycleRecord, indent = 2))
   end postCI