run docstrings tests with mocha

.github/workflows/ci.yml

@@ -93,7 +93,6 @@ jobs:
             # Run the slower tasks on the fastest runner only
             build_playground: true
             benchmarks: true
-            docstring_tests: true
           - os: buildjet-2vcpu-ubuntu-2204-arm # ARM
             ocaml_compiler: ocaml-variants.5.2.1+options,ocaml-option-static
             upload_binaries: true
@@ -313,11 +312,6 @@ jobs:
       - name: Run tests
         run: node scripts/test.js -all
 
-      - name: Docstrings tests
-        if: matrix.docstring_tests
-        # Ignore functions that are not available on Node 18
-        run: node tests/docstrings_examples/DocTest.res.mjs --ignore-runtime-tests "Array.toReversed, Array.toSorted, Promise.withResolvers, Set.union, Set.isSupersetOf, Set.isSubsetOf, Set.isDisjointFrom, Set.intersection, Set.symmetricDifference, Set.difference"
-
       - name: Check for diffs in tests folder
         run: git diff --ignore-cr-at-eol --exit-code tests
 

scripts/test.js

@@ -132,18 +132,54 @@ async function runTests() {
       console.log(`Skipping docstrings tests on ${process.platform}`);
     } else {
       console.log("Running runtime docstrings tests");
+
+      const generated_mocha_test_res = path.join(
+        "tests",
+        "docstrings_examples",
+        "generated_mocha_test.res",
+      );
+
+      // Remove `generated_mocha_test.res` if file exists
+      if (fs.existsSync(generated_mocha_test_res)) {
+        console.log(`Removing ${generated_mocha_test_res}`);
+        fs.unlinkSync(generated_mocha_test_res);
+      }
+
+      cp.execSync(`${rescript_exe} build`, {
+        cwd: path.join(__dirname, "..", "tests/docstrings_examples"),
+        stdio: [0, 1, 2],
+      });
+
+      // Generate rescript file with all tests `generated_mocha_test.res`
+      cp.execSync(
+        `node ${path.join("tests", "docstrings_examples", "DocTest.res.mjs")}`,
+        {
+          cwd: path.join(__dirname, ".."),
+          stdio: [0, 1, 2],
+        },
+      );
+
+      // Build again to check if generated_mocha_test.res has syntax or type erros
       cp.execSync(`${rescript_exe} build`, {
         cwd: path.join(__dirname, "..", "tests/docstrings_examples"),
         stdio: [0, 1, 2],
       });
-      // Ignore some tests not supported by node v18
-      // cp.execSync(
-      //   `node ${path.join("tests", "docstrings_examples", "DocTest.res.mjs")} --ignore-runtime-tests "Array.toReversed, Array.toSorted, Promise.withResolvers, Set.union, Set.isSupersetOf, Set.isSubsetOf, Set.isDisjointFrom, Set.intersection, Set.symmetricDifference, Set.difference"`,
-      //   {
-      //     cwd: path.join(__dirname, ".."),
-      //     stdio: [0, 1, 2],
-      //   },
-      // );
+
+      // Format generated_mocha_test.res
+      console.log("Formatting generated_mocha_test.res");
+      cp.execSync(`./cli/rescript format ${generated_mocha_test_res}`, {
+        cwd: path.join(__dirname, ".."),
+        stdio: [0, 1, 2],
+      });
+
+      console.log("Run mocha test");
+      cp.execSync(
+        `npx mocha ${path.join("tests", "docstrings_examples", "generated_mocha_test.res.mjs")}`,
+        {
+          cwd: path.join(__dirname, ".."),
+          stdio: [0, 1, 2],
+        },
+      );
     }
   }
 }

tests/docstrings_examples/.gitignore

@@ -0,0 +1,2 @@
+generated_mocha_test.res
+generated_mocha_test.res.mjs

tests/docstrings_examples/DocTest.res

@@ -9,24 +9,34 @@ type example = {
   docstrings: array<string>,
 }
 
-type error =
-  | ReScriptError(string)
-  | RuntimeError({rescript: string, js: string, error: string})
-
-let bscBin = Path.join(["cli", "bsc"])
-
-let parsed = Util.parseArgs({
-  args: Process.argv->Array.sliceToEnd(~start=2),
-  options: dict{"ignore-runtime-tests": {Util.type_: "string"}},
-})
-
-let ignoreRuntimeTests = switch parsed.values->Dict.get("ignore-runtime-tests") {
-| Some(v) =>
-  v
-  ->String.split(",")
-  ->Array.map(s => s->String.trim)
-| None => []
-}
+// Only major version
+let nodeVersion =
+  Process.version
+  ->String.replace("v", "")
+  ->String.split(".")
+  ->Array.get(0)
+  ->Option.getExn(~message="Failed to find major version of Node")
+  ->Int.fromString
+  ->Option.getExn(~message="Failed to convert node version to Int")
+
+let ignoreRuntimeTests = [
+  (
+    // Ignore some tests not supported by node v18
+    18,
+    [
+      "Array.toReversed",
+      "Array.toSorted",
+      "Promise.withResolvers",
+      "Set.union",
+      "Set.isSupersetOf",
+      "Set.isSubsetOf",
+      "Set.isDisjointFrom",
+      "Set.intersection",
+      "Set.symmetricDifference",
+      "Set.difference",
+    ],
+  ),
+]
 
 let getOutput = buffer =>
   buffer
@@ -158,142 +168,85 @@ let extractExamples = async () => {
   examples
 }
 
-let compileTest = async (~code) => {
-  // NOTE: warnings argument (-w) should be before eval (-e) argument
-  let args = ["-w", "-3-109-44", "-e", code]
-  let {stderr, stdout} = await SpawnAsync.run(~command=bscBin, ~args)
+let main = async () => {
+  let examples = await extractExamples()
 
-  stderr->Array.length > 0 ? Error(stderr->getOutput) : Ok(stdout->getOutput)
-}
+  examples->Array.sort((a, b) => String.compare(a.id, b.id))
 
-let compileExamples = async examples => {
-  Console.log(`Compiling ${examples->Array.length->Int.toString} examples from docstrings...`)
+  let dict = dict{}
 
-  let compiled = []
-  let compilationErrors = []
+  examples->Array.forEach(cur => {
+    let modulePath = cur.id->String.split(".")
 
-  await examples->ArrayUtils.forEachAsyncInBatches(~batchSize, async example => {
-    // let id = example.id->String.replaceAll(".", "__")
-    let rescriptCode = example->getCodeBlocks
+    let id =
+      modulePath
+      ->Array.slice(~start=0, ~end=Array.length(modulePath) - 1)
+      ->Array.join(".")
 
-    switch await compileTest(~code=rescriptCode) {
-    | Ok(jsCode) => compiled->Array.push((example, rescriptCode, jsCode))
-    | Error(err) => compilationErrors->Array.push((example, ReScriptError(err)))
+    let previous = switch dict->Dict.get(id) {
+    | Some(p) => p
+    | None => []
     }
-  })
-
-  (compiled, compilationErrors)
-}
-
-let runTest = async code => {
-  let {stdout, stderr, code: exitCode} = await SpawnAsync.run(
-    ~command="node",
-    ~args=["-e", code, "--input-type", "commonjs"],
-    ~options={cwd: Process.cwd(), timeout: 2000},
-  )
-
-  // Some expressions, like `console.error("error")` are printed to stderr,
-  // but exit code is 0
-  let std = switch exitCode->Null.toOption {
-  | Some(exitCode) if exitCode == 0.0 && Array.length(stderr) > 0 => stderr->Ok
-  | Some(exitCode) if exitCode == 0.0 => stdout->Ok
-  | None | Some(_) => Error(Array.length(stderr) > 0 ? stderr : stdout)
-  }
 
-  switch std {
-  | Ok(buf) => Ok(buf->getOutput)
-  | Error(buf) => Error(buf->getOutput)
-  }
-}
-
-let runExamples = async compiled => {
-  Console.log(`Running ${compiled->Array.length->Int.toString} compiled examples...`)
-
-  let tests = compiled->Array.filter((({id}, _, _)) => !(ignoreRuntimeTests->Array.includes(id)))
-
-  let runtimeErrors = []
-  await tests->ArrayUtils.forEachAsyncInBatches(~batchSize, async compiled => {
-    let (example, rescriptCode, jsCode) = compiled
-
-    switch await runTest(jsCode) {
-    | Ok(_) => ()
-    | Error(error) =>
-      let runtimeError = RuntimeError({rescript: rescriptCode, js: jsCode, error})
-      runtimeErrors->Array.push((example, runtimeError))
-    }
+    dict->Dict.set(id, Array.concat([cur], previous))
   })
 
-  runtimeErrors
-}
+  let output = []
 
-let indentOutputCode = code => {
-  let indent = String.repeat(" ", 2)
-
-  code
-  ->String.split("\n")
-  ->Array.map(s => `${indent}${s}`)
-  ->Array.join("\n")
-}
+  dict->Dict.forEachWithKey((examples, key) => {
+    let codeExamples = examples->Array.filterMap(example => {
+      let ignoreExample = Array.find(
+        ignoreRuntimeTests,
+        ((version, tests)) => {
+          nodeVersion === version && Array.includes(tests, example.id)
+        },
+      )->Option.isSome
 
-let printErrors = errors => {
-  errors->Array.forEach(((example, errors)) => {
-    let red = s => `\x1B[1;31m${s}\x1B[0m`
-    let cyan = s => `\x1b[36m${s}\x1b[0m`
-    let kind = switch example.kind {
-    | "moduleAlias" => "module alias"
-    | other => other
-    }
-
-    let a = switch errors {
-    | ReScriptError(error) =>
-      let err =
-        error
-        ->String.split("\n")
-        // Drop line of filename
-        ->Array.filterWithIndex((_, i) => i !== 2)
-        ->Array.join("\n")
-
-      `${"error"->red}: failed to compile examples from ${kind} ${example.id->cyan}
-${err}`
-    | RuntimeError({rescript, js, error}) =>
-      let indent = String.repeat(" ", 2)
-
-      `${"runtime error"->red}: failed to run examples from ${kind} ${example.id->cyan}
-
-${indent}${"ReScript"->cyan}
-
-${rescript->indentOutputCode}
-
-${indent}${"Compiled Js"->cyan}
-
-${js->indentOutputCode}
-
-${indent}${"stacktrace"->red}
-
-${error->indentOutputCode}
-`
+      switch ignoreExample {
+      | true =>
+        Console.warn(
+          `Ignoring ${example.id} tests. Not supported by Node ${nodeVersion->Int.toString}`,
+        )
+        None
+      | false =>
+        let code = getCodeBlocks(example)
+
+        switch String.length(code) === 0 {
+        | true => None
+        | false =>
+          // Let's add the examples inside a Test module because some examples
+          // have type definitions that are not supported inside a block.
+          // Also add unit type `()`
+          Some(
+            `test("${example.id->String.split(".")->Array.at(-1)->Option.getExn}", () => {
+  module Test = {
+    ${code}
+  }
+  ()
+})`,
+          )
+        }
+      }
+    })
+
+    switch Array.length(codeExamples) > 0 {
+    | true =>
+      let content = `describe("${key}", () => {
+${codeExamples->Array.join("\n")}
+ })`
+      output->Array.push(content)
+    | false => ()
     }
-
-    Process.stderrWrite(a)
   })
-}
 
-let main = async () => {
-  let examples = await extractExamples()
-  let (compiled, compilationErrors) = await compileExamples(examples)
-  let runtimeErrors = await runExamples(compiled)
+  let dirname = url->URL.fileURLToPath->Path.dirname
+  let filepath = Path.join([dirname, "generated_mocha_test.res"])
+  let fileContent = `open Mocha
+@@warning("-32-34-60-37-109-3-44")
 
-  let allErrors = Array.concat(runtimeErrors, compilationErrors)
+${output->Array.join("\n")}`
 
-  if allErrors->Array.length > 0 {
-    printErrors(allErrors)
-    1
-  } else {
-    Console.log("All examples passed successfully")
-    0
-  }
+  await Fs.writeFile(filepath, fileContent)
 }
 
-let exitCode = await main()
-
-Process.exit(exitCode)
+let () = await main()