feat(cli): add --include-flaky-tests to parsing (#878)

---------

Co-authored-by: Anton Malinskiy <[email protected]>

cli/src/main/kotlin/com/malinskiy/marathon/cli/MarathonStartConfiguration.kt

@@ -27,5 +27,5 @@ fun ParseCommandCliConfiguration.toMarathonStartConfiguration(): MarathonStartCo
         marathonfile = this.marathonfile,
         bugsnagReporting = false,
         analyticsTracking = false,
-        executionCommand = ParseCommand(this.outputFileName),
+        executionCommand = ParseCommand(this.outputFileName, this.includeFlakyTests),
     )

cli/src/main/kotlin/com/malinskiy/marathon/cli/args/CliCommands.kt

@@ -59,10 +59,14 @@ class Parse(
         .file()
         .default(File("Marathonfile"))
     private val parseOutputFileName by option("--output", "-o", help="Output file name in yaml format")
+    private val includeFlakyTests by option("--include-flaky-tests", "-f", help="Include/exclude flaky tests that will have preventive retries according to the current flakinessStrategy")
+        .convert { it.toBoolean() }
+        .default(false)
     override fun run() {
         val parseCommandCliConfiguration = ParseCommandCliConfiguration(
             marathonfile = marathonfile,
-            outputFileName = parseOutputFileName
+            outputFileName = parseOutputFileName,
+            includeFlakyTests = includeFlakyTests,
         )
         starter(parseCommandCliConfiguration)
     }

cli/src/main/kotlin/com/malinskiy/marathon/cli/args/CliConfigurations.kt

@@ -14,5 +14,6 @@ data class MarathonRunCommandCliConfiguration(
 
 data class ParseCommandCliConfiguration(
     val marathonfile: File,
-    val outputFileName: String?
+    val outputFileName: String?,
+    val includeFlakyTests: Boolean,
 ) : CliConfiguration()

core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt

@@ -139,19 +139,24 @@ class Marathon(
         val parsedFilteredTests = applyTestFilters(parsedAllTests)
 
         if (executionCommand is ParseCommand) {
+            // Delay potentially querying remote TSDB unless user requested to get flaky tests
+            val flakyTests = if (executionCommand.includeFlakyTests) {
+                prepareTestShard(parsedFilteredTests, analytics).flakyTests.toList()
+            } else emptyList()
             marathonTestParseCommand.execute(
                 tests = parsedFilteredTests,
+                flakyTests = flakyTests,
                 outputFileName = executionCommand.outputFileName
             )
             stopKoin()
             return true
         }
 
+        val shard = prepareTestShard(parsedFilteredTests, analytics)
+
         usageTracker.trackEvent(Event.TestsTotal(parsedAllTests.size))
         usageTracker.trackEvent(Event.TestsRun(parsedFilteredTests.size))
 
-        val shard = prepareTestShard(parsedFilteredTests, analytics)
-
         log.info("Scheduling ${parsedFilteredTests.size} tests")
         log.debug(parsedFilteredTests.joinToString(", ") { it.toTestName() })
         val currentCoroutineContext = coroutineContext

core/src/main/kotlin/com/malinskiy/marathon/config/ExecutionCommands.kt

@@ -3,7 +3,8 @@ package com.malinskiy.marathon.config
 sealed class ExecutionCommand
 
 data class ParseCommand(
-    val outputFileName: String?
+    val outputFileName: String?,
+    val includeFlakyTests: Boolean = false,
 ) : ExecutionCommand()
 
 object MarathonRunCommand : ExecutionCommand()

core/src/main/kotlin/com/malinskiy/marathon/execution/command/parse/MarathonTestParseCommand.kt

@@ -45,8 +45,9 @@ class MarathonTestParseCommand(private val outputDir: File) {
         )
     }
 
-    fun execute(tests: List<Test>, outputFileName: String?) {
-        val parseResult = ParseCommandResult(tests)
+    fun execute(tests: List<Test>, flakyTests: List<Test>, outputFileName: String?) {
+        val parsedFlakyTests = flakyTests.ifEmpty { null }
+        val parseResult = ParseCommandResult(tests, parsedFlakyTests)
         val res = mapper.writeValueAsString(parseResult)
 
         log.info { "Parse execute mode. Result" }

core/src/main/kotlin/com/malinskiy/marathon/execution/command/parse/ParseCommandResult.kt

@@ -3,5 +3,6 @@ package com.malinskiy.marathon.execution.command.parse
 import com.malinskiy.marathon.test.Test
 
 data class ParseCommandResult(
-    val tests: List<Test>
+    val tests: List<Test>,
+    val flakyTests: List<Test>? = null,
 )

core/src/test/kotlin/com/malinskiy/marathon/execution/command/parse/MarathonTestParseCommandTest.kt

@@ -26,7 +26,7 @@ class MarathonTestParseCommandTest {
     fun `test parse command should create a file with zero tests`() {
         val testList = listOf<com.malinskiy.marathon.test.Test>()
 
-        marathonTestParseCommand.execute(testList, RESULT_FILE_NAME)
+        marathonTestParseCommand.execute(testList, emptyList(), RESULT_FILE_NAME)
 
         val file = File(tempRootDir.absolutePath, RESULT_FILE_NAME_WITH_EXT)
         file.exists() shouldBe true
@@ -40,7 +40,7 @@ class MarathonTestParseCommandTest {
         val test1 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method1", emptyList())
         val testList = listOf(test1)
 
-        marathonTestParseCommand.execute(testList, RESULT_FILE_NAME)
+        marathonTestParseCommand.execute(testList, emptyList(), RESULT_FILE_NAME)
 
         val file = File(tempRootDir.absolutePath, RESULT_FILE_NAME_WITH_EXT)
         file.exists() shouldBe true
@@ -56,14 +56,34 @@ class MarathonTestParseCommandTest {
         val test3 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method3", emptyList())
         val testList = listOf(test1, test2, test3)
 
-        marathonTestParseCommand.execute(testList, RESULT_FILE_NAME)
+        marathonTestParseCommand.execute(testList, emptyList(), RESULT_FILE_NAME)
 
         val file = File(tempRootDir.absolutePath, RESULT_FILE_NAME_WITH_EXT)
         file.exists() shouldBe true
 
         val content = file.readText()
         content.trimIndent() shouldBeEqualTo severalTestsResult
     }
+
+    @Test
+    fun `test parse command should create a file with several tests and flaky tests`() {
+        val test1 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method1", emptyList())
+        val test2 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method2", emptyList())
+        val test3 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method3", emptyList())
+        val testList = listOf(test1, test2, test3)
+
+        val flakyTest1 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method1", emptyList())
+        val flakyTest2 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method2", emptyList())
+        val flakyTestList = listOf(flakyTest1, flakyTest2)
+
+        marathonTestParseCommand.execute(testList, flakyTestList, RESULT_FILE_NAME)
+
+        val file = File(tempRootDir.absolutePath, RESULT_FILE_NAME_WITH_EXT)
+        file.exists() shouldBe true
+
+        val content = file.readText()
+        content.trimIndent() shouldBeEqualTo severalTestsAndFlakyTestsResult
+    }
 }
 
 private val zeroTestsResult: String = """---
@@ -90,3 +110,27 @@ tests:
   clazz: "SimpleTest"
   method: "method3"
   metaProperties: []""".trimIndent()
+
+private val severalTestsAndFlakyTestsResult: String = """---
+tests:
+- pkg: "com.example"
+  clazz: "SimpleTest"
+  method: "method1"
+  metaProperties: []
+- pkg: "com.example"
+  clazz: "SimpleTest"
+  method: "method2"
+  metaProperties: []
+- pkg: "com.example"
+  clazz: "SimpleTest"
+  method: "method3"
+  metaProperties: []
+flakyTests:
+- pkg: "com.example"
+  clazz: "SimpleTest"
+  method: "method1"
+  metaProperties: []
+- pkg: "com.example"
+  clazz: "SimpleTest"
+  method: "method2"
+  metaProperties: []""".trimIndent()

docs/runner/intro/execute.md

@@ -37,14 +37,18 @@ Options:
 ## Parse command
 ```shell-session
 foo@bar $ marathon parse -h
-Usage: marathon parse [OPTIONS]
+Usage: marathon parse [<options>]
 
   Print the list of tests without executing them
 
 Options:
-  -m, --marathonfile PATH  Marathonfile file path
-  -o, --output TEXT        Output file name in yaml format
-  -h, --help               Show this message and exit
+  -m, --marathonfile=<path>  Marathonfile file path
+  -o, --output=<text>        Output file name in yaml format
+  -f, --include-flaky-tests=<value>
+                             Include/exclude flaky tests that will have
+                             preventive retries according to the current
+                             flakinessStrategy
+  -h, --help                 Show this message and exit
 ```
 
 ## Version command