-
Notifications
You must be signed in to change notification settings - Fork 91
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for Hedgehog property testing #78
Changes from all commits
48026cb
9537fa6
5a50759
b17edc2
11ab263
2c7e56f
5609ed0
13d17eb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
package munit | ||
|
||
import hedgehog.core.PropertyConfig | ||
|
||
case class HedgehogConfig(config: PropertyConfig) extends Tag("HedgehogConfig") |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
package munit | ||
|
||
import scala.util.control.NoStackTrace | ||
|
||
import hedgehog.core._ | ||
import hedgehog.runner.Test | ||
|
||
class HedgehogFailException(message: String, val report: Report, val seed: Long) | ||
extends Exception(message) | ||
with NoStackTrace | ||
|
||
object HedgehogFailException { | ||
|
||
def fromReport(report: Report, seed: Long): Option[HedgehogFailException] = { | ||
report.status match { | ||
case OK => | ||
None | ||
case Failed(shrinks, log) => | ||
val coverage = Test.renderCoverage(report.coverage, report.tests) | ||
val message = render( | ||
s"Falsified after ${report.tests.value} passed tests and ${shrinks.value} shrinks using seed ${seed}", | ||
log.map(renderLog) ++ coverage | ||
) | ||
Some(new HedgehogFailException(message, report, seed)) | ||
case GaveUp => | ||
val coverage = Test.renderCoverage(report.coverage, report.tests) | ||
val message = render( | ||
s"Gave up after ${report.tests.value} passed tests using seed value $seed. ${report.discards.value} were discarded", | ||
coverage | ||
) | ||
Some(new HedgehogFailException(message, report, seed)) | ||
} | ||
} | ||
|
||
private def render(msg: String, extras: List[String]): String = | ||
(msg :: extras.map(e => "> " + e)).mkString("\n") | ||
|
||
// From Hedgehog, but customized to *not* include the stack trace and instead print exception toString | ||
private def renderLog(log: Log): String = { | ||
log match { | ||
case ForAll(name, value) => | ||
s"${name.value}: $value" | ||
case Info(value) => | ||
value | ||
case Error(e) => | ||
e.toString | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,95 @@ | ||
package munit | ||
|
||
import hedgehog._ | ||
import hedgehog.core.{PropertyConfig, PropertyT, Seed} | ||
import hedgehog.predef.Monad | ||
|
||
import munit.internal.Compat._ | ||
|
||
/** | ||
* Provides the ability to write property based tests using the Hedgehog library. | ||
* | ||
* Properties are defined by calling one of the various `property` or `propertyF` | ||
* methods and passing a `PropertyT[Result]` or `PropertyT[F[Result]]` respectively. | ||
* For example: | ||
* {{{ | ||
* property("additive identity") { | ||
* Gen.int(Range.linearFrom(0, Int.MinValue, Int.MaxValue)).forAll.map { n => | ||
* assertEquals(n + 0, n) | ||
* } | ||
* } | ||
* }}} | ||
* | ||
* Note both Hedgehog `Result` and munit assertions are supported. | ||
*/ | ||
trait HedgehogSuite extends FunSuite { | ||
|
||
/** | ||
* Provides the default configuration for running property tests. | ||
* This can be overriden to change the default or can be changed on | ||
* a test-by-test basis by using an overload of `property` and `propertyF`. | ||
*/ | ||
protected def hedgehogPropertyConfig: PropertyConfig = PropertyConfig.default | ||
protected def hedgehogSeed: Long = System.currentTimeMillis() | ||
|
||
def property( | ||
name: String | ||
)(prop: PropertyT[Result])(implicit loc: Location): Unit = | ||
property(new TestOptions(name, Set.empty, loc))(prop) | ||
|
||
def property( | ||
options: TestOptions | ||
)(prop: PropertyT[Result])(implicit loc: Location): Unit = { | ||
val config = options.tags.collectFirst { case HedgehogConfig(config) => config }.getOrElse(hedgehogPropertyConfig) | ||
test(options)(check(prop, config, hedgehogSeed)) | ||
} | ||
|
||
def propertyF[F[_]: Monad]( | ||
name: String | ||
)(prop: PropertyT[F[Result]])(implicit loc: Location): Unit = | ||
propertyF(new TestOptions(name, Set.empty, loc))(prop) | ||
|
||
def propertyF[F[_]: Monad]( | ||
options: TestOptions | ||
)(prop: PropertyT[F[Result]])(implicit loc: Location): Unit = { | ||
val config = options.tags.collectFirst { case HedgehogConfig(config) => config }.getOrElse(hedgehogPropertyConfig) | ||
test(options)(checkF(prop, config, hedgehogSeed)) | ||
} | ||
|
||
/** | ||
* Checks the supplied `Property[Result]`, throwing a `HedgehogFailException` | ||
* if the property was falsified. | ||
*/ | ||
private def check( | ||
prop: PropertyT[Result], | ||
config: PropertyConfig = hedgehogPropertyConfig, | ||
seed: Long = hedgehogSeed | ||
)(implicit loc: Location): Unit = { | ||
val report = Property.check(config, prop, Seed.fromLong(seed)) | ||
HedgehogFailException.fromReport(report, seed) match { | ||
case None => () | ||
case Some(t) => throw t | ||
} | ||
} | ||
|
||
/** | ||
* Checks the supplied `PropertyT[F[Result]]`, throwing a `HedgehogFailException` | ||
* if the property was falsified. | ||
* | ||
* Note: the exception is thrown within a call to `map` on the effect type. Hence, | ||
* this should only be used with effect types that handle exceptions thrown from | ||
* `map`. | ||
*/ | ||
private def checkF[F[_]: Monad]( | ||
prop: PropertyT[F[Result]], | ||
config: PropertyConfig = hedgehogPropertyConfig, | ||
seed: Long = hedgehogSeed | ||
)(implicit loc: Location): F[Unit] = { | ||
??? // Waiting for https://github.com/hedgehogqa/scala-hedgehog/pull/147 | ||
} | ||
|
||
/** | ||
* Supports writing properties with munit assertions instead of Hedgehog `Result`s. | ||
*/ | ||
implicit val unitToResult: Conversion[Unit, Result] = _ => Result.Success | ||
} |
Original file line number | Diff line number | Diff line change | ||||||||
---|---|---|---|---|---|---|---|---|---|---|
@@ -0,0 +1,73 @@ | ||||||||||
package munit | ||||||||||
|
||||||||||
import hedgehog.{Gen, Range, Result} | ||||||||||
import hedgehog.core.SuccessCount | ||||||||||
|
||||||||||
class HedgehogFrameworkSuite extends HedgehogSuite { | ||||||||||
|
||||||||||
// The default property seed changes on each test run so we fix it here so that test failures are always the same | ||||||||||
override val hedgehogSeed = 123L | ||||||||||
|
||||||||||
private val genInt: Gen[Int] = | ||||||||||
Gen.int(Range.linearFrom(0, Int.MinValue, Int.MaxValue)) | ||||||||||
|
||||||||||
property("result check (true)") { | ||||||||||
for { | ||||||||||
l1 <- Gen.list(genInt, Range.linear(0, 100)).forAll | ||||||||||
l2 <- Gen.list(genInt, Range.linear(0, 100)).forAll | ||||||||||
} yield Result.assert(l1.size + l2.size == (l1 ::: l2).size) | ||||||||||
} | ||||||||||
|
||||||||||
property("result check (false)") { | ||||||||||
genInt.forAll.map(n => Result.assert(scala.math.sqrt(n * n) == n)) | ||||||||||
} | ||||||||||
|
||||||||||
property("tagged".tag(new Tag("a"))) { | ||||||||||
genInt.forAll.map(n => Result.assert(n + 0 == n)) | ||||||||||
} | ||||||||||
|
||||||||||
property("assertions (true)") { | ||||||||||
genInt.forAll.map { n => | ||||||||||
assertEquals(n * 2, n + n) | ||||||||||
assertEquals(n * 0, 0) | ||||||||||
} | ||||||||||
} | ||||||||||
|
||||||||||
property("assertions (false)") { | ||||||||||
genInt.forAll.map { n => | ||||||||||
assertEquals(n * 1, n) | ||||||||||
assertEquals(n * n, n) | ||||||||||
assertEquals(n + 0, n) | ||||||||||
} | ||||||||||
} | ||||||||||
|
||||||||||
property( | ||||||||||
"custom config".tag(HedgehogConfig(hedgehogPropertyConfig.copy(testLimit = SuccessCount(1000)))) | ||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is quite a mouthful, I wonder if we want to add custom tags for the most common configuration options? Just brainstorming, it would be nice if the user could do something like this instead 🤔
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we go down this route I would still namespace the tags
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is totally optional BTW, just a suggestion. I'm happy to merge this PR as is. I believe we can update this PR to skip projects where the cross-build is missing There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I agree it's too verbose as-is. I'll update like you suggested. I think I'll add support for passing a seed as a tag as well. I can also remove the One note about tags - figuring out the supported tags is not discoverable via auto-completion. We could address that via extension methods - what do you think? |
||||||||||
) { | ||||||||||
genInt.forAll.map(n => assertEquals(n + 0, n)) | ||||||||||
} | ||||||||||
} | ||||||||||
|
||||||||||
object HedgehogFrameworkSuite | ||||||||||
extends FrameworkTest( | ||||||||||
classOf[HedgehogFrameworkSuite], | ||||||||||
s"""|==> success munit.HedgehogFrameworkSuite.result check (true) | ||||||||||
|==> failure munit.HedgehogFrameworkSuite.result check (false) - Falsified after 0 passed tests and 24 shrinks using seed 123 | ||||||||||
|> -1 | ||||||||||
|==> success munit.HedgehogFrameworkSuite.tagged | ||||||||||
|==> success munit.HedgehogFrameworkSuite.assertions (true) | ||||||||||
|==> failure munit.HedgehogFrameworkSuite.assertions (false) - Falsified after 0 passed tests and 24 shrinks using seed 123 | ||||||||||
|> -1 | ||||||||||
|> munit.FailException: /scala/munit/HedgehogFrameworkSuite.scala:39 | ||||||||||
|38: assertEquals(n * 1, n) | ||||||||||
|39: assertEquals(n * n, n) | ||||||||||
|40: assertEquals(n + 0, n) | ||||||||||
|values are not the same | ||||||||||
|=> Obtained | ||||||||||
|1 | ||||||||||
|=> Diff (- obtained, + expected) | ||||||||||
|-1 | ||||||||||
|+-1 | ||||||||||
|==> success munit.HedgehogFrameworkSuite.custom config | ||||||||||
|""".stripMargin | ||||||||||
) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It would be nice to have a hedgehog release on Maven Central.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Agreed, there's an open issue for it and lead maintainer has been working it but not sure how close it is to being merged.