Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for Hedgehog property testing #78

Closed
wants to merge 8 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,34 @@ lazy val munitScalacheckJVM = munitScalacheck.jvm
lazy val munitScalacheckJS = munitScalacheck.js
lazy val munitScalacheckNative = munitScalacheck.native

lazy val munitHedgehog = crossProject(JSPlatform, JVMPlatform, NativePlatform)
.in(file("munit-hedgehog"))
.dependsOn(munit)
.settings(
moduleName := "munit-hedgehog",
sharedSettings,
crossScalaVersions := List(scala213, scala212, scala211, dotty),
resolvers += "bintray-scala-hedgehog".at(
"https://dl.bintray.com/hedgehogqa/scala-hedgehog"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be nice to have a hedgehog release on Maven Central.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, there's an open issue for it and lead maintainer has been working it but not sure how close it is to being merged.

),
libraryDependencies += ("qa.hedgehog" %%% "hedgehog-runner" % "97854199ef795a5dfba15478fd9abe66035ddea2")
.withDottyCompat(scalaVersion.value)
)
.jvmSettings(
skip in publish := customScalaJSVersion.isDefined
)
.nativeConfigure(sharedNativeConfigure)
.nativeSettings(
sharedNativeSettings,
skip in publish := customScalaJSVersion.isDefined
)
.jsSettings(sharedJSSettings)
lazy val munitHedgehogJVM = munitHedgehog.jvm
lazy val munitHedgehogJS = munitHedgehog.js
lazy val munitHedgehogNative = munitHedgehog.native

lazy val tests = crossProject(JSPlatform, JVMPlatform, NativePlatform)
.dependsOn(munit, munitScalacheck)
.dependsOn(munit, munitScalacheck, munitHedgehog)
.enablePlugins(BuildInfoPlugin)
.settings(
sharedSettings,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package munit

import hedgehog.core.PropertyConfig

case class HedgehogConfig(config: PropertyConfig) extends Tag("HedgehogConfig")
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package munit

import scala.util.control.NoStackTrace

import hedgehog.core._
import hedgehog.runner.Test

class HedgehogFailException(message: String, val report: Report, val seed: Long)
extends Exception(message)
with NoStackTrace

object HedgehogFailException {

def fromReport(report: Report, seed: Long): Option[HedgehogFailException] = {
report.status match {
case OK =>
None
case Failed(shrinks, log) =>
val coverage = Test.renderCoverage(report.coverage, report.tests)
val message = render(
s"Falsified after ${report.tests.value} passed tests and ${shrinks.value} shrinks using seed ${seed}",
log.map(renderLog) ++ coverage
)
Some(new HedgehogFailException(message, report, seed))
case GaveUp =>
val coverage = Test.renderCoverage(report.coverage, report.tests)
val message = render(
s"Gave up after ${report.tests.value} passed tests using seed value $seed. ${report.discards.value} were discarded",
coverage
)
Some(new HedgehogFailException(message, report, seed))
}
}

private def render(msg: String, extras: List[String]): String =
(msg :: extras.map(e => "> " + e)).mkString("\n")

// From Hedgehog, but customized to *not* include the stack trace and instead print exception toString
private def renderLog(log: Log): String = {
log match {
case ForAll(name, value) =>
s"${name.value}: $value"
case Info(value) =>
value
case Error(e) =>
e.toString
}
}
}
95 changes: 95 additions & 0 deletions munit-hedgehog/shared/src/main/scala/munit/HedgehogSuite.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package munit

import hedgehog._
import hedgehog.core.{PropertyConfig, PropertyT, Seed}
import hedgehog.predef.Monad

import munit.internal.Compat._

/**
* Provides the ability to write property based tests using the Hedgehog library.
*
* Properties are defined by calling one of the various `property` or `propertyF`
* methods and passing a `PropertyT[Result]` or `PropertyT[F[Result]]` respectively.
* For example:
* {{{
* property("additive identity") {
* Gen.int(Range.linearFrom(0, Int.MinValue, Int.MaxValue)).forAll.map { n =>
* assertEquals(n + 0, n)
* }
* }
* }}}
*
* Note both Hedgehog `Result` and munit assertions are supported.
*/
trait HedgehogSuite extends FunSuite {

/**
* Provides the default configuration for running property tests.
* This can be overriden to change the default or can be changed on
* a test-by-test basis by using an overload of `property` and `propertyF`.
*/
protected def hedgehogPropertyConfig: PropertyConfig = PropertyConfig.default
protected def hedgehogSeed: Long = System.currentTimeMillis()

def property(
name: String
)(prop: PropertyT[Result])(implicit loc: Location): Unit =
property(new TestOptions(name, Set.empty, loc))(prop)

def property(
options: TestOptions
)(prop: PropertyT[Result])(implicit loc: Location): Unit = {
val config = options.tags.collectFirst { case HedgehogConfig(config) => config }.getOrElse(hedgehogPropertyConfig)
test(options)(check(prop, config, hedgehogSeed))
}

def propertyF[F[_]: Monad](
name: String
)(prop: PropertyT[F[Result]])(implicit loc: Location): Unit =
propertyF(new TestOptions(name, Set.empty, loc))(prop)

def propertyF[F[_]: Monad](
options: TestOptions
)(prop: PropertyT[F[Result]])(implicit loc: Location): Unit = {
val config = options.tags.collectFirst { case HedgehogConfig(config) => config }.getOrElse(hedgehogPropertyConfig)
test(options)(checkF(prop, config, hedgehogSeed))
}

/**
* Checks the supplied `Property[Result]`, throwing a `HedgehogFailException`
* if the property was falsified.
*/
private def check(
prop: PropertyT[Result],
config: PropertyConfig = hedgehogPropertyConfig,
seed: Long = hedgehogSeed
)(implicit loc: Location): Unit = {
val report = Property.check(config, prop, Seed.fromLong(seed))
HedgehogFailException.fromReport(report, seed) match {
case None => ()
case Some(t) => throw t
}
}

/**
* Checks the supplied `PropertyT[F[Result]]`, throwing a `HedgehogFailException`
* if the property was falsified.
*
* Note: the exception is thrown within a call to `map` on the effect type. Hence,
* this should only be used with effect types that handle exceptions thrown from
* `map`.
*/
private def checkF[F[_]: Monad](
prop: PropertyT[F[Result]],
config: PropertyConfig = hedgehogPropertyConfig,
seed: Long = hedgehogSeed
)(implicit loc: Location): F[Unit] = {
??? // Waiting for https://github.com/hedgehogqa/scala-hedgehog/pull/147
}

/**
* Supports writing properties with munit assertions instead of Hedgehog `Result`s.
*/
implicit val unitToResult: Conversion[Unit, Result] = _ => Result.Success
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ object Compat {
p.productElementNames
def collectionClassName(i: Iterable[_]): String =
i.asInstanceOf[{ def collectionClassName: String }].collectionClassName
type Conversion[-A, +B] = A => B
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ object Compat {
Iterator.continually("")
def collectionClassName(i: Iterable[_]): String =
i.stringPrefix
type Conversion[-A, +B] = A => B
}
73 changes: 73 additions & 0 deletions tests/shared/src/main/scala/munit/HedgehogFrameworkSuite.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package munit

import hedgehog.{Gen, Range, Result}
import hedgehog.core.SuccessCount

class HedgehogFrameworkSuite extends HedgehogSuite {

// The default property seed changes on each test run so we fix it here so that test failures are always the same
override val hedgehogSeed = 123L

private val genInt: Gen[Int] =
Gen.int(Range.linearFrom(0, Int.MinValue, Int.MaxValue))

property("result check (true)") {
for {
l1 <- Gen.list(genInt, Range.linear(0, 100)).forAll
l2 <- Gen.list(genInt, Range.linear(0, 100)).forAll
} yield Result.assert(l1.size + l2.size == (l1 ::: l2).size)
}

property("result check (false)") {
genInt.forAll.map(n => Result.assert(scala.math.sqrt(n * n) == n))
}

property("tagged".tag(new Tag("a"))) {
genInt.forAll.map(n => Result.assert(n + 0 == n))
}

property("assertions (true)") {
genInt.forAll.map { n =>
assertEquals(n * 2, n + n)
assertEquals(n * 0, 0)
}
}

property("assertions (false)") {
genInt.forAll.map { n =>
assertEquals(n * 1, n)
assertEquals(n * n, n)
assertEquals(n + 0, n)
}
}

property(
"custom config".tag(HedgehogConfig(hedgehogPropertyConfig.copy(testLimit = SuccessCount(1000))))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is quite a mouthful, I wonder if we want to add custom tags for the most common configuration options? Just brainstorming, it would be nice if the user could do something like this instead 🤔

Suggested change
"custom config".tag(HedgehogConfig(hedgehogPropertyConfig.copy(testLimit = SuccessCount(1000))))
"custom config".tag(TestLimit(1000))

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we go down this route I would still namespace the tags

Suggested change
"custom config".tag(HedgehogConfig(hedgehogPropertyConfig.copy(testLimit = SuccessCount(1000))))
"custom config".tag(HedgehogTestLimit(1000))

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is totally optional BTW, just a suggestion. I'm happy to merge this PR as is. I believe we can update this PR to skip projects where the cross-build is missing

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree it's too verbose as-is. I'll update like you suggested. I think I'll add support for passing a seed as a tag as well.

I can also remove the propertyF stuff for now (until Hedgehog merges support for that).

One note about tags - figuring out the supported tags is not discoverable via auto-completion. We could address that via extension methods - what do you think?

) {
genInt.forAll.map(n => assertEquals(n + 0, n))
}
}

object HedgehogFrameworkSuite
extends FrameworkTest(
classOf[HedgehogFrameworkSuite],
s"""|==> success munit.HedgehogFrameworkSuite.result check (true)
|==> failure munit.HedgehogFrameworkSuite.result check (false) - Falsified after 0 passed tests and 24 shrinks using seed 123
|> -1
|==> success munit.HedgehogFrameworkSuite.tagged
|==> success munit.HedgehogFrameworkSuite.assertions (true)
|==> failure munit.HedgehogFrameworkSuite.assertions (false) - Falsified after 0 passed tests and 24 shrinks using seed 123
|> -1
|> munit.FailException: /scala/munit/HedgehogFrameworkSuite.scala:39
|38: assertEquals(n * 1, n)
|39: assertEquals(n * n, n)
|40: assertEquals(n + 0, n)
|values are not the same
|=> Obtained
|1
|=> Diff (- obtained, + expected)
|-1
|+-1
|==> success munit.HedgehogFrameworkSuite.custom config
|""".stripMargin
)
3 changes: 2 additions & 1 deletion tests/shared/src/test/scala/munit/FrameworkSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ class FrameworkSuite extends BaseFrameworkSuite {
TestTransformFrameworkSuite,
ValueTransformCrashFrameworkSuite,
ValueTransformFrameworkSuite,
ScalaCheckFrameworkSuite
ScalaCheckFrameworkSuite,
HedgehogFrameworkSuite
)
tests.foreach { t => check(t) }
}