Result file logging enhancements and restrictions

CHANGELOG

@@ -1,8 +1,12 @@
 4.0.0
 =====
-11 November 2024
+12 November 2024
 - Introduce profiles to support multiple launch configurations
   - `-p|--profile` option
+ - Internal engine enhancements
+  - Replace redundant multi scoped data caches with one single scoped cache
+  - Manage settings internally without exposing them as system properties
+  - Manage implicits at lifecycle level using boundary level caching
 - Require Java 17+ instead of Java 11+
 - Use logback instead of log4j for logging
 - Support `empty` literal in DSLs wherever `blank` is accepted
@@ -12,12 +16,11 @@
 - Add `@{**}` placeholder to template matcher to support ignoring multiple lines
 - Print gwen launch arguments on startup
 - Synchronise system process bindings when resolving value from output stream
-- Only log results file records if results format is sepcified in launch options
-- Internal engine enhancements
-  - Replace redundant multi scoped data caches with one single scoped cache
-  - Manage settings internally without exposing them as system properties
-  - Manage implicits at lifecycle level using boundary level caching
-  - Rename step lambdas to step actions
+- Do not permit scope attribute on results file configuration when logging with annoation or DSL
+- Only log results file records if results format is specified in launch options
+- Skip logging to result file if the scope or status don't match
+  - and disable calling step when logged via DSL
+- Log unresolved field value if interpolation fails during result file logging
 - Update dependencies
   - Update scala from v3.5.0 to v3.5.2
   - Update gherkin from v29.0.0 to v30.0.0
@@ -36,21 +39,6 @@
     - For attaching files to Gwen reports (where the name of the attachment will be the name of the file)
   - `I log record to <resultsFileId> file`
     - For explicitly logging records to configured results files
-- Drop deprecated features:
-  - Gwen Workspaces warning
-  - `-p|--properties` option
-  - Report portal integration
-  - Matrix tables
-  - Core DSL steps
-    - `I wait <duration> second[s] when <element> is <actioned>`
-    - `I wait until <condition> when <element> is <actioned>`
-  - Core DSL steps in favour of base step with `@Timeout` and `@Delay` annotations
-    - `<step> <until|while> <condition> using <delay> <delayUnit> delay`
-    - `<step> <until|while> <condition> using <timeout> <timeoutUnit> timeout`
-    - `<step> <until|while> <condition> using <delay> <delayUnit> delay and <timeout> <timeoutUnit> timeout`
-    - `<step> <until|while> <condition> using no delay and <timeout> <timeoutUnit> timeout`
-    - `<step> <until|while> <condition> using no delay`
-    - `<step> <until|while> <condition> using no delay and <timeout> <timeoutUnit> timeout`
 - Raise deprecations as errors by default instead of logging them as warnings
   | Setting                     | Old default | New default |
   | :-------------------------- | :---------: | :---------: |
@@ -90,6 +78,21 @@
   | `record.index`                        | `gwen.table.record.index`                     |
   | `iteration.number`                    | `gwen.iteration.number`                       |
   | `iteration.index`                     | `gwen.iteration.index`                        |
+- Drop deprecated features:
+  - Gwen Workspaces warning
+  - `-p|--properties` option
+  - Report portal integration
+  - Matrix tables
+  - Core DSL steps
+    - `I wait <duration> second[s] when <element> is <actioned>`
+    - `I wait until <condition> when <element> is <actioned>`
+  - Core DSL steps in favour of base step with `@Timeout` and `@Delay` annotations
+    - `<step> <until|while> <condition> using <delay> <delayUnit> delay`
+    - `<step> <until|while> <condition> using <timeout> <timeoutUnit> timeout`
+    - `<step> <until|while> <condition> using <delay> <delayUnit> delay and <timeout> <timeoutUnit> timeout`
+    - `<step> <until|while> <condition> using no delay and <timeout> <timeoutUnit> timeout`
+    - `<step> <until|while> <condition> using no delay`
+    - `<step> <until|while> <condition> using no delay and <timeout> <timeoutUnit> timeout`
 
 3.66.0
 ======

src/main/scala/gwen/core/eval/action/unit/LogResultsRecord.scala

@@ -35,14 +35,10 @@ class LogResultsRecord[T <: EvalContext](resultsFileId: String) extends UnitStep
     val resultsFile = ctx.options.resultFiles.find(_.id == resultsFileId) getOrElse {
       Errors.resultsFileError(s"No such result file: gwen.reports.results.files.$resultsFileId setting not found")
     }
-    resultsFile.scope foreach { scope =>
-      Errors.resultsFileError(s"Cannot explicitly write to $resultsFileId results file having scope: ${scope.nodeType}${scope.nodeName.map(n => s": $n").getOrElse("")}")
+    if (resultsFile.scope.nonEmpty) {
+       Errors.resultsFileError(s"Scope not permitted on results file when logging with step${Errors.at(step.sourceRef)} - (remove gwen.report.results.files.${resultsFile.id}.scope setting or don't use step DSL)")
     }
-    resultsFile.status foreach { status =>
-      Errors.resultsFileError(s"Cannot explicitly write to $resultsFileId results file having status: $status")
-    }
-    if (ctx.options.reportFormats.contains(ReportFormat.results)) {
-      resultsFile.logRecord(ctx)
+    if (resultsFile.logRecord(parent, ctx, ctx.options)) {
       step
     } else {
       step.copy(withEvalStatus = Passed(System.nanoTime - start, abstained = true))

src/main/scala/gwen/core/eval/engine/ExamplesEngine.scala

@@ -31,6 +31,7 @@ import gwen.core.node.gherkin.SpecNormaliser
 import gwen.core.node.gherkin.Step
 import gwen.core.node.gherkin.StepKeyword
 import gwen.core.node.gherkin.Tag
+import gwen.core.result.ResultFile
 import gwen.core.status.Passed
 import gwen.core.status.Pending
 import gwen.core.status.StatusKeyword
@@ -47,8 +48,9 @@ import java.io.File
 trait ExamplesEngine[T <: EvalContext] extends SpecNormaliser with LazyLogging {
   engine: EvalEngine[T] =>
 
-  def evaluateExamples(parent: GwenNode, examples: List[Examples], ctx: T): List[Examples] = {    
+  def evaluateExamples(parent: GwenNode, examples: List[Examples], ctx: T): List[Examples] = {   
     examples map { exs =>
+      ResultFile.parseAnnotation(exs.tags, ctx.options.resultFiles, exs.nodeType) 
       if (exs.scenarios.isEmpty) {
         transitionExamples(exs, Passed(0, abstained = !ctx.options.dryRun), ctx)
       } else {

src/main/scala/gwen/core/eval/engine/RuleEngine.scala

@@ -23,6 +23,7 @@ import gwen.core.eval.EvalEngine
 import gwen.core.node.GwenNode
 import gwen.core.node.gherkin.Rule
 import gwen.core.node.gherkin.Spec
+import gwen.core.result.ResultFile
 import gwen.core.status._
 
 import scala.util.chaining._
@@ -43,6 +44,7 @@ trait RuleEngine[T <: EvalContext] extends LazyLogging with ImplicitValueKeys {
   }
 
   private def evaluateOrTransitionRule(parent: GwenNode, rule: Rule, dataRecord: Option[DataRecord], ctx: T, acc: List[Rule]): Rule = {
+    ResultFile.parseAnnotation(rule.tags, ctx.options.resultFiles, rule.nodeType)
     EvalStatus(acc.map(_.evalStatus)) match {
       case status @ Failed(_, error) =>
         val isSoftAssert = ctx.evaluate(false) { status.isSoftAssertionError }

src/main/scala/gwen/core/eval/engine/ScenarioEngine.scala

@@ -44,6 +44,9 @@ import scala.util.chaining._
 import com.typesafe.scalalogging.LazyLogging
 
 import java.util.concurrent.CopyOnWriteArrayList
+import gwen.core.node.gherkin.Tag
+import gwen.core.node.NodeType
+import gwen.core.result.ResultFile
 
 /**
   * Scenario evaluation engine.
@@ -116,6 +119,7 @@ trait ScenarioEngine[T <: EvalContext] extends SpecNormaliser with LazyLogging w
         ctx.reset(StateLevel.scenario)
       }
     }
+    ResultFile.parseAnnotation(scenario.tags, ctx.options.resultFiles, scenario.nodeType)
     EvalStatus(acc.map(_.evalStatus)) match {
       case status @ Failed(_, error) =>
         val isSoftAssert = ctx.evaluate(false) { status.isSoftAssertionError }

src/main/scala/gwen/core/eval/engine/SpecEngine.scala

@@ -25,6 +25,7 @@ import gwen.core.node.gherkin.Dialect
 import gwen.core.node.gherkin.Spec
 import gwen.core.node.gherkin.SpecType
 import gwen.core.node.gherkin.SpecPrinter
+import gwen.core.result.ResultFile
 import gwen.core.result.SpecResult
 import gwen.core.status._
 
@@ -69,6 +70,7 @@ trait SpecEngine[T <: EvalContext] extends LazyLogging with ImplicitValueKeys {
   private def evaluateSpec(parent: GwenNode, spec: Spec, metaResults: List[SpecResult], dataRecord: Option[DataRecord], ctx: T): SpecResult = {
     val specType = spec.specType
     ctx.topScope.pushObject(SpecType.toString, specType)
+    ResultFile.parseAnnotation(spec.feature.tags, ctx.options.resultFiles, spec.nodeType) 
     try {
       beforeSpec(spec, ctx)
       val started = {

src/main/scala/gwen/core/eval/engine/StepDefEngine.scala

@@ -26,6 +26,7 @@ import gwen.core.node.gherkin.Scenario
 import gwen.core.node.gherkin.SpecNormaliser
 import gwen.core.node.gherkin.Step
 import gwen.core.node.gherkin.table.DataTable
+import gwen.core.result.ResultFile
 import gwen.core.status.Loaded
 import gwen.core.status.Passed
 
@@ -53,6 +54,7 @@ trait StepDefEngine[T <: EvalContext] extends SpecNormaliser with LazyLogging wi
     * Loads a stepdef to memory.
     */
   private [engine] def loadStepDef(parent: GwenNode, stepDef: Scenario, ctx: T): Scenario = {
+    ResultFile.parseAnnotation(stepDef.tags, ctx.options.resultFiles, stepDef.nodeType) 
     ctx.stepDefScope.boundary(stepDef.name, Nil) {
       beforeStepDef(stepDef, ctx)
       logger.info(s"Loading ${stepDef.keyword}: ${stepDef.name}")
@@ -106,6 +108,7 @@ trait StepDefEngine[T <: EvalContext] extends SpecNormaliser with LazyLogging wi
         withStepDef = Some(stepDef)
       )
       checkStepDefRules(sdStep, ctx)
+      ResultFile.parseAnnotation(stepDef.tags, ctx.options.resultFiles, stepDef.nodeType) 
       ctx.paramScope.boundary(stepDef.name, stepDef.params) {
         val dataTableOpt = stepDef.tags.find(_.name.startsWith(Annotations.DataTable.toString)) map { tag => DataTable(tag, step) }
         val nonEmptyDataTableOpt = dataTableOpt.filter(_.records.nonEmpty)

src/main/scala/gwen/core/report/results/ResultReportsGenerator.scala

@@ -69,78 +69,59 @@ class ResultReportsGenerator(options: GwenOptions, info: GwenInfo)
 
   override def afterSpec(event: NodeEvent[SpecResult]): Unit = {
     val result = event.source 
-    filterFiles(result.spec, result.spec.feature.tags) foreach { resFile => 
-      report(resFile, result.evalStatus, event.env)
+    candidateFiles(result.spec, result.spec.feature.tags) foreach { resFile => 
+      report(resFile, result, event.env)
     }
   }
 
   override def afterRule(event: NodeEvent[Rule]): Unit = { 
     val rule = event.source
     if (rule.scenarios.nonEmpty) {
-      filterFiles(rule, rule.tags) foreach { resFile => 
-        report(resFile, rule.evalStatus, event.env)
+      candidateFiles(rule, rule.tags) foreach { resFile => 
+        report(resFile, rule, event.env)
       }
     }
   }
 
   override def afterScenario(event: NodeEvent[Scenario]): Unit = { 
     val scenario = event.source
     if (!scenario.isOutline && scenario.steps.nonEmpty) {
-      filterFiles(scenario, scenario.tags) foreach { resFile => 
-        report(resFile, scenario.evalStatus, event.env)
+      candidateFiles(scenario, scenario.tags) foreach { resFile => 
+        report(resFile, scenario, event.env)
       }
     }
   }
 
   override def afterExamples(event: NodeEvent[Examples]): Unit = { 
     val examples = event.source
     if (examples.scenarios.nonEmpty) {
-      filterFiles(examples, examples.tags) foreach { resFile => 
-        report(resFile, examples.evalStatus, event.env)
+      candidateFiles(examples, examples.tags) foreach { resFile => 
+        report(resFile, examples, event.env)
       }
     }
   }
 
   override def afterStepDef(event: NodeEvent[Scenario]): Unit = { 
     val stepDef = event.source
     if (stepDef.isStepDefCall) {
-      filterFiles(stepDef, stepDef.tags) foreach { resFile => 
-        report(resFile, stepDef.evalStatus, event.env)
+      candidateFiles(stepDef, stepDef.tags) foreach { resFile => 
+        report(resFile, stepDef, event.env)
       }
     }
   }
 
-  private def filterFiles(node: GwenNode, tags: List[Tag]): List[ResultFile] = {
-    val annotatedIds = tags.filter(_.name == Annotations.Results.toString) flatMap { annotation =>
-      annotation.value flatMap { id =>
-        if (resultFiles.filter(_.id == id).isEmpty) {
-          addError(s"gwen.reports.results.files.$id setting not found for results file denoted by id in $annotation annotation${Errors.at(annotation.sourceRef)}")
-          None
-        } else {
-          Some(id)
-        }
-      }
-    }
-    val scopedIds = resultFiles filter { resFile => 
-      resFile.scope.map(_.nodeType) map { resFileNodeType =>
-        if (resFileNodeType == node.nodeType) {
-          resFile.scope.map(_.nodeName.map(n => node.name.matches("(.* -- )?" + n + "( -- .*)?$")).getOrElse(true)).getOrElse(true)
-        } else {
-          false
-        }
-      } getOrElse false
-    } map(_.id)
+  private def candidateFiles(node: GwenNode, tags: List[Tag]): List[ResultFile] = {
+    val annotatedIds = ResultFile.parseAnnotation(tags, resultFiles, node.nodeType)
+    val scopedIds = resultFiles.filter(f => f.scope.nonEmpty).map(_.id)
     val ids = (annotatedIds ++ scopedIds).distinct
     resultFiles.filter(resFile => ids.contains(resFile.id))
   }
 
-  private def report(resFile: ResultFile, evalStatus: EvalStatus, env: Environment): Unit = {
-    if (resFile.status.map(_ == evalStatus.keyword).getOrElse(true)) {
-      try {
-        resFile.logRecord(env)
-      } catch {
-        case e: Errors.ResultsFileException => e.errors.foreach(addError)
-      }
+  private def report(resFile: ResultFile, node: GwenNode, env: Environment): Unit = {
+    try {
+      resFile.logRecord(node, env, options)
+    } catch {
+      case e: Errors.ResultsFileException => e.errors.foreach(addError)
     }
   }
 

src/main/scala/gwen/core/result/ResultFile.scala

@@ -18,6 +18,11 @@ package gwen.core.result
 
 import gwen.core._
 import gwen.core.data.CsvDataSource
+import gwen.core.node.gherkin.Tag
+import gwen.core.node.gherkin.Annotations
+import gwen.core.node.GwenNode
+import gwen.core.node.NodeType
+import gwen.core.report.ReportFormat
 import gwen.core.state.Environment
 import gwen.core.status.StatusKeyword
 
@@ -27,25 +32,36 @@ import java.io.File
 
 case class ResultFile(id: String, file: File, scope: Option[ResultScope], status: Option[StatusKeyword], fields: List[ResultField]) {
 
-  def logRecord(env: Environment): Unit = {
-    var errors: List[String] = Nil
-    val record = fields map { field =>
-      val value = {
-        Try(env.getBoundValue(field.ref)) getOrElse {
-          field.defaultValue getOrElse {
-            errors = (errors ++ List(s"Unbound ${field.name} field${ if (field.name != field.ref) s" reference: ${field.ref}" else ""} in ${file} results file id ${id}"))
-            s"Unbound ref: ${field.ref}"
+  def logRecord(node: GwenNode, env: Environment, options: GwenOptions): Boolean = {
+    val resultsEnabled = options.reportFormats.contains(ReportFormat.results)
+    val nodeTypeOK = scope.map(_.nodeType == node.nodeType).getOrElse(true)
+    val nodeNameOK = scope.map(_.nodeName.map(n => node.name.matches("(.* -- )?" + n + "( -- .*)?$")).getOrElse(true)).getOrElse(true)
+    val statusOK = status.map(_ == node.evalStatus.keyword).getOrElse(true)
+    if (resultsEnabled && nodeTypeOK && nodeNameOK && statusOK) {
+      var errors: List[String] = Nil
+      val record = fields map { field =>
+        val value = {
+          Try(env.getBoundValue(field.ref)) getOrElse {
+            env.topScope.getOpt(field.ref) getOrElse {
+              field.defaultValue getOrElse {
+                errors = (errors ++ List(s"Unbound ${field.name} field${ if (field.name != field.ref) s" reference: ${field.ref}" else ""} in ${file} results file id ${id}"))
+                s"Unbound ref: ${field.ref}"
+              }
+            }
           }
         }
+        if (value.trim != "" && FileIO.isCsvFile(file)) Formatting.escapeCSV(Formatting.escapeNewLineChars(value))
+        else value
       }
-      if (value.trim != "" && FileIO.isCsvFile(file)) Formatting.escapeCSV(Formatting.escapeNewLineChars(value))
-      else value
-    }
-    this.synchronized {
-      file.appendLine(record.mkString(","))
-    }
-    if (errors.nonEmpty) {
-      Errors.resultsFileErrors(errors)
+      if (errors.nonEmpty) {
+        Errors.resultsFileErrors(errors)
+      }
+      this.synchronized {
+        file.appendLine(record.mkString(","))
+      }
+      true
+    } else {
+      false
     }
   }
 
@@ -106,6 +122,27 @@ object ResultFile {
     fileSettings.map(_._1).filter(s => !s.contains(".fields") && !s.matches(""".*((\.\d+\.)(""" + ResultFieldAtts.values.mkString("|") + "))$")).map(s => s.substring(s.lastIndexOf(".") + 1)).foreach(ResultFile.validateSettingName)
     ResultFile(id, file, scope, status, fields)
   }
+
+  def parseAnnotation(tags: List[Tag], resultFiles: List[ResultFile], nodeType: NodeType): List[String] = {
+    tags.filter(_.name.startsWith(Annotations.Results.toString)) flatMap { annotation => 
+      annotation.toString match {
+        case r"""@Results\((.+?)$value\)""" =>
+          Tag.parseListValue(annotation.sourceRef, Annotations.Results, None, value) map { id => 
+            resultFiles.find(_.id == id) match {
+              case Some(resFile) =>
+                if (resFile.scope.nonEmpty) {
+                  Errors.resultsFileError(s"Scope not permitted on $id results file when logging with $annotation annotation${Errors.at(annotation.sourceRef)} - (remove gwen.report.results.files.$id.scope setting or don't use annotation)")
+                } else {
+                  id
+                }
+              case None =>
+                Errors.resultsFileError(s"Results file not found with id: $id - check your gwen.report.results.files.$id setting or $annotation annotation${Errors.at(annotation.sourceRef)}")
+            }
+          }
+        case _ => Errors.invalidTagError(s"""Invalid Results annotation: $annotation${Errors.at(annotation.sourceRef)} - correct syntax is @Results('id') or @Results(['id1','id2','idN'])""")
+      }
+    }
+  }
 }
 
 enum ResultFileAtts: