[Heartbeat] Fix missing monitor status on 1 of 2 attempts (#36704)

[Heartbeat] Fix missing monitor.status value in initial attempt where max_attempts > 2. Introduced in #36623 adding tests to the scenario runner as well.

Original cause was this PR #36519 that did not produce the correct monitor.status: down when the monitor is retried with the second attempt.

heartbeat/monitors/wrappers/summarizer/plugstatestat.go

@@ -63,13 +63,15 @@ func (ssp *BrowserStateStatusPlugin) BeforeSummary(event *beat.Event) BeforeSumm
 	}
 
 	res := ssp.cssp.BeforeSummary(event)
-
+	// Browsers don't set this prior, so we set this here, as opposed to lightweight monitors
 	_, _ = event.PutValue("monitor.status", string(ssp.cssp.js.Status))
+
+	_, _ = event.PutValue("synthetics", mapstr.M{"type": "heartbeat/summary"})
 	return res
 }
 
 func (ssp *BrowserStateStatusPlugin) BeforeRetry() {
-	// noop
+	ssp.cssp.BeforeRetry()
 }
 
 // LightweightStateStatusPlugin encapsulates the writing of the primary fields used by the summary,
@@ -108,7 +110,7 @@ func (ssp *LightweightStateStatusPlugin) BeforeSummary(event *beat.Event) Before
 }
 
 func (ssp *LightweightStateStatusPlugin) BeforeRetry() {
-	// noop
+	ssp.cssp.BeforeRetry()
 }
 
 type commonSSP struct {
@@ -162,21 +164,19 @@ func (ssp *commonSSP) BeforeSummary(event *beat.Event) BeforeSummaryActions {
 		"summary": &jsCopy,
 		"state":   ms,
 	}
-	if ssp.sf.Type == "browser" {
-		fields["synthetics"] = mapstr.M{"type": "heartbeat/summary"}
-	}
-	eventext.MergeEventFields(event, fields)
 
-	if retry {
-		// mutate the js into the state for the next attempt
-		ssp.js.BumpAttempt()
-	}
+	eventext.MergeEventFields(event, fields)
 
-	logp.L().Debugf("attempt info: %v == %v && %d < %d", ssp.js.Status, lastStatus, ssp.js.Attempt, ssp.js.MaxAttempts)
+	logp.L().Debugf("attempt info: current(%v) == lastStatus(%v) && attempts(%d < %d)", ssp.js.Status, lastStatus, ssp.js.Attempt, ssp.js.MaxAttempts)
 
 	if retry {
 		return RetryBeforeSummary
 	}
 
 	return 0
 }
+
+func (ssp *commonSSP) BeforeRetry() {
+	// mutate the js into the state for the next attempt
+	ssp.js.BumpAttempt()
+}

x-pack/heartbeat/scenarios/basics_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/monitorstate"
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/summarizer/jobsummary"
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers/summarizer/summarizertesthelper"
+	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/scenarios/framework"
 )
 
@@ -99,25 +100,22 @@ func TestLightweightUrls(t *testing.T) {
 
 func TestLightweightSummaries(t *testing.T) {
 	t.Parallel()
-	scenarioDB.RunTag(t, "lightweight", func(t *testing.T, mtr *framework.MonitorTestRun, err error) {
+	scenarioDB.RunTagWithSeparateTwists(t, "lightweight", StdAttemptTwists, func(t *testing.T, mtr *framework.MonitorTestRun, err error) {
 		all := mtr.Events()
-		lastEvent, firstEvents := all[len(all)-1], all[:len(all)-1]
+		lastEvent := all[len(all)-1]
 		testslike.Test(t,
 			SummaryValidatorForStatus(mtr.Meta.Status),
 			lastEvent.Fields)
 
-		for _, e := range firstEvents {
-			summary, _ := e.GetValue("summary")
-			require.Nil(t, summary)
-		}
+		requireOneSummaryPerAttempt(t, all)
 	})
 }
 
 func TestBrowserSummaries(t *testing.T) {
 	t.Parallel()
-	scenarioDB.RunTag(t, "browser", func(t *testing.T, mtr *framework.MonitorTestRun, err error) {
+	scenarioDB.RunTagWithSeparateTwists(t, "browser", StdAttemptTwists, func(t *testing.T, mtr *framework.MonitorTestRun, err error) {
 		all := mtr.Events()
-		lastEvent, firstEvents := all[len(all)-1], all[:len(all)-1]
+		lastEvent := all[len(all)-1]
 
 		testslike.Test(t,
 			lookslike.Compose(
@@ -126,13 +124,30 @@ func TestBrowserSummaries(t *testing.T) {
 			),
 			lastEvent.Fields)
 
-		for _, e := range firstEvents {
-			summary, _ := e.GetValue("summary")
-			require.Nil(t, summary)
-		}
+		monStatus, _ := lastEvent.GetValue("monitor.status")
+		summaryIface, _ := lastEvent.GetValue("summary")
+		summary := summaryIface.(*jobsummary.JobSummary)
+		require.Equal(t, string(summary.Status), monStatus, "expected summary status and mon status to be equal in event: %v", lastEvent.Fields)
+
+		requireOneSummaryPerAttempt(t, all)
+
 	})
 }
 
+func requireOneSummaryPerAttempt(t *testing.T, events []*beat.Event) {
+	attemptCounter := uint16(1)
+	// ensure we only have one summary per attempt
+	for _, e := range events {
+		summaryIface, _ := e.GetValue("summary")
+		if summaryIface != nil {
+			summary := summaryIface.(*jobsummary.JobSummary)
+			require.Equal(t, attemptCounter, summary.Attempt)
+			require.LessOrEqual(t, summary.Attempt, summary.MaxAttempts)
+			attemptCounter++
+		}
+	}
+}
+
 func TestRunFromOverride(t *testing.T) {
 	t.Parallel()
 	scenarioDB.RunAllWithATwist(t, TwistAddRunFrom, func(t *testing.T, mtr *framework.MonitorTestRun, err error) {

x-pack/heartbeat/scenarios/browserscenarios.go

@@ -51,7 +51,7 @@ func init() {
 		framework.Scenario{
 			Name: "failing-browser",
 			Type: "browser",
-			Tags: []string{"browser", "browser-inline", "down"},
+			Tags: []string{"browser", "browser-inline", "down", "browser-down"},
 			Runner: func(t *testing.T) (config mapstr.M, meta framework.ScenarioRunMeta, close func(), err error) {
 				err = os.Setenv("ELASTIC_SYNTHETICS_CAPABLE", "true")
 				if err != nil {

x-pack/heartbeat/scenarios/framework/framework.go

@@ -214,6 +214,12 @@ func (sdb *ScenarioDB) RunTagWithATwist(t *testing.T, tagName string, twist *Twi
 	}
 }
 
+func (sdb *ScenarioDB) RunTagWithSeparateTwists(t *testing.T, tagName string, twists []*Twist, callback func(*testing.T, *MonitorTestRun, error)) {
+	for _, twist := range twists {
+		sdb.RunTagWithATwist(t, tagName, twist, callback)
+	}
+}
+
 type MonitorTestRun struct {
 	StdFields stdfields.StdMonitorFields
 	Meta      ScenarioRunMeta

x-pack/heartbeat/scenarios/twists.go

@@ -36,6 +36,9 @@ func TwistMultiRun(times int) *framework.Twist {
 	})
 }
 
+// StdAttemptTwists is a list of real world attempt numbers, that is to say both one and two twists.
+var StdAttemptTwists = []*framework.Twist{TwistMaxAttempts(1), TwistMaxAttempts(2)}
+
 func TwistMaxAttempts(maxAttempts int) *framework.Twist {
 	return framework.MakeTwist(fmt.Sprintf("run with %d max_attempts", maxAttempts), func(s framework.Scenario) framework.Scenario {
 		s.Tags = append(s.Tags, "retry")