Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve Verification of FA Success Response #70

Merged
merged 3 commits into from
Aug 1, 2023
Merged

Improve Verification of FA Success Response #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
170c930
Don't test FAR logs when FAR pod resides in the unhealthy node
razo7 Jul 31, 2023
3d82d20
Test success log with node name
razo7 Jul 31, 2023
cf4e247
Typo fix
razo7 Aug 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions controllers/fenceagentsremediation_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ func (r *FenceAgentsRemediationReconciler) Reconcile(ctx context.Context, req ct
r.Log.Error(err, "Fence Agent response wasn't a success message", "CR's Name", req.Name)
return emptyResult, err
}
r.Log.Info("Fence Agent command was finished successfully", "Fence Agent", far.Spec.Agent, "Node name", req.Name, "Response", SuccessFAResponse)

// Reboot was finished and now we remove workloads (pods and their VA)
r.Log.Info("Manual workload deletion", "Fence Agent", far.Spec.Agent, "Node Name", req.Name)
Expand Down
25 changes: 19 additions & 6 deletions test/e2e/far_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,14 +346,29 @@ func makeNodeUnready(node *corev1.Node) {
log.Info("node is unready", "node name", node.GetName())
}

// checkFarLogs gets the FAR pod and checks whether its logs have logString
func checkFarLogs(logString string) {
// buildExpectedLogOutput returns a string with a node identifier and a success message for the reboot action
func buildExpectedLogOutput(nodeName, successMessage string) string {
expectedString := fmt.Sprintf("\"Node name\": \"%s\", \"Response\": \"%s", nodeName, successMessage)
log.Info("Substring to search in the logs", "expectedString", expectedString)
return expectedString
}

// checkFarLogs gets the FAR pod and checks whether it's logs have logString, and if the pod was in the unhealthyNode
// then we don't look for the expected logString
func checkFarLogs(unhealthyNodeName, logString string) {
EventuallyWithOffset(1, func() string {
pod, err := utils.GetFenceAgentsRemediationPod(k8sClient)
if err != nil {
log.Error(err, "failed to get FAR pod. Might try again")
return ""
}
if pod.Spec.NodeName == unhealthyNodeName {
// When reboot is running on FAR node, then FAR pod will be recreated on a new node
// and since the FA command won't be executed again, then the log won't include
// any success message, so we won't verfiy the FAR success message on this scenario
log.Info("The created FAR CR is for the node FAR pod resides, thus we won't test its logs", "expected string", logString)
return logString
}
logs, err := e2eUtils.GetLogs(clientSet, pod, containerName)
if err != nil {
if apiErrors.IsNotFound(err) {
Expand Down Expand Up @@ -413,10 +428,8 @@ func checkRemediation(nodeName string, nodeBootTimeBefore time.Time, oldPodCreat
wasFarTaintAdded(nodeName)

By("Check if the response of the FA was a success")
// TODO: When reboot is running only once and it is running on FAR node, then FAR pod will
// be recreated on a new node and since the FA command won't be exuected again, then the log
// won't include any success message
checkFarLogs(controllers.SuccessFAResponse)
expectedLog := buildExpectedLogOutput(nodeName, controllers.SuccessFAResponse)
checkFarLogs(nodeName, expectedLog)

By("Getting new node's boot time")
wasNodeRebooted(nodeName, nodeBootTimeBefore)
Expand Down