From 612861870b1f5e71376b071980cffe1dc819fb1a Mon Sep 17 00:00:00 2001 From: Shubham Chaudhary Date: Thu, 15 Oct 2020 18:03:02 +0530 Subject: [PATCH] [Cherry-Pick for v1.9.0] (#180) * fix(node-io-stress): Add fix for node-io-stress (#177) Signed-off-by: Udit Gaurav * fix(abort): fixing the result updation for abort (#178) Signed-off-by: shubhamchaudhary * refactor(experiment): covered uncaught errors (#179) Signed-off-by: shubhamchaudhary Co-authored-by: UDIT GAURAV <35391335+uditgaurav@users.noreply.github.com> --- chaoslib/litmus/node-io-stress/lib/node-io-stress.go | 2 +- .../generic/container-kill/experiment/container-kill.go | 6 ++++-- experiments/generic/disk-fill/experiment/disk-fill.go | 6 ++++-- .../kubelet-service-kill/experiment/kubelet-service-kill.go | 6 ++++-- experiments/generic/node-cpu-hog/experiment/node-cpu-hog.go | 6 ++++-- experiments/generic/node-drain/experiment/node-drain.go | 6 ++++-- .../generic/node-io-stress/experiment/node-io-stress.go | 6 ++++-- .../generic/node-memory-hog/experiment/node-memory-hog.go | 6 ++++-- experiments/generic/node-taint/experiment/node-taint.go | 6 ++++-- experiments/generic/pod-cpu-hog/experiment/pod-cpu-hog.go | 6 ++++-- experiments/generic/pod-delete/experiment/pod-delete.go | 3 +-- .../generic/pod-io-stress/experiment/pod-io-stress.go | 6 ++++-- .../generic/pod-memory-hog/experiment/pod-memory-hog.go | 6 ++++-- .../experiment/pod-network-corruption.go | 6 ++++-- .../experiment/pod-network-duplication.go | 6 ++++-- .../pod-network-latency/experiment/pod-network-latency.go | 6 ++++-- .../generic/pod-network-loss/experiment/pod-network-loss.go | 6 ++++-- pkg/result/chaosresult.go | 4 +++- 18 files changed, 65 insertions(+), 34 deletions(-) diff --git a/chaoslib/litmus/node-io-stress/lib/node-io-stress.go b/chaoslib/litmus/node-io-stress/lib/node-io-stress.go index ac7d0b66f..228bdf65c 100644 --- a/chaoslib/litmus/node-io-stress/lib/node-io-stress.go +++ b/chaoslib/litmus/node-io-stress/lib/node-io-stress.go @@ -124,7 +124,7 @@ func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie Image: experimentsDetails.LIBImage, ImagePullPolicy: apiv1.PullAlways, Command: []string{ - "/stress-ng", + "stress-ng", }, Args: GetContainerArguments(experimentsDetails), }, diff --git a/experiments/generic/container-kill/experiment/container-kill.go b/experiments/generic/container-kill/experiment/container-kill.go index a96ce909a..0e25d672f 100644 --- a/experiments/generic/container-kill/experiment/container-kill.go +++ b/experiments/generic/container-kill/experiment/container-kill.go @@ -35,8 +35,10 @@ func ContainerKill(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/disk-fill/experiment/disk-fill.go b/experiments/generic/disk-fill/experiment/disk-fill.go index 3af00458e..1153b8c69 100644 --- a/experiments/generic/disk-fill/experiment/disk-fill.go +++ b/experiments/generic/disk-fill/experiment/disk-fill.go @@ -33,8 +33,10 @@ func DiskFill(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/kubelet-service-kill/experiment/kubelet-service-kill.go b/experiments/generic/kubelet-service-kill/experiment/kubelet-service-kill.go index 5b8c77018..d7913f0b1 100644 --- a/experiments/generic/kubelet-service-kill/experiment/kubelet-service-kill.go +++ b/experiments/generic/kubelet-service-kill/experiment/kubelet-service-kill.go @@ -33,8 +33,10 @@ func KubeletServiceKill(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/node-cpu-hog/experiment/node-cpu-hog.go b/experiments/generic/node-cpu-hog/experiment/node-cpu-hog.go index 639991f63..1876cadee 100644 --- a/experiments/generic/node-cpu-hog/experiment/node-cpu-hog.go +++ b/experiments/generic/node-cpu-hog/experiment/node-cpu-hog.go @@ -35,8 +35,10 @@ func NodeCPUHog(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/node-drain/experiment/node-drain.go b/experiments/generic/node-drain/experiment/node-drain.go index 30b8ddf36..fe7a5a399 100644 --- a/experiments/generic/node-drain/experiment/node-drain.go +++ b/experiments/generic/node-drain/experiment/node-drain.go @@ -33,8 +33,10 @@ func NodeDrain(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/node-io-stress/experiment/node-io-stress.go b/experiments/generic/node-io-stress/experiment/node-io-stress.go index 2d519f289..52bc10abb 100644 --- a/experiments/generic/node-io-stress/experiment/node-io-stress.go +++ b/experiments/generic/node-io-stress/experiment/node-io-stress.go @@ -35,8 +35,10 @@ func NodeIOStress(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/node-memory-hog/experiment/node-memory-hog.go b/experiments/generic/node-memory-hog/experiment/node-memory-hog.go index 08cdd9180..055857ed9 100644 --- a/experiments/generic/node-memory-hog/experiment/node-memory-hog.go +++ b/experiments/generic/node-memory-hog/experiment/node-memory-hog.go @@ -35,8 +35,10 @@ func NodeMemoryHog(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/node-taint/experiment/node-taint.go b/experiments/generic/node-taint/experiment/node-taint.go index 3895af354..adda11363 100644 --- a/experiments/generic/node-taint/experiment/node-taint.go +++ b/experiments/generic/node-taint/experiment/node-taint.go @@ -33,8 +33,10 @@ func NodeTaint(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/pod-cpu-hog/experiment/pod-cpu-hog.go b/experiments/generic/pod-cpu-hog/experiment/pod-cpu-hog.go index 9f780c6e1..b8a4fe42b 100644 --- a/experiments/generic/pod-cpu-hog/experiment/pod-cpu-hog.go +++ b/experiments/generic/pod-cpu-hog/experiment/pod-cpu-hog.go @@ -36,8 +36,10 @@ func PodCPUHog(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/pod-delete/experiment/pod-delete.go b/experiments/generic/pod-delete/experiment/pod-delete.go index bc406b0cb..5b6b6e513 100644 --- a/experiments/generic/pod-delete/experiment/pod-delete.go +++ b/experiments/generic/pod-delete/experiment/pod-delete.go @@ -35,10 +35,9 @@ func PodDelete(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet if err := probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { log.Fatalf("Unable to initialise probes details from chaosengine, err: %v", err) - } //Updating the chaos result in the beginning of experiment diff --git a/experiments/generic/pod-io-stress/experiment/pod-io-stress.go b/experiments/generic/pod-io-stress/experiment/pod-io-stress.go index 11a960817..328442744 100644 --- a/experiments/generic/pod-io-stress/experiment/pod-io-stress.go +++ b/experiments/generic/pod-io-stress/experiment/pod-io-stress.go @@ -35,8 +35,10 @@ func PodIOStress(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/pod-memory-hog/experiment/pod-memory-hog.go b/experiments/generic/pod-memory-hog/experiment/pod-memory-hog.go index 8215dd50b..15eb33a1d 100644 --- a/experiments/generic/pod-memory-hog/experiment/pod-memory-hog.go +++ b/experiments/generic/pod-memory-hog/experiment/pod-memory-hog.go @@ -36,8 +36,10 @@ func PodMemoryHog(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/pod-network-corruption/experiment/pod-network-corruption.go b/experiments/generic/pod-network-corruption/experiment/pod-network-corruption.go index b4627d068..4e8f1dd6b 100644 --- a/experiments/generic/pod-network-corruption/experiment/pod-network-corruption.go +++ b/experiments/generic/pod-network-corruption/experiment/pod-network-corruption.go @@ -36,8 +36,10 @@ func PodNetworkCorruption(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/pod-network-duplication/experiment/pod-network-duplication.go b/experiments/generic/pod-network-duplication/experiment/pod-network-duplication.go index 9a2010462..6015d6274 100644 --- a/experiments/generic/pod-network-duplication/experiment/pod-network-duplication.go +++ b/experiments/generic/pod-network-duplication/experiment/pod-network-duplication.go @@ -36,8 +36,10 @@ func PodNetworkDuplication(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/pod-network-latency/experiment/pod-network-latency.go b/experiments/generic/pod-network-latency/experiment/pod-network-latency.go index afd80ed3f..6af5c0e1d 100644 --- a/experiments/generic/pod-network-latency/experiment/pod-network-latency.go +++ b/experiments/generic/pod-network-latency/experiment/pod-network-latency.go @@ -36,8 +36,10 @@ func PodNetworkLatency(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginning of experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/experiments/generic/pod-network-loss/experiment/pod-network-loss.go b/experiments/generic/pod-network-loss/experiment/pod-network-loss.go index d3be47de9..a044990ab 100644 --- a/experiments/generic/pod-network-loss/experiment/pod-network-loss.go +++ b/experiments/generic/pod-network-loss/experiment/pod-network-loss.go @@ -36,8 +36,10 @@ func PodNetworkLoss(clients clients.ClientSets) { // Intialise Chaos Result Parameters types.SetResultAttributes(&resultDetails, chaosDetails) - // Intialise the probe details - probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails) + // Intialise the probe details. Bail out upon error, as we haven't entered exp business logic yet + if err = probe.InitializeProbesInChaosResultDetails(&chaosDetails, clients, &resultDetails); err != nil { + log.Fatalf("Unable to initialize the probes, err: %v", err) + } //Updating the chaos result in the beginningo f experiment log.Infof("[PreReq]: Updating the chaos result of %v experiment (SOT)", experimentsDetails.ExperimentName) diff --git a/pkg/result/chaosresult.go b/pkg/result/chaosresult.go index fbcb50aa2..e79efae51 100644 --- a/pkg/result/chaosresult.go +++ b/pkg/result/chaosresult.go @@ -39,7 +39,9 @@ func ChaosResult(chaosDetails *types.ChaosDetails, clients clients.ClientSets, r return err } - if chaosDetails.EngineName != "" { + // as the chaos pod won't be available for stopped phase + // skipping the derivation of labels from chaos pod, if phase is stopped + if chaosDetails.EngineName != "" && resultDetails.Phase != "Stopped" { // Getting chaos pod label and passing it in chaos result chaosPod, err := clients.KubeClient.CoreV1().Pods(chaosDetails.ChaosNamespace).Get(chaosDetails.ChaosPodName, metav1.GetOptions{}) if err != nil {