Print container status on reconcile timeout (#58)

* Print container status on reconcile timeout Signed-off-by: Raul Sevilla <[email protected]> * Add node name to debug information Signed-off-by: Raul Sevilla <[email protected]> --------- Signed-off-by: Raul Sevilla <[email protected]>
cloud-bulldozer · Oct 27, 2023 · ffe4f3c · ffe4f3c
1 parent efc206d
commit ffe4f3c
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 8 deletions.
diff --git a/pkg/config/types.go b/pkg/config/types.go
@@ -34,16 +34,16 @@ type Config struct {
 	Concurrency int32 `yaml:"concurrency" json:"concurrency"`
 	// Procs processes per client pod
 	Procs int `yaml:"procs" json:"procs"`
-	// Tool defines the tool to run the benchmark scenario
+	// Tool defines the tool to run the benchmark scenario. Example: wrk
 	Tool string `yaml:"tool" json:"tool"`
-	// ServerReplicas number of server (nginx) replicas backed by the routes. Example: wrk
+	// ServerReplicas number of server (nginx) replicas backed by the routes
 	ServerReplicas int32 `yaml:"serverReplicas" json:"serverReplicas"`
 	// Tuning defines a tuning patch for the default IngressController object
 	Tuning string `yaml:"tuningPatch" json:"tuningPatch"`
 	// Delay defines a delay between samples
-	Delay time.Duration `yaml:"delay"`
+	Delay time.Duration `yaml:"delay" json:"delay"`
 	// Warmup enables warmup: Indexing will be disabled in this scenario. Default is false
 	Warmup bool `yaml:"warmup" json:"-"`
 	// RequestTimeout defines the tool request timeout
-	RequestTimeout time.Duration `yaml:"requestTimeout"`
+	RequestTimeout time.Duration `yaml:"requestTimeout" json:"requestTimeout"`
 }
diff --git a/pkg/runner/exec.go b/pkg/runner/exec.go
@@ -92,7 +92,7 @@ func runBenchmark(cfg config.Config, clusterMetadata tools.ClusterMetadata) ([]t
 			}
 		}
 		if err = errGroup.Wait(); err != nil {
-			log.Error("Errors found during execution, skipping sample")
+			log.Errorf("Errors found during execution, skipping sample: %s", err)
 			continue
 		}
 		genResultSummary(&result)

diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go
@@ -32,6 +32,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/dynamic"
@@ -225,7 +226,7 @@ func reconcileNs(cfg config.Config) error {
 		if err != nil {
 			return err
 		}
-		if *d.Spec.Replicas == replicas {
+		if d.Status.ReadyReplicas == replicas {
 			return nil
 		}
 		deployment.Spec.Replicas = &replicas
@@ -243,9 +244,11 @@ func reconcileNs(cfg config.Config) error {
 
 func waitForDeployment(ns, deployment string, maxWaitTimeout time.Duration) error {
 	var errMsg string
+	var dep *appsv1.Deployment
+	var err error
 	log.Infof("Waiting for replicas from deployment %s in ns %s to be ready", deployment, ns)
-	err := wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (bool, error) {
-		dep, err := clientSet.AppsV1().Deployments(ns).Get(context.TODO(), deployment, metav1.GetOptions{})
+	err = wait.PollUntilContextTimeout(context.TODO(), time.Second, maxWaitTimeout, true, func(ctx context.Context) (bool, error) {
+		dep, err = clientSet.AppsV1().Deployments(ns).Get(context.TODO(), deployment, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -259,6 +262,17 @@ func waitForDeployment(ns, deployment string, maxWaitTimeout time.Duration) erro
 	})
 	if err != nil && errMsg != "" {
 		log.Error(errMsg)
+		failedPods, _ := clientSet.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{
+			FieldSelector: "status.phase=Pending",
+			LabelSelector: labels.SelectorFromSet(dep.Spec.Selector.MatchLabels).String(),
+		})
+		for _, pod := range failedPods.Items {
+			for _, cs := range pod.Status.ContainerStatuses {
+				if cs.State.Waiting != nil {
+					log.Errorf("%v@%v: %v", pod.Name, pod.Spec.NodeName, cs.State.Waiting.Message)
+				}
+			}
+		}
 	}
 	return err
 }