use pointer to update token

internal/tfhandler/qworker.go

@@ -45,47 +45,54 @@ MainLoop:
 			continue
 		}
 
+		failureRequeueRate := 100 * time.Second
+
 		result, err := i.clientset.Cluster(i.clusterName).Poll(namespace, name).Read(ctx, &tf)
 		if err != nil {
-			log.Println(err)
-			i.requeueAfter(tf, 30*time.Second)
+			log.Printf(".... ERROR: %s \t(%s/%s)", err, namespace, name)
+			i.requeueAfter(tf, failureRequeueRate, false)
+			continue
+		}
+
+		if result == nil {
+			log.Printf(".... API did not return results \t(%s/%s)", namespace, name)
+			i.requeueAfter(tf, failureRequeueRate, false)
 			continue
 		}
 
 		if strings.Contains(result.Data.StatusInfo.Message, "workflow has not completed") {
-			i.requeueAfter(tf, 30*time.Second)
+			i.requeueAfter(tf, 30*time.Second, true)
 			continue
 		}
 
 		if !result.IsSuccess {
 			log.Printf(".... %s \t(%s/%s)", result.ErrMsg, namespace, name)
-			if strings.Contains(result.ErrMsg, fmt.Sprintf(`terraforms.tf.galleybytes.com "%s" not found`, name)) {
-				// Do not requeue since this resource is not even registered with the API
-				continue
-			}
-			i.requeueAfter(tf, 30*time.Second)
+			// Even after a failure, continue to check for a success. There is a possibility a debug session
+			// may fix the session. Slowing down requeue rate.
+			i.requeueAfter(tf, failureRequeueRate, false)
 			continue
 		}
 
-		// The result returned from the API was successful. Validate the data before removing from queue
 		list, ok := result.Data.Data.([]interface{})
+		// In the following results, if validate the API response structure occurs, there might be
+		// breaking changes in either this client or in the API. Requeue and hope it was a blip.
 		if !ok {
-			log.Printf("ERROR api response in unexpected format %T \t(%s/%s)", result.Data.Data, namespace, name)
-			i.requeueAfter(tf, 30*time.Second)
+			log.Printf(".... ERROR api response in unexpected format %T \t(%s/%s)", result.Data.Data, namespace, name)
+			i.requeueAfter(tf, failureRequeueRate, false)
 			continue
 		}
 
 		for _, item := range list {
 			_, ok := item.(string)
 			if !ok {
-				log.Printf("ERROR api response item in unexpected format %T \t(%s/%s)", item, namespace, name)
-				i.requeueAfter(tf, 30*time.Second)
+				log.Printf(".... ERROR api response item in unexpected format %T \t(%s/%s)", item, namespace, name)
+				i.requeueAfter(tf, failureRequeueRate, false)
 				continue MainLoop
 			}
 			_, err := base64.StdEncoding.DecodeString(item.(string))
 			if err != nil {
-				log.Printf("ERROR api response item cannot be decoded \t(%s/%s)", namespace, name)
-				i.requeueAfter(tf, 30*time.Second)
+				log.Printf(".... ERROR api response item cannot be decoded \t(%s/%s)", namespace, name)
+				i.requeueAfter(tf, failureRequeueRate, false)
 				continue MainLoop
 			}
 		}
@@ -125,10 +132,12 @@ func shouldPoll(tf tfv1beta1.Terraform) bool {
 	return tf.Spec.OutputsSecret != ""
 }
 
-func (i informer) requeueAfter(tf tfv1beta1.Terraform, t time.Duration) {
+func (i informer) requeueAfter(tf tfv1beta1.Terraform, t time.Duration, showGenericWaitingMsg bool) {
 	go func() {
 		time.Sleep(t)
 		i.queue.PushBack(tf)
 	}()
-	log.Printf(".... Waiting for workflow completion. \t(%s/%s)", tf.Namespace, tf.Name)
+	if showGenericWaitingMsg {
+		log.Printf(".... Waiting for workflow completion. \t(%s/%s)", tf.Namespace, tf.Name)
+	}
 }

internal/tfhandler/tfhandler.go

@@ -183,7 +183,7 @@ func (i informer) createAndUpdate(tf *tfv1beta1.Terraform) {
 	for _, action := range []string{"CREATE", "UPDATE"} {
 		result := eventQueryRetrier(i.clientset.Cluster(i.clusterName).Event(), action, tf)
 		if result == nil {
-			log.Println("An unknown error has occurred")
+			log.Printf("Unknown error occurred in %s \t(%s/%s)\n", action, tf.Namespace, tf.Name)
 			return
 		}
 		if !result.next {

pkg/tfoapiclient/client.go

@@ -54,7 +54,7 @@ type config struct {
 
 type Clientset struct {
 	http.Client
-	config config
+	config *config
 }
 
 type ClientSetup struct {
@@ -70,7 +70,7 @@ func NewClientset(host, username, password string, insecureSkipVerify bool) (*Cl
 	client := http.Client{Transport: tr}
 	tfoapiClientset := Clientset{
 		Client: client,
-		config: config{
+		config: &config{
 			Host:     host,
 			Username: username,
 			Password: password,