Add retry for rpaas resources

tsuru · May 20, 2021 · faad3c9 · faad3c9
1 parent b309092
commit faad3c9
Show file tree

Hide file tree

Showing 4 changed files with 52 additions and 13 deletions.
diff --git a/rpaas/provider.go b/rpaas/provider.go
@@ -7,14 +7,17 @@ package rpaas
 import (
 	"context"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/sirupsen/logrus"
 	"github.com/tsuru/tsuru/cmd"
 	"istio.io/pkg/log"
 
+	"github.com/tsuru/rpaas-operator/pkg/rpaas/client"
 	rpaas_client "github.com/tsuru/rpaas-operator/pkg/rpaas/client"
 )
 
@@ -109,3 +112,21 @@ func providerConfigure(ctx context.Context, d *schema.ResourceData, terraformVer
 
 	return p, nil
 }
+
+func rpaasRetry(ctx context.Context, d *schema.ResourceData, retryFunc func() error) error {
+	return resource.RetryContext(ctx, d.Timeout(schema.TimeoutCreate), func() *resource.RetryError {
+		err := retryFunc()
+
+		if err == nil {
+			return nil
+		}
+
+		if errUnexpected, ok := err.(*client.ErrUnexpectedStatusCode); ok {
+			if strings.Contains(errUnexpected.Body, "event locked") {
+				return resource.RetryableError(err)
+			}
+		}
+
+		return resource.NonRetryableError(err)
+	})
+}
diff --git a/rpaas/resource_rpaas_autoscale.go b/rpaas/resource_rpaas_autoscale.go
@@ -83,7 +83,10 @@ func resourceRpaasAutoscaleCreate(ctx context.Context, d *schema.ResourceData, m
 		args.CPU = pointerToInt32(int32(v.(int)))
 	}
 
-	err = rpaasClient.UpdateAutoscale(ctx, args)
+	err = rpaasRetry(ctx, d, func() error {
+		return rpaasClient.UpdateAutoscale(ctx, args)
+	})
+
 	if err != nil {
 		return diag.Errorf("Unable to create autoscale for instance %s: %v", instance, err)
 	}
@@ -148,9 +151,11 @@ func resourceRpaasAutoscaleUpdate(ctx context.Context, d *schema.ResourceData, m
 		args.CPU = pointerToInt32(int32(v.(int)))
 	}
 
-	err = rpaasClient.UpdateAutoscale(ctx, args)
+	err = rpaasRetry(ctx, d, func() error {
+		return rpaasClient.UpdateAutoscale(ctx, args)
+	})
 	if err != nil {
-		return diag.Errorf("Unable to create autoscale for instance %s: %v", instance, err)
+		return diag.Errorf("Unable to update autoscale for instance %s: %v", instance, err)
 	}
 	return nil
 }
@@ -165,9 +170,12 @@ func resourceRpaasAutoscaleDelete(ctx context.Context, d *schema.ResourceData, m
 		return diag.Errorf("Unable to create client for service %s: %v", serviceName, err)
 	}
 
-	err = rpaasClient.RemoveAutoscale(ctx, rpaas_client.RemoveAutoscaleArgs{
-		Instance: instance,
+	err = rpaasRetry(ctx, d, func() error {
+		return rpaasClient.RemoveAutoscale(ctx, rpaas_client.RemoveAutoscaleArgs{
+			Instance: instance,
+		})
 	})
+
 	if err != nil {
 		return diag.Errorf("Unable to remove autoscale for instance %s: %v", instance, err)
 	}

diff --git a/rpaas/resource_rpaas_block.go b/rpaas/resource_rpaas_block.go
@@ -80,7 +80,10 @@ func resourceRpaasBlockCreate(ctx context.Context, d *schema.ResourceData, meta
 		Content:  content,
 	}
 
-	err = rpaasClient.UpdateBlock(ctx, args)
+	err = rpaasRetry(ctx, d, func() error {
+		return rpaasClient.UpdateBlock(ctx, args)
+	})
+
 	if err != nil {
 		return diag.Errorf("Unable to create/update block %s for instance %s: %v", blockName, instance, err)
 	}
@@ -128,9 +131,11 @@ func resourceRpaasBlockDelete(ctx context.Context, d *schema.ResourceData, meta
 		return diag.Errorf("Unable to create client for service %s: %v", serviceName, err)
 	}
 
-	err = rpaasClient.DeleteBlock(ctx, rpaas_client.DeleteBlockArgs{
-		Instance: instance,
-		Name:     blockName,
+	err = rpaasRetry(ctx, d, func() error {
+		return rpaasClient.DeleteBlock(ctx, rpaas_client.DeleteBlockArgs{
+			Instance: instance,
+			Name:     blockName,
+		})
 	})
 
 	if err != nil {

diff --git a/rpaas/resource_rpaas_route.go b/rpaas/resource_rpaas_route.go
@@ -92,7 +92,10 @@ func resourceRpaasRouteCreate(ctx context.Context, d *schema.ResourceData, meta
 		args.Destination = destination.(string)
 	}
 
-	err = rpaasClient.UpdateRoute(ctx, args)
+	err = rpaasRetry(ctx, d, func() error {
+		return rpaasClient.UpdateRoute(ctx, args)
+	})
+
 	if err != nil {
 		return diag.Errorf("Unable to create/update route %s for instance %s: %v", path, instance, err)
 	}
@@ -146,9 +149,11 @@ func resourceRpaasRouteDelete(ctx context.Context, d *schema.ResourceData, meta
 		return diag.Errorf("Unable to create client for service %s: %v", serviceName, err)
 	}
 
-	err = rpaasClient.DeleteRoute(ctx, rpaas_client.DeleteRouteArgs{
-		Instance: instance,
-		Path:     path,
+	err = rpaasRetry(ctx, d, func() error {
+		return rpaasClient.DeleteRoute(ctx, rpaas_client.DeleteRouteArgs{
+			Instance: instance,
+			Path:     path,
+		})
 	})
 
 	if err != nil {