Merge pull request #48 from buildkite/delete-queue-before-completing-…

…lifecycle

Delete queue before completing lifecycle

daemon.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"os"
 	"os/exec"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -45,10 +46,11 @@ func (d *Daemon) Start(ctx context.Context) error {
 	if err := d.Queue.Create(); err != nil {
 		return err
 	}
+
+	// ensure the queue deletion happens only once
+	var deleteOnce sync.Once
 	defer func() {
-		if err := d.Queue.Delete(); err != nil {
-			log.WithError(err).Error("Failed to delete queue")
-		}
+		deleteOnce.Do(d.deleteQueue)
 	}()
 
 	if err := d.Queue.Subscribe(); err != nil {
@@ -92,7 +94,10 @@ func (d *Daemon) Start(ctx context.Context) error {
 				continue
 			}
 
-			d.handleMessage(ctx, msg)
+			d.handleMessage(ctx, msg, func() {
+				// delete the queue before we complete
+				deleteOnce.Do(d.deleteQueue)
+			})
 		}
 	}()
 
@@ -114,15 +119,14 @@ func (d *Daemon) Start(ctx context.Context) error {
 			}
 
 			executeCtx.Info("Handler finished successfully")
-
 		}
 	}()
 
 	log.Info("Listening for lifecycle notifications")
 	return d.Queue.Receive(ctx, ch)
 }
 
-func (d *Daemon) handleMessage(ctx context.Context, m AutoscalingMessage) {
+func (d *Daemon) handleMessage(ctx context.Context, m AutoscalingMessage, complete func()) {
 	logCtx := log.WithFields(log.Fields{
 		"transition": m.Transition,
 		"instanceid": m.InstanceID,
@@ -160,6 +164,8 @@ func (d *Daemon) handleMessage(ctx context.Context, m AutoscalingMessage) {
 
 	executeLogCtx.Info("Handler finished successfully")
 
+	complete()
+
 	if err = completeLifecycle(d.AutoScaling, m); err != nil {
 		logCtx.WithError(err).Error("Failed to complete lifecycle action")
 		return
@@ -168,6 +174,12 @@ func (d *Daemon) handleMessage(ctx context.Context, m AutoscalingMessage) {
 	logCtx.Info("Lifecycle action completed successfully")
 }
 
+func (d *Daemon) deleteQueue() {
+	if err := d.Queue.Delete(); err != nil {
+		log.WithError(err).Error("Failed to delete queue")
+	}
+}
+
 func executeHandler(ctx context.Context, command *os.File, args []string) error {
 	cmd := exec.CommandContext(ctx, command.Name(), args...)
 	cmd.Env = os.Environ()