add smtp retries (#4115)

* add smtp retries

* use existing retry package instead

* clean up commented code

* treat any error from smtp SendMail as temporary

* only treat 4xx series SMTP errors as temporary

* make sure err is nil checked

---------

Co-authored-by: Jordan Nimlos <[email protected]>
Co-authored-by: nimjor <[email protected]>

remotemonitor/config.go

@@ -35,6 +35,9 @@ type Config struct {
 		// ServerAddr is the address of the SMTP server, including port.
 		ServerAddr string
 		User, Pass string
+
+		// Retries is the number of times to retry sending with an email integration key.
+		Retries int
 	}
 
 	// Instances determine what remote GoAlert instances will be monitored and send potential errors.
@@ -94,6 +97,9 @@ func (cfg Config) Validate() error {
 	if cfg.SMTP.From == "" {
 		return errors.New("SMTP from address is required")
 	}
+	if cfg.SMTP.Retries > 5 || cfg.SMTP.Retries < 0 {
+		return errors.New("SMTP retries must be between 0 and 5")
+	}
 
 	return nil
 }

remotemonitor/monitor.go

@@ -2,7 +2,6 @@ package remotemonitor
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -14,8 +13,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/target/goalert/config"
 	"github.com/target/goalert/notification/twilio"
+	"github.com/target/goalert/retry"
 )
 
 // Monitor will check for functionality and communication between itself and one or more instances.
@@ -187,12 +188,22 @@ func (m *Monitor) createEmailAlert(i Instance, dedup, summary, details string) e
 		auth = smtp.PlainAuth("", m.cfg.SMTP.User, m.cfg.SMTP.Pass, host)
 	}
 
-	err = smtp.SendMail(m.cfg.SMTP.ServerAddr, auth, m.cfg.SMTP.From, []string{addr.Address}, []byte(msg))
-	if err != nil {
-		return fmt.Errorf("send email: %w", err)
-	}
-
-	return nil
+	err = retry.DoTemporaryError(func(_ int) error {
+		err = smtp.SendMail(m.cfg.SMTP.ServerAddr, auth, m.cfg.SMTP.From, []string{addr.Address}, []byte(msg))
+		if err == nil {
+			return nil
+		}
+		if strings.HasPrefix(err.Error(), "4") { // SMTP server return codes beginning with 4 are considered transient
+			err = retry.TemporaryError(err)
+		}
+		return errors.Wrap(err, "send email")
+	},
+		retry.Log(m.context()),
+		retry.Limit(m.cfg.SMTP.Retries),
+		retry.FibBackoff(time.Second),
+	)
+
+	return err
 }
 
 func (m *Monitor) loop() {