Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

heartbeats: Add support for specifying additional details #3559

Merged
merged 15 commits into from
Jan 25, 2024
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion engine/heartbeatmanager/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func NewDB(ctx context.Context, db *sql.DB, a *alert.Store) (*DB, error) {
set last_state = 'unhealthy'
from rows
where mon.id = rows.id
returning mon.id, name, service_id, last_heartbeat
returning mon.id, name, service_id, last_heartbeat, coalesce(additional_details, '')
`),
fetchHealthy: p.P(`
with rows as (
Expand Down
10 changes: 8 additions & 2 deletions engine/heartbeatmanager/process.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/target/goalert/permission"
"github.com/target/goalert/util/log"
"github.com/target/goalert/util/sqlutil"
"github.com/target/goalert/validation/validate"
)

// UpdateAll will process all heartbeats opening and closing alerts as needed.
Expand Down Expand Up @@ -38,9 +39,13 @@ func (db *DB) processAll(ctx context.Context) error {
return errors.Wrap(err, "fetch unhealthy heartbeats")
}
for _, row := range bad {
details := "Last heartbeat: " + row.LastHeartbeat.Format(time.UnixDate)
if row.AddlDetails != "" {
details += "\n\n" + row.AddlDetails
}
a, isNew, err := db.alertStore.CreateOrUpdateTx(row.Context(ctx), tx, &alert.Alert{
Summary: fmt.Sprintf("Heartbeat monitor '%s' expired.", row.Name),
Details: "Last heartbeat: " + row.LastHeartbeat.Format(time.UnixDate),
Details: validate.SanitizeText(details, alert.MaxDetailsLength),
Status: alert.StatusTriggered,
ServiceID: row.ServiceID,
Dedup: &alert.DedupID{
Expand Down Expand Up @@ -97,6 +102,7 @@ type row struct {
Name string
ServiceID string
LastHeartbeat time.Time
AddlDetails string
}

func (r row) Context(ctx context.Context) context.Context {
Expand All @@ -115,7 +121,7 @@ func (db *DB) unhealthy(ctx context.Context, tx *sql.Tx) ([]row, error) {
var result []row
for rows.Next() {
var r row
err = rows.Scan(&r.ID, &r.Name, &r.ServiceID, &r.LastHeartbeat)
err = rows.Scan(&r.ID, &r.Name, &r.ServiceID, &r.LastHeartbeat, &r.AddlDetails)
if err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions gadb/models.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

127 changes: 118 additions & 9 deletions graphql2/generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 14 additions & 3 deletions graphql2/graphqlapp/heartbeatmonitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ func (a *HeartbeatMonitor) Href(ctx context.Context, hb *heartbeat.Monitor) (str
cfg := config.FromContext(ctx)
return cfg.CallbackURL("/api/v2/heartbeat/" + url.PathEscape(hb.ID)), nil
}
func (a *HeartbeatMonitor) AdditionalDetails(ctx context.Context, hb *heartbeat.Monitor) (string, error) {
return hb.AdditionalDetails, nil
}

func (q *Query) HeartbeatMonitor(ctx context.Context, id string) (*heartbeat.Monitor, error) {
return (*App)(q).FindOneHeartbeatMonitor(ctx, id)
Expand All @@ -33,10 +36,15 @@ func (m *Mutation) CreateHeartbeatMonitor(ctx context.Context, input graphql2.Cr
serviceID = *input.ServiceID
}
err = withContextTx(ctx, m.DB, func(ctx context.Context, tx *sql.Tx) error {
var details string
if input.AdditionalDetails != nil {
details = *input.AdditionalDetails
}
hb = &heartbeat.Monitor{
ServiceID: serviceID,
Name: input.Name,
Timeout: time.Duration(input.TimeoutMinutes) * time.Minute,
ServiceID: serviceID,
Name: input.Name,
Timeout: time.Duration(input.TimeoutMinutes) * time.Minute,
AdditionalDetails: details,
}
hb, err = m.HeartbeatStore.CreateTx(ctx, tx, hb)
return err
Expand All @@ -56,6 +64,9 @@ func (m *Mutation) UpdateHeartbeatMonitor(ctx context.Context, input graphql2.Up
if input.TimeoutMinutes != nil {
hb.Timeout = time.Duration(*input.TimeoutMinutes) * time.Minute
}
if input.AdditionalDetails != nil {
hb.AdditionalDetails = *input.AdditionalDetails
}

return m.HeartbeatStore.UpdateTx(ctx, tx, hb)
})
Expand Down
14 changes: 8 additions & 6 deletions graphql2/models_gen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions graphql2/schema.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -971,12 +971,14 @@ input CreateHeartbeatMonitorInput {
serviceID: ID
name: String!
timeoutMinutes: Int!
additionalDetails: String
}

input UpdateHeartbeatMonitorInput {
id: ID!
name: String
timeoutMinutes: Int
additionalDetails: String
}

enum HeartbeatMonitorState {
Expand All @@ -993,6 +995,7 @@ type HeartbeatMonitor {
lastState: HeartbeatMonitorState!
lastHeartbeat: ISOTimestamp
href: String!
additionalDetails: String!
}

type Label {
Expand Down
6 changes: 5 additions & 1 deletion heartbeat/monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"time"

"github.com/jackc/pgtype"
"github.com/target/goalert/alert"
"github.com/target/goalert/util/sqlutil"
"github.com/target/goalert/validation/validate"
)
Expand All @@ -15,6 +16,8 @@ type Monitor struct {
ServiceID string `json:"service_id,omitempty"`
Timeout time.Duration `json:"timeout,omitempty"`

AdditionalDetails string

lastState State
lastHeartbeat time.Time
}
Expand All @@ -31,6 +34,7 @@ func (m Monitor) Normalize() (*Monitor, error) {
validate.UUID("ServiceID", m.ServiceID),
validate.IDName("Name", m.Name),
validate.Duration("Timeout", m.Timeout, 5*time.Minute, 9000*time.Hour),
validate.Text("AdditionalDetails", m.AdditionalDetails, 0, alert.MaxDetailsLength),
)
if err != nil {
return nil, err
Expand All @@ -47,7 +51,7 @@ func (m *Monitor) scanFrom(scanFn func(...interface{}) error) error {
timeout pgtype.Interval
)

err := scanFn(&m.ID, &m.Name, &m.ServiceID, &timeout, &m.lastState, &t)
err := scanFn(&m.ID, &m.Name, &m.ServiceID, &timeout, &m.lastState, &t, &m.AdditionalDetails)
if err != nil {
return err
}
Expand Down
Loading
Loading