feat(nvidia/error-xid-sxid): new component based on persistent xid, sxid event history

components/accelerator/nvidia/bad-envs/component.go

@@ -17,12 +17,12 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, bad_envs_id.Name)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, bad_envs_id.Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/clock-speed/component.go

@@ -22,12 +22,13 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/clock/component.go

@@ -22,12 +22,13 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/ecc/component.go

@@ -22,12 +22,13 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/error-xid-sxid/component.go

@@ -0,0 +1,128 @@
+// Package errorxidsxid implements NVIDIA GPU driver Xid/SXid error detector.
+package errorxidsxid
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/leptonai/gpud/components"
+	nvidia_error_xid_sxid_id "github.com/leptonai/gpud/components/accelerator/nvidia/error-xid-sxid/id"
+	nvidia_query "github.com/leptonai/gpud/components/accelerator/nvidia/query"
+	nvidia_xid_sxid_state "github.com/leptonai/gpud/components/accelerator/nvidia/query/xid-sxid-state"
+	"github.com/leptonai/gpud/components/query"
+	"github.com/leptonai/gpud/log"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func New(ctx context.Context, cfg Config) components.Component {
+	cfg.Query.SetDefaultsIfNotSet()
+
+	// this starts the Xid poller via "nvml.StartDefaultInstance"
+	cctx, ccancel := context.WithCancel(ctx)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, nvidia_error_xid_sxid_id.Name)
+
+	return &component{
+		rootCtx: ctx,
+		cancel:  ccancel,
+		poller:  nvidia_query.GetDefaultPoller(),
+		db:      cfg.Query.State.DB,
+	}
+}
+
+var _ components.Component = (*component)(nil)
+
+type component struct {
+	rootCtx context.Context
+	cancel  context.CancelFunc
+	poller  query.Poller
+	db      *sql.DB
+}
+
+func (c *component) Name() string { return nvidia_error_xid_sxid_id.Name }
+
+func (c *component) States(ctx context.Context) ([]components.State, error) {
+	return nil, nil
+}
+
+const (
+	EventNameErroXid  = "error_xid"
+	EventNameErroSXid = "error_sxid"
+
+	EventKeyUnixSeconds    = "unix_seconds"
+	EventKeyData           = "data"
+	EventKeyEncoding       = "encoding"
+	EventValueEncodingJSON = "json"
+)
+
+func (c *component) Events(ctx context.Context, since time.Time) ([]components.Event, error) {
+	events, err := nvidia_xid_sxid_state.ReadEvents(ctx, c.db, nvidia_xid_sxid_state.WithSince(since))
+	if err != nil {
+		return nil, err
+	}
+
+	convertedEvents := make([]components.Event, 0, len(events))
+	for _, event := range events {
+		if xidDetail := event.ToXidDetail(); xidDetail != nil {
+			msg := fmt.Sprintf("xid %d detected by %s (%s)",
+				event.EventID,
+				event.DataSource,
+				humanize.Time(time.Unix(event.UnixSeconds, 0)),
+			)
+			xidBytes, _ := xidDetail.JSON()
+
+			convertedEvents = append(convertedEvents, components.Event{
+				Time:    metav1.Time{Time: time.Unix(event.UnixSeconds, 0)},
+				Name:    EventNameErroXid,
+				Message: msg,
+				ExtraInfo: map[string]string{
+					EventKeyUnixSeconds: strconv.FormatInt(event.UnixSeconds, 10),
+					EventKeyData:        string(xidBytes),
+					EventKeyEncoding:    EventValueEncodingJSON,
+				},
+			})
+			continue
+		}
+
+		if sxidDetail := event.ToSXidDetail(); sxidDetail != nil {
+			msg := fmt.Sprintf("sxid %d detected by %s (%s)",
+				event.EventID,
+				event.DataSource,
+				humanize.Time(time.Unix(event.UnixSeconds, 0)),
+			)
+			sxidBytes, _ := sxidDetail.JSON()
+
+			convertedEvents = append(convertedEvents, components.Event{
+				Time:    metav1.Time{Time: time.Unix(event.UnixSeconds, 0)},
+				Name:    EventNameErroSXid,
+				Message: msg,
+				ExtraInfo: map[string]string{
+					EventKeyUnixSeconds: strconv.FormatInt(event.UnixSeconds, 10),
+					EventKeyData:        string(sxidBytes),
+					EventKeyEncoding:    EventValueEncodingJSON,
+				},
+			})
+			continue
+		}
+	}
+	return convertedEvents, nil
+}
+
+func (c *component) Metrics(ctx context.Context, since time.Time) ([]components.Metric, error) {
+	log.Logger.Debugw("querying metrics", "since", since)
+
+	return nil, nil
+}
+
+func (c *component) Close() error {
+	log.Logger.Debugw("closing component")
+
+	// safe to call stop multiple times
+	_ = c.poller.Stop(nvidia_error_xid_sxid_id.Name)
+
+	return nil
+}

components/accelerator/nvidia/error-xid-sxid/config.go

@@ -0,0 +1,32 @@
+package errorxidsxid
+
+import (
+	"database/sql"
+	"encoding/json"
+
+	query_config "github.com/leptonai/gpud/components/query/config"
+)
+
+type Config struct {
+	Query query_config.Config `json:"query"`
+}
+
+func ParseConfig(b any, db *sql.DB) (*Config, error) {
+	raw, err := json.Marshal(b)
+	if err != nil {
+		return nil, err
+	}
+	cfg := new(Config)
+	err = json.Unmarshal(raw, cfg)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.Query.State != nil {
+		cfg.Query.State.DB = db
+	}
+	return cfg, nil
+}
+
+func (cfg Config) Validate() error {
+	return nil
+}

components/accelerator/nvidia/error-xid-sxid/id/id.go

@@ -0,0 +1,4 @@
+// Package id is the identifier for the nvidia error xid sxid component.
+package id
+
+const Name = "accelerator-nvidia-error-xid-sxid"

components/accelerator/nvidia/error/component.go

@@ -18,12 +18,13 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/fabric-manager/component.go

@@ -21,7 +21,8 @@ func New(ctx context.Context, cfg Config) (components.Component, error) {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Log.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	if err := cfg.Log.Validate(); err != nil {
 		ccancel()
@@ -38,7 +39,7 @@ func New(ctx context.Context, cfg Config) (components.Component, error) {
 	return &component{
 		rootCtx:   ctx,
 		cancel:    ccancel,
-		poller:    nvidia_query.DefaultPoller,
+		poller:    nvidia_query.GetDefaultPoller(),
 		logPoller: fabric_manager_log.GetDefaultPoller(),
 	}, nil
 }

components/accelerator/nvidia/fabric-manager/component_test.go

@@ -6,10 +6,11 @@ import (
 	"testing"
 	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	query_config "github.com/leptonai/gpud/components/query/config"
 	query_log_config "github.com/leptonai/gpud/components/query/log/config"
+	"github.com/leptonai/gpud/pkg/sqlite"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestComponentLog(t *testing.T) {
@@ -28,13 +29,22 @@ func TestComponentLog(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	db, err := sqlite.Open(":memory:")
+	if err != nil {
+		t.Fatalf("failed to open database: %v", err)
+	}
+	defer db.Close()
+
 	pollInterval := 3 * time.Second
 	component, err := New(
 		ctx,
 		Config{
 			Log: query_log_config.Config{
 				Query: query_config.Config{
 					Interval: metav1.Duration{Duration: pollInterval},
+					State: &query_config.State{
+						DB: db,
+					},
 				},
 				BufferSize:    query_log_config.DefaultBufferSize,
 				File:          f.Name(),

components/accelerator/nvidia/gsp-firmware-mode/component.go

@@ -17,12 +17,12 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, nvidia_gsp_firmware_mode_id.Name)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, nvidia_gsp_firmware_mode_id.Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/infiniband/component.go

@@ -19,12 +19,13 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/info/component.go

@@ -18,12 +18,13 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/memory/component.go

@@ -22,12 +22,13 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/nccl/component.go

@@ -20,12 +20,12 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, nvidia_nccl_id.Name)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, nvidia_nccl_id.Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }
 

components/accelerator/nvidia/nvlink/component.go

@@ -22,12 +22,13 @@ func New(ctx context.Context, cfg Config) components.Component {
 	cfg.Query.SetDefaultsIfNotSet()
 
 	cctx, ccancel := context.WithCancel(ctx)
-	nvidia_query.DefaultPoller.Start(cctx, cfg.Query, Name)
+	nvidia_query.SetDefaultPoller(cfg.Query.State.DB)
+	nvidia_query.GetDefaultPoller().Start(cctx, cfg.Query, Name)
 
 	return &component{
 		rootCtx: ctx,
 		cancel:  ccancel,
-		poller:  nvidia_query.DefaultPoller,
+		poller:  nvidia_query.GetDefaultPoller(),
 	}
 }