diff --git a/cmd/root.go b/cmd/root.go index cea0b059..cb4069e5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,18 +2,18 @@ package cmd import ( "fmt" + "log" "os" "github.com/metal-toolbox/firmware-syncer/internal/app" + "github.com/metal-toolbox/firmware-syncer/pkg/types" "github.com/spf13/cobra" ) var ( - debug bool - trace bool - dryRun bool - cfgFile string - logLevel int // 0 - info, 1 - debug, 2 - trace + cfgFile string + inventoryKind string + logLevel string ) // rootCmd represents the base command when called without any subcommands @@ -26,15 +26,13 @@ var rootCmd = &cobra.Command{ os.Exit(1) } - syncerApp, err := app.New(cfgFile, logLevel) + syncerApp, err := app.New(cmd.Context(), types.InventoryKind(inventoryKind), cfgFile, logLevel) if err != nil { - fmt.Println(err) - os.Exit(1) + log.Fatal(err) } - err = syncerApp.SyncFirmwares(cmd.Context(), dryRun) + err = syncerApp.SyncFirmwares(cmd.Context()) if err != nil { - fmt.Println(err) - os.Exit(1) + syncerApp.Logger.Fatal(err) } }, } @@ -52,22 +50,8 @@ func Execute() { } } -func setLogLevel() { - logLevel = app.LogLevelInfo - - if debug { - logLevel = app.LogLevelDebug - } - - if trace { - logLevel = app.LogLevelTrace - } -} - func init() { - cobra.OnInitialize(setLogLevel) - rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Enable debug logging (can be used with --trace)") - rootCmd.PersistentFlags().BoolVarP(&trace, "trace", "t", false, "Enable trace logging (can be used with --debug)") + rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "set logging level - debug, trace") rootCmd.PersistentFlags().StringVarP(&cfgFile, "config-file", "c", "", "Syncer configuration file") - rootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "", false, "Don't sync anything, just initialize") + rootCmd.PersistentFlags().StringVar(&inventoryKind, "inventory", "serverservice", "Inventory to publish firmwares.") } diff --git a/internal/app/app.go b/internal/app/app.go index fdc265f5..5a0396bc 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -2,11 +2,18 @@ package app import ( "context" + "net/url" "os" + "strings" "github.com/bmc-toolbox/common" + "github.com/jeremywohl/flatten" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/spf13/viper" + runtime "github.com/banzaicloud/logrus-runtime-formatter" "github.com/metal-toolbox/firmware-syncer/internal/config" "github.com/metal-toolbox/firmware-syncer/internal/vendors" "github.com/metal-toolbox/firmware-syncer/internal/vendors/asrockrack" @@ -15,138 +22,337 @@ import ( "github.com/metal-toolbox/firmware-syncer/internal/vendors/intel" "github.com/metal-toolbox/firmware-syncer/internal/vendors/mellanox" "github.com/metal-toolbox/firmware-syncer/internal/vendors/supermicro" + "github.com/metal-toolbox/firmware-syncer/pkg/types" ) -var ( - LogLevelInfo = 0 - LogLevelDebug = 1 - LogLevelTrace = 2 -) - -type Syncer struct { - dryRun bool - config *config.Syncer - logger *logrus.Logger +// App holds attributes for the firmware-syncer application +type App struct { + // Viper loads configuration parameters. + v *viper.Viper + // firmware-syncer configuration. + Config *config.Configuration + // Logger is the app logger + Logger *logrus.Logger vendors []vendors.Vendor } -// nolint:gocyclo // silence cyclo warning for now until function can be re-worked -// New returns a Syncer object configured with Providers -func New(configFile string, logLevel int) (*Syncer, error) { - // Setup logger - var logger = logrus.New() - logger.Out = os.Stdout - - switch logLevel { - case LogLevelDebug: - logger.SetLevel(logrus.DebugLevel) - case LogLevelTrace: - logger.SetLevel(logrus.TraceLevel) +// nolint:gocyclo // Instantiating new app is cyclomatic +// New returns a new instance of the firmware-syncer app +func New(ctx context.Context, inventoryKind types.InventoryKind, cfgFile, logLevel string) (*App, error) { + app := &App{ + v: viper.New(), + Config: &config.Configuration{}, + Logger: logrus.New(), + } + if err := app.LoadConfiguration(cfgFile, inventoryKind); err != nil { + return nil, err + } + + switch types.LogLevel(logLevel) { + case types.LogLevelDebug: + app.Logger.Level = logrus.DebugLevel + case types.LogLevelTrace: + app.Logger.Level = logrus.TraceLevel default: - logger.SetLevel(logrus.InfoLevel) + app.Logger.Level = logrus.InfoLevel } - // Load configuration - cfgSyncer, err := config.LoadSyncerConfig(configFile) - if err != nil { - logger.Error(err.Error()) - return nil, err + runtimeFormatter := &runtime.Formatter{ + ChildFormatter: &logrus.JSONFormatter{}, + File: true, + Line: true, + BaseNameOnly: true, } + app.Logger.SetFormatter(runtimeFormatter) + // Load firmware manifest - firmwaresByVendor, err := config.LoadFirmwareManifest(context.TODO(), cfgSyncer.FirmwareManifestURL) + firmwaresByVendor, err := config.LoadFirmwareManifest(ctx, app.Config.FirmwareManifestURL) if err != nil { - logger.Error(err.Error()) + app.Logger.Error(err.Error()) return nil, err } - var fwVendors []vendors.Vendor - for vendor, firmwares := range firmwaresByVendor { switch vendor { case common.VendorDell: var dup vendors.Vendor - dup, err = dell.NewDUP(context.TODO(), firmwares, cfgSyncer, logger) + dup, err = dell.NewDUP(ctx, firmwares, app.Config, app.Logger) if err != nil { - logger.Error("Failed to initialize Dell vendor: " + err.Error()) + app.Logger.Error("Failed to initialize Dell vendor: " + err.Error()) return nil, err } - fwVendors = append(fwVendors, dup) + app.vendors = append(app.vendors, dup) case common.VendorAsrockrack: var asrr vendors.Vendor - asrr, err = asrockrack.New(context.TODO(), firmwares, cfgSyncer, logger) + asrr, err = asrockrack.New(ctx, firmwares, app.Config, app.Logger) if err != nil { - logger.Error("Failed to initialize ASRockRack vendor:" + err.Error()) + app.Logger.Error("Failed to initialize ASRockRack vendor:" + err.Error()) return nil, err } - fwVendors = append(fwVendors, asrr) + app.vendors = append(app.vendors, asrr) case common.VendorSupermicro: var sm vendors.Vendor - sm, err = supermicro.New(context.TODO(), firmwares, cfgSyncer, logger) + sm, err = supermicro.New(ctx, firmwares, app.Config, app.Logger) if err != nil { - logger.Error("Failed to initialize Supermicro vendor: " + err.Error()) + app.Logger.Error("Failed to initialize Supermicro vendor: " + err.Error()) return nil, err } - fwVendors = append(fwVendors, sm) + app.vendors = append(app.vendors, sm) case common.VendorMellanox: var mlx vendors.Vendor - mlx, err = mellanox.New(context.TODO(), firmwares, cfgSyncer, logger) + mlx, err = mellanox.New(ctx, firmwares, app.Config, app.Logger) if err != nil { - logger.Error("Failed to initialize Mellanox vendor: " + err.Error()) + app.Logger.Error("Failed to initialize Mellanox vendor: " + err.Error()) return nil, err } - fwVendors = append(fwVendors, mlx) + app.vendors = append(app.vendors, mlx) case common.VendorIntel: var i vendors.Vendor - i, err = intel.New(context.TODO(), firmwares, cfgSyncer, logger) + i, err = intel.New(ctx, firmwares, app.Config, app.Logger) if err != nil { - logger.Error("Failed to initialize Intel vendor: " + err.Error()) + app.Logger.Error("Failed to initialize Intel vendor: " + err.Error()) return nil, err } - fwVendors = append(fwVendors, i) + app.vendors = append(app.vendors, i) case "equinix": var e vendors.Vendor - e, err = equinix.New(context.TODO(), firmwares, cfgSyncer, logger) + e, err = equinix.New(ctx, firmwares, app.Config, app.Logger) if err != nil { - logger.Error("Failed to initialize Equinix vendor: " + err.Error()) + app.Logger.Error("Failed to initialize Equinix vendor: " + err.Error()) return nil, err } - fwVendors = append(fwVendors, e) + app.vendors = append(app.vendors, e) default: - logger.Error("Vendor not supported: " + vendor) + app.Logger.Error("Vendor not supported: " + vendor) continue } } - return &Syncer{ - config: cfgSyncer, - logger: logger, - vendors: fwVendors, - }, nil + return app, nil } // SyncFirmwares syncs all firmware files from the configured providers -func (s *Syncer) SyncFirmwares(ctx context.Context, dryRun bool) error { - s.dryRun = dryRun - - for _, v := range s.vendors { +func (a *App) SyncFirmwares(ctx context.Context) error { + for _, v := range a.vendors { err := v.Sync(ctx) if err != nil { - s.logger.Error("Failed to sync: " + err.Error()) + a.Logger.Error("Failed to sync: " + err.Error()) + } + } + + return nil +} + +// nolint:gocyclo // config load is cyclomatic +// LoadConfiguration loads application configuration +// +// Reads in the cfgFile when available and overrides from environment variables. +func (a *App) LoadConfiguration(cfgFile string, inventoryKind types.InventoryKind) error { + a.v.SetConfigType("yaml") + a.v.SetEnvPrefix(types.AppName) + a.v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + a.v.AutomaticEnv() + + // these are initialized here so viper can read in configuration from env vars + // once https://github.com/spf13/viper/pull/1429 is merged, this can go. + a.Config.ServerserviceOptions = &config.ServerserviceOptions{} + a.Config.FirmwareRepository = &config.S3Bucket{} + a.Config.AsRockRackRepository = &config.S3Bucket{} + + if cfgFile != "" { + fh, err := os.Open(cfgFile) + if err != nil { + return errors.Wrap(config.ErrConfig, err.Error()) + } + + if err = a.v.ReadConfig(fh); err != nil { + return errors.Wrap(config.ErrConfig, "ReadConfig error: "+err.Error()) + } + } + + a.v.SetDefault("log.level", "info") + + if err := a.envBindVars(); err != nil { + return errors.Wrap(config.ErrConfig, "env var bind error: "+err.Error()) + } + + if err := a.v.Unmarshal(a.Config); err != nil { + return errors.Wrap(config.ErrConfig, "Unmarshal error: "+err.Error()) + } + + err := a.envVarAppOverrides() + if err != nil { + return errors.Wrap(config.ErrConfig, "app env overrides error: "+err.Error()) + } + + if inventoryKind == types.InventoryStoreServerservice { + if err := a.envVarServerserviceOverrides(); err != nil { + return errors.Wrap(config.ErrConfig, "serverservice env overrides error: "+err.Error()) } } return nil } + +// nolint:gocyclo // env var load is cyclomatic +func (a *App) envVarAppOverrides() error { + if a.v.GetString("log.level") != "" { + a.Config.LogLevel = a.v.GetString("log.level") + } + + if a.v.GetString("s3.endpoint") != "" { + a.Config.FirmwareRepository.Endpoint = a.v.GetString("s3.endpoint") + } + + if a.v.GetString("s3.bucket") != "" { + a.Config.FirmwareRepository.Bucket = a.v.GetString("s3.bucket") + } + + if a.v.GetString("s3.region") != "" { + a.Config.FirmwareRepository.Region = a.v.GetString("s3.region") + } + + if a.v.GetString("s3.access.key") != "" { + a.Config.FirmwareRepository.AccessKey = a.v.GetString("s3.access.key") + } + + if a.v.GetString("s3.secret.key") != "" { + a.Config.FirmwareRepository.SecretKey = a.v.GetString("s3.secret.key") + } + + if a.v.GetString("asrr.s3.region") != "" { + a.Config.AsRockRackRepository.Region = a.v.GetString("asrr.s3.region") + } + + if a.v.GetString("asrr.s3.endpoint") != "" { + a.Config.AsRockRackRepository.Endpoint = a.v.GetString("asrr.s3.endpoint") + } + + if a.v.GetString("asrr.s3.bucket") != "" { + a.Config.AsRockRackRepository.Bucket = a.v.GetString("asrr.s3.bucket") + } + + if a.v.GetString("asrr.s3.access.key") != "" { + a.Config.AsRockRackRepository.AccessKey = a.v.GetString("asrr.s3.access.key") + } + + if a.v.GetString("asrr.s3.secret.key") != "" { + a.Config.AsRockRackRepository.SecretKey = a.v.GetString("asrr.s3.secret.key") + } + + if a.v.GetString("github.openbmc.token") != "" { + a.Config.GithubOpenBmcToken = a.v.GetString("github.openbmc.token") + } + + return nil +} + +// envBindVars binds environment variables to the struct +// without a configuration file being unmarshalled, +// this is a workaround for a viper bug, +// +// This can be replaced by the solution in https://github.com/spf13/viper/pull/1429 +// once that PR is merged. +func (a *App) envBindVars() error { + envKeysMap := map[string]interface{}{} + if err := mapstructure.Decode(a.Config, &envKeysMap); err != nil { + return err + } + + // Flatten nested conf map + flat, err := flatten.Flatten(envKeysMap, "", flatten.DotStyle) + if err != nil { + return errors.Wrap(err, "Unable to flatten config") + } + + for k := range flat { + if err := a.v.BindEnv(k); err != nil { + return errors.Wrap(config.ErrConfig, "env var bind error: "+err.Error()) + } + } + + return nil +} + +// Server service configuration options + +// nolint:gocyclo // parameter validation is cyclomatic +func (a *App) envVarServerserviceOverrides() error { + if a.Config.ServerserviceOptions == nil { + a.Config.ServerserviceOptions = &config.ServerserviceOptions{} + } + + if a.v.GetString("serverservice.endpoint") != "" { + a.Config.ServerserviceOptions.Endpoint = a.v.GetString("serverservice.endpoint") + } + + endpointURL, err := url.Parse(a.Config.ServerserviceOptions.Endpoint) + if err != nil { + return errors.New("serverservice endpoint URL error: " + err.Error()) + } + + a.Config.ServerserviceOptions.EndpointURL = endpointURL + + if a.v.GetString("serverservice.disable.oauth") != "" { + a.Config.ServerserviceOptions.DisableOAuth = a.v.GetBool("serverservice.disable.oauth") + } + + if a.Config.ServerserviceOptions.DisableOAuth { + return nil + } + + if a.v.GetString("serverservice.oidc.issuer.endpoint") != "" { + a.Config.ServerserviceOptions.OidcIssuerEndpoint = a.v.GetString("serverservice.oidc.issuer.endpoint") + } + + if a.Config.ServerserviceOptions.OidcIssuerEndpoint == "" { + return errors.New("serverservice oidc.issuer.endpoint not defined") + } + + if a.v.GetString("serverservice.oidc.audience.endpoint") != "" { + a.Config.ServerserviceOptions.OidcAudienceEndpoint = a.v.GetString("serverservice.oidc.audience.endpoint") + } + + if a.Config.ServerserviceOptions.OidcAudienceEndpoint == "" { + return errors.New("serverservice oidc.audience.endpoint not defined") + } + + if a.v.GetString("serverservice.oidc.client.secret") != "" { + a.Config.ServerserviceOptions.OidcClientSecret = a.v.GetString("serverservice.oidc.client.secret") + } + + if a.Config.ServerserviceOptions.OidcClientSecret == "" { + return errors.New("serverservice.oidc.client.secret not defined") + } + + if a.v.GetString("serverservice.oidc.client.id") != "" { + a.Config.ServerserviceOptions.OidcClientID = a.v.GetString("serverservice.oidc.client.id") + } + + if a.Config.ServerserviceOptions.OidcClientID == "" { + return errors.New("serverservice.oidc.client.id not defined") + } + + if a.v.GetString("serverservice.oidc.client.scopes") != "" { + a.Config.ServerserviceOptions.OidcClientScopes = a.v.GetStringSlice("serverservice.oidc.client.scopes") + } + + if len(a.Config.ServerserviceOptions.OidcClientScopes) == 0 { + return errors.New("serverservice oidc.client.scopes not defined") + } + + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 14aeafbc..5610aefe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,31 +3,65 @@ package config import ( "context" "encoding/json" - "errors" "io" "net/http" "net/url" - "os" "strings" "time" - "gopkg.in/yaml.v2" + "github.com/pkg/errors" + "github.com/metal-toolbox/firmware-syncer/pkg/types" serverservice "go.hollow.sh/serverservice/pkg/api/v1" ) var ( + ErrConfig = errors.New("configuration error") ErrProviderAttributes = errors.New("provider config missing required attribute(s)") ErrNoFileChecksum = errors.New("file upstreamURL declared with no checksum (Provider.UtilityChecksum)") ErrProviderNotSupported = errors.New("provider not suppported") ) -type Syncer struct { - ServerServiceURL string `yaml:"serverserviceURL"` - RepositoryURL string `yaml:"repositoryURL"` - RepositoryRegion string `yaml:"repositoryRegion"` - ArtifactsURL string `yaml:"artifactsURL"` - FirmwareManifestURL string `yaml:"firmwareManifestURL"` +// Config holds application configuration read from a YAML or set by env variables. +type Configuration struct { + // LogLevel is the app verbose logging level. + // one of - info, debug, trace + LogLevel string `mapstructure:"log_level"` + + InventoryKind types.InventoryKind `mapstructure:"inventory_kind"` + + // ServerserviceOptions defines the serverservice client configuration parameters + // + // This parameter is required when StoreKind is set to serverservice. + ServerserviceOptions *ServerserviceOptions `mapstructure:"serverservice"` + + // FirmwareRepository defines configuration for the s3 bucket firmware will be synced to + FirmwareRepository *S3Bucket `mapstructure:"s3bucket"` + + // AsRockRackRepository defines configuration for the asrockrack s3 source firmware bucket + AsRockRackRepository *S3Bucket `mapstructure:"s3bucket"` + + // ArtifactsURL defines the artifacts URL used by all firmware + ArtifactsURL string `mapstructure:"artifacts_url"` + + // FirmwareManifestURL defines the URL for modeldata.json + FirmwareManifestURL string `mapstructure:"firmware_manifest_url"` + + // GithubOpenBmcToken defines the token used to access internal openbmc repository + GithubOpenBmcToken string `mapstructure:"github_openbmc_token"` +} + +// ServerserviceOptions defines configuration for the Serverservice client. +// https://github.com/metal-toolbox/hollow-serverservice +type ServerserviceOptions struct { + EndpointURL *url.URL + Endpoint string `mapstructure:"endpoint"` + OidcIssuerEndpoint string `mapstructure:"oidc_issuer_endpoint"` + OidcAudienceEndpoint string `mapstructure:"oidc_audience_endpoint"` + OidcClientSecret string `mapstructure:"oidc_client_secret"` + OidcClientID string `mapstructure:"oidc_client_id"` + OidcClientScopes []string `mapstructure:"oidc_client_scopes"` + DisableOAuth bool `mapstructure:"disable_oauth"` } // FirmwareRecord from modeldata.json @@ -59,22 +93,6 @@ type S3Bucket struct { SecretKey string `mapstructure:"secret_key"` } -func LoadSyncerConfig(configFile string) (*Syncer, error) { - b, err := os.ReadFile(configFile) - if err != nil { - return nil, err - } - - var config *Syncer - - err = yaml.Unmarshal(b, &config) - if err != nil { - return nil, err - } - - return config, nil -} - func LoadFirmwareManifest(ctx context.Context, manifestURL string) (map[string][]*serverservice.ComponentFirmwareVersion, error) { var httpClient = &http.Client{ Timeout: time.Second * 15, diff --git a/internal/inventory/serverservice.go b/internal/inventory/serverservice.go index 47c4be74..3ee1d406 100644 --- a/internal/inventory/serverservice.go +++ b/internal/inventory/serverservice.go @@ -3,11 +3,11 @@ package inventory import ( "context" "net/url" - "os" "strings" "github.com/coreos/go-oidc" "github.com/google/uuid" + "github.com/metal-toolbox/firmware-syncer/internal/config" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/exp/slices" @@ -27,64 +27,50 @@ type ServerService struct { logger *logrus.Logger } -func New(ctx context.Context, serverServiceURL, artifactsURL string, logger *logrus.Logger) (*ServerService, error) { - if artifactsURL == "" { - return nil, errors.New("missing artifacts URL") - } - - clientSecret := os.Getenv("SERVERSERVICE_CLIENT_SECRET") - - if clientSecret == "" { - return nil, errors.New("missing server service client secret") - } +func New(ctx context.Context, cfg *config.ServerserviceOptions, artifactsURL string, logger *logrus.Logger) (*ServerService, error) { + var client *serverservice.Client - clientID := os.Getenv("SERVERSERVICE_CLIENT_ID") + var err error - if clientID == "" { - return nil, errors.New("missing server service client id") + if !cfg.DisableOAuth { + client, err = newClientWithOAuth(ctx, cfg, logger) + if err != nil { + return nil, err + } + } else { + client, err = serverservice.NewClientWithToken("fake", cfg.Endpoint, nil) + if err != nil { + return nil, err + } } - oidcProviderEndpoint := os.Getenv("SERVERSERVICE_OIDC_PROVIDER_ENDPOINT") - - if oidcProviderEndpoint == "" { - return nil, errors.New("missing server service oidc provider endpoint") - } + return &ServerService{ + artifactsURL: artifactsURL, + client: client, + logger: logger, + }, nil +} - provider, err := oidc.NewProvider(ctx, oidcProviderEndpoint) +func newClientWithOAuth(ctx context.Context, cfg *config.ServerserviceOptions, logger *logrus.Logger) (client *serverservice.Client, err error) { + provider, err := oidc.NewProvider(ctx, cfg.OidcIssuerEndpoint) if err != nil { return nil, err } - audience := os.Getenv("SERVERSERVICE_AUDIENCE_ENDPOINT") - - if audience == "" { - return nil, errors.New("missing server service audience URL") - } - - scopes := []string{ - "create:server-component-firmwares", - "read:server-component-firmwares", - "update:server-component-firmwares", - } - oauthConfig := clientcredentials.Config{ - ClientID: clientID, - ClientSecret: clientSecret, + ClientID: cfg.OidcClientID, + ClientSecret: cfg.OidcClientSecret, TokenURL: provider.Endpoint().TokenURL, - Scopes: scopes, - EndpointParams: url.Values{"audience": {audience}}, + Scopes: cfg.OidcClientScopes, + EndpointParams: url.Values{"audience": {cfg.OidcAudienceEndpoint}}, } - c, err := serverservice.NewClient(serverServiceURL, oauthConfig.Client(ctx)) + client, err = serverservice.NewClient(cfg.EndpointURL.String(), oauthConfig.Client(ctx)) if err != nil { return nil, err } - return &ServerService{ - artifactsURL: artifactsURL, - client: c, - logger: logger, - }, nil + return client, nil } // getArtifactsURL returns the https artifactsURL for the given s3 dstURL diff --git a/internal/vendors/asrockrack/asrockrack.go b/internal/vendors/asrockrack/asrockrack.go index 51d795d9..76211fcd 100644 --- a/internal/vendors/asrockrack/asrockrack.go +++ b/internal/vendors/asrockrack/asrockrack.go @@ -2,7 +2,6 @@ package asrockrack import ( "context" - "os" "github.com/metal-toolbox/firmware-syncer/internal/config" "github.com/metal-toolbox/firmware-syncer/internal/inventory" @@ -28,38 +27,9 @@ type ASRockRack struct { tmpFs rcloneFs.Fs } -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfgSyncer *config.Syncer, logger *logrus.Logger) (vendors.Vendor, error) { - // RepositoryURL required - if cfgSyncer.RepositoryURL == "" { - return nil, errors.Wrap(config.ErrProviderAttributes, "RepositoryURL not defined") - } - - // TODO: For now set this configuration from env vars but ideally this should come from - // somewhere else. Maybe a per provider config? - srcS3Config := &config.S3Bucket{ - Region: os.Getenv("ASRR_S3_REGION"), - Endpoint: os.Getenv("ASRR_S3_ENDPOINT"), - Bucket: os.Getenv("ASRR_S3_BUCKET"), - AccessKey: os.Getenv("ASRR_S3_ACCESS_KEY"), - SecretKey: os.Getenv("ASRR_S3_SECRET_KEY"), - } - - // parse S3 endpoint and bucket from cfgSyncer.RepositoryURL - s3DstEndpoint, s3DstBucket, err := config.ParseRepositoryURL(cfgSyncer.RepositoryURL) - if err != nil { - return nil, err - } - - dstS3Config := &config.S3Bucket{ - Region: cfgSyncer.RepositoryRegion, - Endpoint: s3DstEndpoint, - Bucket: s3DstBucket, - AccessKey: os.Getenv("S3_ACCESS_KEY"), - SecretKey: os.Getenv("S3_SECRET_KEY"), - } - +func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { // init inventory - i, err := inventory.New(ctx, cfgSyncer.ServerServiceURL, cfgSyncer.ArtifactsURL, logger) + i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) if err != nil { return nil, err } @@ -67,12 +37,12 @@ func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersio // init rclone filesystems for tmp, dst and src files vendors.SetRcloneLogging(logger) - dstFs, err := vendors.InitS3Fs(ctx, dstS3Config, "/") + dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") if err != nil { return nil, err } - srcFs, err := vendors.InitS3Fs(ctx, srcS3Config, "/") + srcFs, err := vendors.InitS3Fs(ctx, cfg.AsRockRackRepository, "/") if err != nil { return nil, err } @@ -87,8 +57,8 @@ func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersio logger: logger, metrics: vendors.NewMetrics(), inventory: i, - srcCfg: srcS3Config, - dstCfg: dstS3Config, + srcCfg: cfg.AsRockRackRepository, + dstCfg: cfg.FirmwareRepository, srcFs: srcFs, dstFs: dstFs, tmpFs: tmpFs, diff --git a/internal/vendors/dell/dell.go b/internal/vendors/dell/dell.go index dbd248f0..260e3f93 100644 --- a/internal/vendors/dell/dell.go +++ b/internal/vendors/dell/dell.go @@ -2,7 +2,6 @@ package dell import ( "context" - "os" "github.com/metal-toolbox/firmware-syncer/internal/config" "github.com/metal-toolbox/firmware-syncer/internal/inventory" @@ -18,7 +17,6 @@ import ( // DUP implements the Vendor interface methods to retrieve dell DUP firmware files type DUP struct { - syncer *config.Syncer dstCfg *config.S3Bucket dstFs rcloneFs.Fs tmpFs rcloneFs.Fs @@ -29,28 +27,9 @@ type DUP struct { } // NewDUP returns a new DUP firmware syncer object -func NewDUP(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfgSyncer *config.Syncer, logger *logrus.Logger) (vendors.Vendor, error) { - // RepositoryURL required - if cfgSyncer.RepositoryURL == "" { - return nil, errors.Wrap(config.ErrProviderAttributes, "RepositoryURL not defined") - } - - // parse S3 endpoint and bucket from cfgProvider.RepositoryURL - s3Endpoint, s3Bucket, err := config.ParseRepositoryURL(cfgSyncer.RepositoryURL) - if err != nil { - return nil, err - } - - s3Cfg := &config.S3Bucket{ - Region: cfgSyncer.RepositoryRegion, - Endpoint: s3Endpoint, - Bucket: s3Bucket, - AccessKey: os.Getenv("S3_ACCESS_KEY"), - SecretKey: os.Getenv("S3_SECRET_KEY"), - } - +func NewDUP(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { // init inventory - i, err := inventory.New(ctx, cfgSyncer.ServerServiceURL, cfgSyncer.ArtifactsURL, logger) + i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) if err != nil { return nil, err } @@ -58,7 +37,7 @@ func NewDUP(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVer // init rclone filesystems for tmp and dst files vendors.SetRcloneLogging(logger) - dstFs, err := vendors.InitS3Fs(ctx, s3Cfg, "/") + dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") if err != nil { return nil, err } @@ -69,8 +48,7 @@ func NewDUP(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVer } return &DUP{ - syncer: cfgSyncer, - dstCfg: s3Cfg, + dstCfg: cfg.FirmwareRepository, dstFs: dstFs, tmpFs: tmpFs, firmwares: firmwares, diff --git a/internal/vendors/equinix/equinix.go b/internal/vendors/equinix/equinix.go index a281579a..03a3028d 100644 --- a/internal/vendors/equinix/equinix.go +++ b/internal/vendors/equinix/equinix.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "net/url" - "os" "strings" "time" @@ -36,35 +35,16 @@ type Equinix struct { tmpFs fs.Fs } -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfgSyncer *config.Syncer, logger *logrus.Logger) (vendors.Vendor, error) { - // RepositoryURL required - if cfgSyncer.RepositoryURL == "" { - return nil, errors.Wrap(config.ErrProviderAttributes, "RepositoryURL not defined") - } - +func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: os.Getenv("GITHUB_OPENBMC_TOKEN")}, + &oauth2.Token{AccessToken: cfg.GithubOpenBmcToken}, ) tc := oauth2.NewClient(ctx, ts) ghClient := github.NewClient(tc) - // parse S3 endpoint and bucket from cfgSyncer.RepositoryURL - s3DstEndpoint, s3DstBucket, err := config.ParseRepositoryURL(cfgSyncer.RepositoryURL) - if err != nil { - return nil, err - } - - dstS3Config := &config.S3Bucket{ - Region: cfgSyncer.RepositoryRegion, - Endpoint: s3DstEndpoint, - Bucket: s3DstBucket, - AccessKey: os.Getenv("S3_ACCESS_KEY"), - SecretKey: os.Getenv("S3_SECRET_KEY"), - } - // init inventory - i, err := inventory.New(ctx, cfgSyncer.ServerServiceURL, cfgSyncer.ArtifactsURL, logger) + i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) if err != nil { return nil, err } @@ -72,7 +52,7 @@ func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersio // init rclone filesystems for tmp and dst files vendors.SetRcloneLogging(logger) - dstFs, err := vendors.InitS3Fs(ctx, dstS3Config, "/") + dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") if err != nil { return nil, err } @@ -88,7 +68,7 @@ func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersio metrics: vendors.NewMetrics(), inventory: i, ghClient: ghClient, - dstCfg: dstS3Config, + dstCfg: cfg.FirmwareRepository, dstFs: dstFs, tmpFs: tmpFs, }, nil diff --git a/internal/vendors/intel/intel.go b/internal/vendors/intel/intel.go index c9eaa38f..d15151a8 100644 --- a/internal/vendors/intel/intel.go +++ b/internal/vendors/intel/intel.go @@ -8,7 +8,6 @@ import ( "github.com/metal-toolbox/firmware-syncer/internal/config" "github.com/metal-toolbox/firmware-syncer/internal/inventory" "github.com/metal-toolbox/firmware-syncer/internal/vendors" - "github.com/pkg/errors" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/operations" "github.com/sirupsen/logrus" @@ -17,7 +16,6 @@ import ( ) type Intel struct { - syncer *config.Syncer firmwares []*serverservice.ComponentFirmwareVersion logger *logrus.Logger metrics *vendors.Metrics @@ -27,35 +25,16 @@ type Intel struct { tmpFs fs.Fs } -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfgSyncer *config.Syncer, logger *logrus.Logger) (vendors.Vendor, error) { - // RepositoryURL required - if cfgSyncer.RepositoryURL == "" { - return nil, errors.Wrap(config.ErrProviderAttributes, "RepositoryURL not defined") - } - - // parse S3 endpoint and bucket from cfgProvider.RepositoryURL - s3DstEndpoint, s3DstBucket, err := config.ParseRepositoryURL(cfgSyncer.RepositoryURL) - if err != nil { - return nil, err - } - - dstS3Config := &config.S3Bucket{ - Region: cfgSyncer.RepositoryRegion, - Endpoint: s3DstEndpoint, - Bucket: s3DstBucket, - AccessKey: os.Getenv("S3_ACCESS_KEY"), - SecretKey: os.Getenv("S3_SECRET_KEY"), - } - +func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { // init inventory - i, err := inventory.New(ctx, cfgSyncer.ServerServiceURL, cfgSyncer.ArtifactsURL, logger) + i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) if err != nil { return nil, err } vendors.SetRcloneLogging(logger) - dstFs, err := vendors.InitS3Fs(ctx, dstS3Config, "/") + dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") if err != nil { return nil, err } @@ -66,12 +45,11 @@ func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersio } return &Intel{ - syncer: cfgSyncer, firmwares: firmwares, logger: logger, metrics: vendors.NewMetrics(), inventory: i, - dstCfg: dstS3Config, + dstCfg: cfg.FirmwareRepository, dstFs: dstFs, tmpFs: tmpFs, }, nil diff --git a/internal/vendors/mellanox/mellanox.go b/internal/vendors/mellanox/mellanox.go index 14de3580..527979b1 100644 --- a/internal/vendors/mellanox/mellanox.go +++ b/internal/vendors/mellanox/mellanox.go @@ -8,7 +8,6 @@ import ( "github.com/metal-toolbox/firmware-syncer/internal/config" "github.com/metal-toolbox/firmware-syncer/internal/inventory" "github.com/metal-toolbox/firmware-syncer/internal/vendors" - "github.com/pkg/errors" rcloneFs "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/operations" "github.com/sirupsen/logrus" @@ -16,7 +15,6 @@ import ( ) type Mellanox struct { - syncer *config.Syncer dstCfg *config.S3Bucket dstFs rcloneFs.Fs tmpFs rcloneFs.Fs @@ -26,35 +24,16 @@ type Mellanox struct { inventory *inventory.ServerService } -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfgSyncer *config.Syncer, logger *logrus.Logger) (vendors.Vendor, error) { - // RepositoryURL required - if cfgSyncer.RepositoryURL == "" { - return nil, errors.Wrap(config.ErrProviderAttributes, "RepositoryURL not defined") - } - - // parse S3 endpoint and bucket from cfgProvider.RepositoryURL - s3DstEndpoint, s3DstBucket, err := config.ParseRepositoryURL(cfgSyncer.RepositoryURL) - if err != nil { - return nil, err - } - - dstS3Config := &config.S3Bucket{ - Region: cfgSyncer.RepositoryRegion, - Endpoint: s3DstEndpoint, - Bucket: s3DstBucket, - AccessKey: os.Getenv("S3_ACCESS_KEY"), - SecretKey: os.Getenv("S3_SECRET_KEY"), - } - +func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { // init inventory - i, err := inventory.New(ctx, cfgSyncer.ServerServiceURL, cfgSyncer.ArtifactsURL, logger) + i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) if err != nil { return nil, err } vendors.SetRcloneLogging(logger) - dstFs, err := vendors.InitS3Fs(ctx, dstS3Config, "/") + dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") if err != nil { return nil, err } @@ -65,12 +44,11 @@ func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersio } return &Mellanox{ - syncer: cfgSyncer, firmwares: firmwares, logger: logger, metrics: vendors.NewMetrics(), inventory: i, - dstCfg: dstS3Config, + dstCfg: cfg.FirmwareRepository, dstFs: dstFs, tmpFs: tmpFs, }, nil diff --git a/internal/vendors/supermicro/supermicro.go b/internal/vendors/supermicro/supermicro.go index f41308fa..9f5c9d4b 100644 --- a/internal/vendors/supermicro/supermicro.go +++ b/internal/vendors/supermicro/supermicro.go @@ -22,7 +22,6 @@ import ( ) type Supermicro struct { - syncer *config.Syncer firmwares []*serverservice.ComponentFirmwareVersion logger *logrus.Logger metrics *vendors.Metrics @@ -32,35 +31,16 @@ type Supermicro struct { tmpFs fs.Fs } -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfgSyncer *config.Syncer, logger *logrus.Logger) (vendors.Vendor, error) { - // RepositoryURL required - if cfgSyncer.RepositoryURL == "" { - return nil, errors.Wrap(config.ErrProviderAttributes, "RepositoryURL not defined") - } - - // parse S3 endpoint and bucket from cfgProvider.RepositoryURL - s3DstEndpoint, s3DstBucket, err := config.ParseRepositoryURL(cfgSyncer.RepositoryURL) - if err != nil { - return nil, err - } - - dstS3Config := &config.S3Bucket{ - Region: cfgSyncer.RepositoryRegion, - Endpoint: s3DstEndpoint, - Bucket: s3DstBucket, - AccessKey: os.Getenv("S3_ACCESS_KEY"), - SecretKey: os.Getenv("S3_SECRET_KEY"), - } - +func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { // init inventory - i, err := inventory.New(ctx, cfgSyncer.ServerServiceURL, cfgSyncer.ArtifactsURL, logger) + i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) if err != nil { return nil, err } vendors.SetRcloneLogging(logger) - dstFs, err := vendors.InitS3Fs(ctx, dstS3Config, "/") + dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") if err != nil { return nil, err } @@ -71,12 +51,11 @@ func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersio } return &Supermicro{ - syncer: cfgSyncer, firmwares: firmwares, logger: logger, metrics: vendors.NewMetrics(), inventory: i, - dstCfg: dstS3Config, + dstCfg: cfg.FirmwareRepository, dstFs: dstFs, tmpFs: tmpFs, }, nil diff --git a/pkg/types/types.go b/pkg/types/types.go new file mode 100644 index 00000000..e582c21c --- /dev/null +++ b/pkg/types/types.go @@ -0,0 +1,23 @@ +package types + +type ( + InventoryKind string + // LogLevel is the logging level string. + LogLevel string +) + +const ( + AppName = "syncer" + + InventoryStoreYAML InventoryKind = "yaml" + InventoryStoreServerservice InventoryKind = "serverservice" + + LogLevelInfo LogLevel = "info" + LogLevelDebug LogLevel = "debug" + LogLevelTrace LogLevel = "trace" +) + +// InventoryKinds returns the supported asset inventory, firmware configuration sources +func InventoryKinds() []InventoryKind { + return []InventoryKind{InventoryStoreYAML, InventoryStoreServerservice} +}