diff --git a/config/provider/inmem.go b/config/provider/inmem.go index 6108b1c8..0b56b795 100644 --- a/config/provider/inmem.go +++ b/config/provider/inmem.go @@ -54,7 +54,6 @@ func (l *InMem) GetKey(key string) string { func (l *InMem) GetObject(key string, result interface{}) error { l.mutex.Lock() defer l.mutex.Unlock() - //return utils.Unmarshal(l.store[key], result) return json.Unmarshal([]byte(l.store[key]), result) } diff --git a/converter/k8s.go b/converter/k8s.go new file mode 100644 index 00000000..7d87c3fb --- /dev/null +++ b/converter/k8s.go @@ -0,0 +1,77 @@ +package converter + +import ( + "bytes" + + "github.com/layer5io/meshkit/models/patterns" + "github.com/layer5io/meshkit/utils" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/pattern" + "gopkg.in/yaml.v3" +) + +type K8sConverter struct{} + +func (k *K8sConverter) Convert(patternFile string) (string, error) { + pattern, err := patterns.GetPatternFormat(patternFile) + if err != nil { + return "", err + } + return NewK8sManifestsFromPatternfile(pattern) +} + +func NewK8sManifestsFromPatternfile(patternFile *pattern.PatternFile) (string, error) { + + buf := bytes.NewBufferString("") + + enc := yaml.NewEncoder(buf) + for _, comp := range patternFile.Components { + err := enc.Encode(CreateK8sResourceStructure(comp)) + if err != nil { + return "", err + } + } + return buf.String(), nil +} + +func CreateK8sResourceStructure(comp *component.ComponentDefinition) map[string]interface{} { + annotations := map[string]interface{}{} + labels := map[string]interface{}{} + + _confMetadata, ok := comp.Configuration["metadata"] + if ok { + confMetadata, err := utils.Cast[map[string]interface{}](_confMetadata) + if err == nil { + + _annotations, ok := confMetadata["annotations"] + if ok { + annotations, _ = utils.Cast[map[string]interface{}](_annotations) + } + + _label, ok := confMetadata["labels"] + + if ok { + labels, _ = utils.Cast[map[string]interface{}](_label) + } + } + } + + component := map[string]interface{}{ + "apiVersion": comp.Component.Version, + "kind": comp.Component.Kind, + "metadata": map[string]interface{}{ + "name": comp.DisplayName, + "annotations": annotations, + "labels": labels, + }, + } + + for k, v := range comp.Configuration { + if k == "apiVersion" || k == "kind" || k == "metadata" { + continue + } + + component[k] = v + } + return component +} diff --git a/go.mod b/go.mod index 412607c0..ea10a40a 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.20 + github.com/meshery/schemas v0.7.31 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 @@ -193,7 +193,7 @@ require ( github.com/novln/docker-parser v1.0.0 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/runc v1.1.12 // indirect + github.com/opencontainers/runc v1.1.14 // indirect github.com/openshift/api v0.0.0-20200803131051-87466835fcc0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect @@ -233,7 +233,7 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect diff --git a/go.sum b/go.sum index 8c11bbdd..9aad5ca9 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.20 h1:1v5+PKjwq/jq0/dayVNJVDbsrh8GXI5QmDYvEBPDdac= -github.com/meshery/schemas v0.7.20/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= +github.com/meshery/schemas v0.7.31 h1:FfVR+oErAiiEomt6sTZI5uKhoyU26QXawT6UDZHbthI= +github.com/meshery/schemas v0.7.31/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= @@ -707,8 +707,8 @@ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2sz github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= -github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= +github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= +github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/openshift/api v0.0.0-20200803131051-87466835fcc0 h1:ngLoHyAD7dNUzZY6cBA+X/DWIRLT56n6PjdN9+hqdvs= @@ -927,8 +927,8 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/models/converter/converter.go b/models/converter/converter.go new file mode 100644 index 00000000..3fda332e --- /dev/null +++ b/models/converter/converter.go @@ -0,0 +1,18 @@ +package converter + +import ( + "github.com/layer5io/meshkit/converter" +) + +type ConvertFormat interface { + Convert(string) (string, error) +} + +func NewFormatConverter(format DesignFormat) (ConvertFormat, error) { + switch format { + case K8sManifest: + return &converter.K8sConverter{}, nil + default: + return nil, ErrUnknownFormat(format) + } +} diff --git a/models/converter/error.go b/models/converter/error.go new file mode 100644 index 00000000..6bc9e788 --- /dev/null +++ b/models/converter/error.go @@ -0,0 +1,15 @@ +package converter + +import ( + "fmt" + + "github.com/layer5io/meshkit/errors" +) + +const ( + ErrUnknownFormatCode = "meshkit-11245" +) + +func ErrUnknownFormat(format DesignFormat) error { + return errors.New(ErrUnknownFormatCode, errors.Alert, []string{fmt.Sprintf("\"%s\" format is not supported", format)}, []string{fmt.Sprintf("Failed to export design in \"%s\" format", format)}, []string{"The format is not supported by the current version of Meshery server"}, []string{"Make sure to export design in one of the supported format"}) +} diff --git a/models/converter/formats.go b/models/converter/formats.go new file mode 100644 index 00000000..45c1f090 --- /dev/null +++ b/models/converter/formats.go @@ -0,0 +1,10 @@ +package converter + +type DesignFormat string + +const ( + HelmChart DesignFormat = "Helm Chart" + DockerCompose DesignFormat = "Docker Compose" + K8sManifest DesignFormat = "Kubernetes Manifest" + Design DesignFormat = "Design" +) \ No newline at end of file diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 86971485..6a512f79 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -102,7 +102,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if mf.Greedy { if mf.Id != "" { - finder = finder.First("model_dbs.id = ?", mf.Id) + finder = finder.Where("model_dbs.id = ?", mf.Id) } if mf.Name != "" && mf.DisplayName != "" { finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") @@ -182,7 +182,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if includeComponents { var components []component.ComponentDefinition finder := db.Model(&component.ComponentDefinition{}). - Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles"). + Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles,component_definition_dbs.capabilities"). Where("component_definition_dbs.model_id = ?", _modelDB.Id) if err := finder.Scan(&components).Error; err != nil { return nil, 0, 0, err diff --git a/models/registration/dir.go b/models/registration/dir.go index fb7db466..f283f1e6 100644 --- a/models/registration/dir.go +++ b/models/registration/dir.go @@ -7,7 +7,10 @@ import ( "reflect" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/oci" + "github.com/layer5io/meshkit/utils" + "github.com/meshery/schemas/models/v1alpha3/relationship" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" @@ -26,69 +29,185 @@ func NewDir(path string) Dir { } /* -PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct. +PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the PackagingUnit struct. Invalid definitions are stored in the regErrStore with error data. */ -func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error) { - pkg := packagingUnit{} - // check if the given is a directory - _, err = os.ReadDir(d.dirpath) +func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ PackagingUnit, err error) { + pkg := PackagingUnit{} + + // Extract the filename to use as entityName in case of errors + filename := filepath.Base(d.dirpath) + + // Check if the given path is accessible + _, err = os.Stat(d.dirpath) if err != nil { - return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err)) + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err))) + return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err)) + } + + // Process the path (file or directory) + err = processDir(d.dirpath, &pkg, regErrStore) + if err != nil { + modelName := "" + if !reflect.ValueOf(pkg.Model).IsZero() { + modelName = pkg.Model.Name + } + regErrStore.InsertEntityRegError("", modelName, entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err))) + return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err)) } - err = filepath.Walk(d.dirpath, func(path string, f os.FileInfo, err error) error { + + if reflect.ValueOf(pkg.Model).IsZero() { + errMsg := fmt.Errorf("model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core must be present, otherwise the model package is considered malformed") + regErrStore.InsertEntityRegError("", "", entity.Model, filename, errMsg) + return pkg, errMsg + } + + return pkg, nil +} + +func processDir(dirPath string, pkg *PackagingUnit, regErrStore RegistrationErrorStore) error { + var tempDirs []string + defer func() { + for _, tempDir := range tempDirs { + os.RemoveAll(tempDir) + } + }() + + return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { if err != nil { - return err + err = utils.ErrFileWalkDir(fmt.Errorf("error accessing path: %w", err), path) + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil } - if f.IsDir() { + if info.IsDir() { return nil } - byt, _ := os.ReadFile(path) - if byt == nil { + + // Read the file content + data, err := os.ReadFile(path) + if err != nil { + err = oci.ErrReadingFile(err) + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) return nil } - var e entity.Entity - e, err = getEntity(byt) + // Check if the file is an OCI artifact + if oci.IsOCIArtifact(data) { + // Extract the OCI artifact + tempDir, err := oci.CreateTempOCIContentDir() + if err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil + } + tempDirs = append(tempDirs, tempDir) + err = oci.UnCompressOCIArtifact(path, tempDir) + if err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil + } + // Recursively process the extracted directory + if err := processDir(tempDir, pkg, regErrStore); err != nil { + return err + } + return nil + } + + // Check if the file is a zip or tar file + if utils.IsZip(path) || utils.IsTarGz(path) { + tempDir, err := os.MkdirTemp("", "nested-extract-") + if err != nil { + err = utils.ErrCreateDir(fmt.Errorf("error creating temp directory for nested archive extraction: %w", err), tempDir) + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil + } + tempDirs = append(tempDirs, tempDir) + if err := utils.ExtractFile(path, tempDir); err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil + } + // Recursively process the extracted directory + if err := processDir(tempDir, pkg, regErrStore); err != nil { + return err + } + return nil + } + + content := data + content, err = utils.YAMLToJSON(content) if err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + return nil + } + // Determine the entity type + entityType, err := utils.FindEntityType(content) + if err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) regErrStore.AddInvalidDefinition(path, err) return nil } - // set it to pkgunit + if entityType == "" { + // Not an entity we care about + return nil + } + + // Get the entity + var e entity.Entity + e, err = getEntity(content) + if err != nil { + regErrStore.InsertEntityRegError("", "", entityType, filepath.Base(path), fmt.Errorf("could not get entity: %w", err)) + regErrStore.AddInvalidDefinition(path, fmt.Errorf("could not get entity: %w", err)) + return nil + } + + // Add the entity to the packaging unit switch e.Type() { case entity.Model: - if !reflect.ValueOf(pkg.model).IsZero() { - // currently models inside models are not handled - return nil - } model, err := utils.Cast[*model.ModelDefinition](e) if err != nil { + modelName := "" + if model != nil { + modelName = model.Name + } + regErrStore.InsertEntityRegError("", modelName, entityType, modelName, ErrGetEntity(err)) regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + return nil } - pkg.model = *model + pkg.Model = *model case entity.ComponentDefinition: comp, err := utils.Cast[*component.ComponentDefinition](e) if err != nil { + componentName := "" + if comp != nil { + componentName = comp.Component.Kind + } + regErrStore.InsertEntityRegError("", "", entityType, componentName, ErrGetEntity(err)) regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + return nil } - pkg.components = append(pkg.components, *comp) + pkg.Components = append(pkg.Components, *comp) case entity.RelationshipDefinition: rel, err := utils.Cast[*relationship.RelationshipDefinition](e) if err != nil { + relationshipName := "" + if rel != nil { + relationshipName = rel.Model.Name + } + regErrStore.InsertEntityRegError("", "", entityType, relationshipName, ErrGetEntity(err)) regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + return nil } - pkg.relationships = append(pkg.relationships, *rel) + pkg.Relationships = append(pkg.Relationships, *rel) + default: + // Unhandled entity type + return nil } return nil }) - if err != nil { - return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err)) - } - if reflect.ValueOf(pkg.model).IsZero() { - err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..") - regErrStore.AddInvalidDefinition(d.dirpath, err) - return pkg, err - } - return pkg, err } diff --git a/models/registration/interface.go b/models/registration/interface.go index 3f9f580b..65872e20 100644 --- a/models/registration/interface.go +++ b/models/registration/interface.go @@ -12,11 +12,11 @@ type RegistrationErrorStore interface { InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error) } -// Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server +// Anything that can be parsed into a PackagingUnit is a RegisterableEntity in Meshery server type RegisterableEntity interface { /* 1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered 2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore */ - PkgUnit(RegistrationErrorStore) (packagingUnit, error) + PkgUnit(RegistrationErrorStore) (PackagingUnit, error) } diff --git a/models/registration/oci.go b/models/registration/oci.go index 8c8af21b..02355cf8 100644 --- a/models/registration/oci.go +++ b/models/registration/oci.go @@ -8,7 +8,7 @@ type OCIImage struct { _ gcrv1.Image } -func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) { - pkg := packagingUnit{} +func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (PackagingUnit, error) { + pkg := PackagingUnit{} return pkg, nil } diff --git a/models/registration/register.go b/models/registration/register.go index 50f86639..02dde7f5 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -10,11 +10,11 @@ import ( "github.com/meshery/schemas/models/v1beta1/model" ) -// packaingUnit is the representation of the atomic unit that can be registered into the capabilities registry -type packagingUnit struct { - model model.ModelDefinition - components []component.ComponentDefinition - relationships []relationship.RelationshipDefinition +// PackagingUnit is the representation of the atomic unit that can be registered into the capabilities registry +type PackagingUnit struct { + Model model.ModelDefinition + Components []component.ComponentDefinition + Relationships []relationship.RelationshipDefinition _ []v1beta1.PolicyDefinition } @@ -22,10 +22,11 @@ type RegistrationHelper struct { regManager *meshmodel.RegistryManager regErrStore RegistrationErrorStore svgBaseDir string + PkgUnits []PackagingUnit // Store successfully registered packagingUnits } func NewRegistrationHelper(svgBaseDir string, regm *meshmodel.RegistryManager, regErrStore RegistrationErrorStore) RegistrationHelper { - return RegistrationHelper{svgBaseDir: svgBaseDir, regManager: regm, regErrStore: regErrStore} + return RegistrationHelper{svgBaseDir: svgBaseDir, regManager: regm, regErrStore: regErrStore, PkgUnits: []PackagingUnit{}} } /* @@ -45,11 +46,15 @@ func (rh *RegistrationHelper) Register(entity RegisterableEntity) { register will return an error if it is not able to register the `model`. If there are errors when registering other entities, they are handled properly but does not stop the registration process. */ -func (rh *RegistrationHelper) register(pkg packagingUnit) { +func (rh *RegistrationHelper) register(pkg PackagingUnit) { + if len(pkg.Components) == 0 && len(pkg.Relationships) == 0 { + //silently exit if the model does not conatin any components or relationships + return + } // 1. Register the model - model := pkg.model + model := pkg.Model - // Dont register anything else if registrant is not there + // Don't register anything else if registrant is not there if model.Registrant.Kind == "" { err := ErrMissingRegistrant(model.Name) rh.regErrStore.InsertEntityRegError(model.Registrant.Kind, "", entity.Model, model.Name, err) @@ -64,24 +69,22 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { var svgCompletePath string - //Write SVG for models - model.Metadata.SvgColor, model.Metadata.SvgWhite, svgCompletePath = WriteAndReplaceSVGWithFileSystemPath(model.Metadata.SvgColor, + // Write SVG for models + model.Metadata.SvgColor, model.Metadata.SvgWhite, svgCompletePath = WriteAndReplaceSVGWithFileSystemPath( + model.Metadata.SvgColor, model.Metadata.SvgWhite, - svgComplete, rh.svgBaseDir, + svgComplete, + rh.svgBaseDir, model.Name, model.Name, ) if svgCompletePath != "" { model.Metadata.SvgComplete = &svgCompletePath } - } model.Registrant.Status = connection.Registered - _, _, err := rh.regManager.RegisterEntity( - model.Registrant, - &model, - ) + _, _, err := rh.regManager.RegisterEntity(model.Registrant, &model) // If model cannot be registered, don't register anything else if err != nil { @@ -91,13 +94,16 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { } hostname := model.Registrant.Kind - modelName := model.Name + + // Prepare slices to hold successfully registered components and relationships + var registeredComponents []component.ComponentDefinition + var registeredRelationships []relationship.RelationshipDefinition // 2. Register components - for _, comp := range pkg.components { + for _, comp := range pkg.Components { comp.Model = model if comp.Styles != nil { - //Write SVG for components + // Write SVG for components comp.Styles.SvgColor, comp.Styles.SvgWhite, comp.Styles.SvgComplete = WriteAndReplaceSVGWithFileSystemPath( comp.Styles.SvgColor, comp.Styles.SvgWhite, @@ -108,23 +114,33 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { ) } - _, _, err := rh.regManager.RegisterEntity( - model.Registrant, - &comp, - ) + _, _, err := rh.regManager.RegisterEntity(model.Registrant, &comp) if err != nil { err = ErrRegisterEntity(err, string(comp.Type()), comp.DisplayName) - rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.ComponentDefinition, comp.DisplayName, err) + rh.regErrStore.InsertEntityRegError(hostname, model.DisplayName, entity.ComponentDefinition, comp.DisplayName, err) + } else { + // Successful registration, add to successfulComponents + registeredComponents = append(registeredComponents, comp) } } // 3. Register relationships - for _, rel := range pkg.relationships { + for _, rel := range pkg.Relationships { rel.Model = model _, _, err := rh.regManager.RegisterEntity(model.Registrant, &rel) if err != nil { err = ErrRegisterEntity(err, string(rel.Type()), string(rel.Kind)) - rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.RelationshipDefinition, rel.Id.String(), err) + rh.regErrStore.InsertEntityRegError(hostname, model.DisplayName, entity.RelationshipDefinition, rel.Id.String(), err) + } else { + // Successful registration, add to successfulRelationships + registeredRelationships = append(registeredRelationships, rel) } } + + // Update pkg with only successfully registered components and relationships + pkg.Components = registeredComponents + pkg.Relationships = registeredRelationships + pkg.Model = model + // Store the successfully registered PackagingUnit + rh.PkgUnits = append(rh.PkgUnits, pkg) } diff --git a/models/registration/tar.go b/models/registration/tar.go index 139a28bb..3849039d 100644 --- a/models/registration/tar.go +++ b/models/registration/tar.go @@ -4,7 +4,7 @@ type Tar struct { _ string } -func (t Tar) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) { - pkg := packagingUnit{} +func (t Tar) PkgUnit(regErrStore RegistrationErrorStore) (PackagingUnit, error) { + pkg := PackagingUnit{} return pkg, nil } diff --git a/schemas/configuration/modelImport.json b/schemas/configuration/modelImport.json index e3693d40..aee0bbc0 100644 --- a/schemas/configuration/modelImport.json +++ b/schemas/configuration/modelImport.json @@ -3,10 +3,7 @@ "properties": { "uploadType": { "title": "Upload method", - "enum": [ - "File Upload", - "URL Import" - ], + "enum": ["File Upload", "URL Import"], "default": "Select the Upload Method", "x-rjsf-grid-area": "12", "description": "Choose the method you prefer to upload your model file. Select 'File Upload' if you have the file on your local system or 'URL Import' if you have the file hosted online." @@ -30,9 +27,7 @@ "x-rjsf-grid-area": "12" } }, - "required": [ - "file" - ] + "required": ["file"] } }, { @@ -49,17 +44,169 @@ "type": "string", "format": "uri", "title": "URL", - "description": "Provide the URL of the design file you want to import. This should be a direct URL to the file, for example: https://raw.github.com/your-design-file.yaml", + "description": "Provide the URL of the design file you want to import.", "x-rjsf-grid-area": "12" + }, + "model": { + "type": "object", + "description": "Provide the details of the model you are uploading incase you are trying to generate a new model", + "properties": { + "modelDisplayName": { + "description":"The model name that would be showed on kanvas and everywhere else mostly of type 'Model Name'", + "type": "string" + }, + "registrant": { + "type": "string", + "enum": ["github", "artifacthub"] + }, + "model": { + "description":"This name is of the type 'model-name'", + "type": "string" + }, + "category": { + "description":"The category of the model. The model will be located under the specific category on kanvas.", + "type": "string", + "enum": [ + "Analytics", + "App Definition and Development", + "Cloud Native Network", + "Cloud Native Storage", + "Database", + "Machine Learning", + "Observability and Analysis", + "Orchestration & Management", + "Platform", + "Provisioning", + "Runtime", + "Security & Compliance", + "Serverless", + "Tools", + "Uncategorized" + ] + }, + "subCategory": { + "type": "string", + "enum": [ + "Academic", + "API Gateway", + "Application Definition & Image Build", + "Automation & Configuration", + "Certified CNFs", + "Certified Kubernetes - Distribution", + "Certified Kubernetes - Hosted", + "Certified Kubernetes - Installer", + "Chaos Engineering", + "Cloud Native Network", + "Cloud Native Storage", + "Container Registry", + "Container Runtime", + "Continuous Integration & Delivery", + "Continuous Optimization", + "Coordination & Service Discovery", + "Database", + "Debugging and Observability", + "End User Supporter", + "Framework", + "Gold", + "Hosted Platform", + "Installable Platform", + "Key Management", + "Kubernetes Certified Service Provider", + "Kubernetes Training Partner", + "Logging", + "Metrics", + "Monitoring", + "Nonprofit", + "Packaging, Registries & Application Delivery", + "PaaS/Container Service", + "Platinum", + "Remote Procedure Call", + "Runtime", + "Scheduling & Orchestration", + "Security", + "Security & Compliance", + "Service Mesh", + "Service Proxy", + "Silver", + "Specifications", + "Streaming & Messaging", + "Toolchain", + "Tools", + "Tracing" + ] + }, + "shape": { + "description":"The shape of the model. The model will be displayed in the specific shape on kanvas.", + "type": "string", + "enum":[ + "rectangle", + "round-rectangle", + "bottom-round-rectangle", + "cut-rectangle", + "shape", + "circle", + "diamond", + "round-rectang", + "hexagon", + "rhomboid", + "triangle", + "cilinder", + "round-triangle", + "round-pentagon", + "sheild", + "vee", + "cylinder", + "round-heptagon", + "concave-hexagon", + "right-rhomboid", + "barrel", + "round-diamond" + ] + }, + "primaryColor": { + "description":"The primary color of the model. This will be the color of the background if the image has one.", + "type": "string", + "pattern": "^#[0-9A-Fa-f]{6}$" + }, + "secondaryColor": { + "type": "string", + "pattern": "^#[0-9A-Fa-f]{6}$" + }, + "svgColor": { + "type": "string", + "default": "" + }, + "svgWhite": { + "type": "string" + }, + "svgComplete": { + "type": "string" + }, + "isAnnotation": { + "type": "boolean", + "title": "Is Annotation", + "default": false, + "description": "Indicate whether this model is an annotation. Annotation means that this model is not a real model but an svg image", + "x-rjsf-grid-area": "6" + }, + "publishToRegistry": { + "type": "boolean", + "title": "Publish to Registry", + "default": true, + "description": "Indicate whether to publish this model to the registry. If marked as false then the model won't be registered in the registry", + "x-rjsf-grid-area": "6" + } + }, + "required": [ + "registrant", + "modelDisplayName", + "model" + ] } }, - "required": [ - "url" - ] + "required": ["url"] } } ], - "required": [ - "uploadType" - ] + "required": ["uploadType"] } diff --git a/schemas/configuration/uiSchemaModelImport.json b/schemas/configuration/uiSchemaModelImport.json index 038796fd..ea24b633 100644 --- a/schemas/configuration/uiSchemaModelImport.json +++ b/schemas/configuration/uiSchemaModelImport.json @@ -1,6 +1,38 @@ { - "uploadType": { - "ui:widget": "radio" + "uploadType": { + "ui:widget": "radio" + }, + "model": { + "ui:options": { + "expand": true }, - "ui:order" : [ "uploadType", "file", "url"] - } + "styles": { + "padding": 0, + "margin": 0 + }, + + "primaryColor": { + "ui:widget": "color" + }, + "secondaryColor": { + "ui:widget": "color" + }, + + "ui:order": [ + "modelDisplayName", + "registrant", + "category", + "subCategory", + "model", + "primaryColor", + "secondaryColor", + "shape", + "svgColor", + "svgComplete", + "svgWhite", + "isAnnotation", + "publishToRegistry" + ] + }, + "ui:order": ["uploadType", "file", "url"] +} diff --git a/utils/component/generator.go b/utils/component/generator.go index 417d59cd..288f2780 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -82,9 +82,9 @@ func Generate(crd string) (component.ComponentDefinition, error) { } scope, _ := extractCueValueFromPath(crdCue, DefaultPathConfig.ScopePath) if scope == "Cluster" { - cmp.Metadata.AdditionalProperties["isNamespaced"] = false + cmp.Metadata.IsNamespaced = false } else if scope == "Namespaced" { - cmp.Metadata.AdditionalProperties["isNamespaced"] = true + cmp.Metadata.IsNamespaced = true } cmp.Component.Kind = name if group != "" { diff --git a/utils/error.go b/utils/error.go index 3981a282..b1e50625 100644 --- a/utils/error.go +++ b/utils/error.go @@ -46,6 +46,7 @@ var ( ErrCopyFileCode = "replace_me" ErrCloseFileCode = "replace_me" ErrCompressToTarGZCode = "meshkit-11248" + ErrOpenFileCode = "replace_me" ) var ( ErrExtractType = errors.New( @@ -230,3 +231,6 @@ func ErrCloseFile(err error) error { []string{"Check for issues with file permissions or disk space and try again."}, ) } +func ErrOpenFile(file string) error { + return errors.New(ErrOpenFileCode, errors.Alert, []string{"unable to open file: ", file}, []string{}, []string{"The file does not exist in the location"}, []string{"Make sure to upload the correct file"}) +} diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index 69bf53af..e3a244fc 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -3,7 +3,6 @@ package kubernetes import ( "context" "encoding/json" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -35,7 +34,6 @@ func GetAllCustomResourcesInCluster(ctx context.Context, client rest.Interface) } var xcrd CRD gvks := []*schema.GroupVersionResource{} - //err = utils.Unmarshal(string(crdresult), &xcrd) err = json.Unmarshal(crdresult, &xcrd) if err != nil { return nil, err diff --git a/utils/utils.go b/utils/utils.go index 73f1141f..77b2dd64 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,7 +1,9 @@ package utils import ( + "archive/tar" "bytes" + "compress/gzip" "encoding/json" "errors" "fmt" @@ -452,3 +454,135 @@ func FindEntityType(content []byte) (entity.EntityType, error) { } return "", ErrInvalidSchemaVersion } + +// RecursiveCastMapStringInterfaceToMapStringInterface will convert a +// map[string]interface{} recursively => map[string]interface{} +func RecursiveCastMapStringInterfaceToMapStringInterface(in map[string]interface{}) map[string]interface{} { + res := ConvertMapInterfaceMapString(in) + out, ok := res.(map[string]interface{}) + if !ok { + fmt.Println("failed to cast") + } + + return out +} + +// ConvertMapInterfaceMapString converts map[interface{}]interface{} => map[string]interface{} +// +// It will also convert []interface{} => []string +func ConvertMapInterfaceMapString(v interface{}) interface{} { + switch x := v.(type) { + case map[interface{}]interface{}: + m := map[string]interface{}{} + for k, v2 := range x { + switch k2 := k.(type) { + case string: + m[k2] = ConvertMapInterfaceMapString(v2) + default: + m[fmt.Sprint(k)] = ConvertMapInterfaceMapString(v2) + } + } + v = m + + case []interface{}: + for i, v2 := range x { + x[i] = ConvertMapInterfaceMapString(v2) + } + + case map[string]interface{}: + for k, v2 := range x { + x[k] = ConvertMapInterfaceMapString(v2) + } + } + + return v +} +func ConvertToJSONCompatible(data interface{}) interface{} { + switch v := data.(type) { + case map[interface{}]interface{}: + m := make(map[string]interface{}) + for key, value := range v { + m[key.(string)] = ConvertToJSONCompatible(value) + } + return m + case []interface{}: + for i, item := range v { + v[i] = ConvertToJSONCompatible(item) + } + } + return data +} +func YAMLToJSON(content []byte) ([]byte, error) { + var jsonData interface{} + if err := yaml.Unmarshal(content, &jsonData); err == nil { + jsonData = ConvertToJSONCompatible(jsonData) + convertedContent, err := json.Marshal(jsonData) + if err == nil { + content = convertedContent + } else { + return nil, ErrUnmarshal(err) + } + } else { + return nil, ErrUnmarshal(err) + } + return content, nil +} +func ExtractFile(filePath string, destDir string) error { + if IsTarGz(filePath) { + return ExtractTarGz(destDir, filePath) + } else if IsZip(filePath) { + return ExtractZip(destDir, filePath) + } + return ErrExtractType +} + +// Convert path to svg Data +func ReadSVGData(baseDir, path string) (string, error) { + fullPath := baseDir + path + svgData, err := os.ReadFile(fullPath) + if err != nil { + return "", err + } + return string(svgData), nil +} +func Compress(src string, buf io.Writer) error { + zr := gzip.NewWriter(buf) + defer zr.Close() + tw := tar.NewWriter(zr) + defer tw.Close() + + return filepath.Walk(src, func(file string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(fi, file) + if err != nil { + return err + } + + relPath, err := filepath.Rel(src, file) + if err != nil { + return err + } + header.Name = filepath.ToSlash(relPath) + + if err := tw.WriteHeader(header); err != nil { + return err + } + + if !fi.IsDir() { + data, err := os.Open(file) + if err != nil { + return err + } + defer data.Close() + + _, err = io.Copy(tw, data) + if err != nil { + return err + } + } + return nil + }) +} diff --git a/utils/walker/github.go b/utils/walker/github.go index 085c0952..bd58eaf5 100644 --- a/utils/walker/github.go +++ b/utils/walker/github.go @@ -7,7 +7,7 @@ import ( "path/filepath" "strings" "sync" - + "errors" "github.com/sirupsen/logrus" ) @@ -164,8 +164,7 @@ func (g *Github) walker(path string, isFile bool) error { if !ok { return fmt.Errorf("[GithubWalker]: GithubAPI responded with: forbidden") } - - return fmt.Errorf(message) + return errors.New(message) } return fmt.Errorf("file not found") }