From 90d348d1b22da32f307014e5d2d690b5bf31e053 Mon Sep 17 00:00:00 2001 From: Eugene Y Date: Thu, 22 Dec 2022 12:59:04 +1100 Subject: [PATCH] feat: added metadata properties substitutions support (#24) Signed-off-by: Eugene Yarshevich --- go.mod | 2 +- internal/compose/convert.go | 81 ++-------------- internal/compose/convert_test.go | 12 +-- internal/compose/templates.go | 121 ++++++++++++++++++++++++ internal/compose/templates_test.go | 145 +++++++++++++++++++++++++++++ 5 files changed, 280 insertions(+), 81 deletions(-) create mode 100644 internal/compose/templates.go create mode 100644 internal/compose/templates_test.go diff --git a/go.mod b/go.mod index a8c6ff8..ebc6293 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/compose-spec/compose-go v1.6.0 github.com/imdario/mergo v0.3.13 + github.com/mitchellh/mapstructure v1.5.0 github.com/score-spec/score-go v0.0.0-20221019054335-3510902b5f8b github.com/spf13/cobra v1.6.0 github.com/stretchr/testify v1.8.0 @@ -16,7 +17,6 @@ require ( github.com/distribution/distribution/v3 v3.0.0-20220725133111-4bf3547399eb // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/internal/compose/convert.go b/internal/compose/convert.go index 2ce8502..86e0bba 100644 --- a/internal/compose/convert.go +++ b/internal/compose/convert.go @@ -10,10 +10,7 @@ package compose import ( "errors" "fmt" - "os" - "regexp" "sort" - "strings" compose "github.com/compose-spec/compose-go/types" score "github.com/score-spec/score-go/types" @@ -21,12 +18,16 @@ import ( // ConvertSpec converts SCORE specification into docker-compose configuration. func ConvertSpec(spec *score.WorkloadSpec) (*compose.Project, ExternalVariables, error) { + context, err := buildContext(spec.Metadata, spec.Resources) + if err != nil { + return nil, nil, fmt.Errorf("preparing context: %w", err) + } for _, cSpec := range spec.Containers { - var externalVars = ExternalVariables(resourcesMap(spec.Resources).listVars()) + var externalVars = ExternalVariables(context.ListEnvVars()) var env = make(compose.MappingWithEquals, len(cSpec.Variables)) for key, val := range cSpec.Variables { - var envVarVal = os.Expand(val, resourcesMap(spec.Resources).mapVar) + var envVarVal = context.Substitute(val) env[key] = &envVarVal } @@ -68,7 +69,7 @@ func ConvertSpec(spec *score.WorkloadSpec) (*compose.Project, ExternalVariables, } volumes[idx] = compose.ServiceVolumeConfig{ Type: "volume", - Source: resourceRefRegex.ReplaceAllString(vol.Source, "$1"), + Source: context.Substitute(vol.Source), Target: vol.Target, ReadOnly: vol.ReadOnly, } @@ -104,71 +105,3 @@ func ConvertSpec(spec *score.WorkloadSpec) (*compose.Project, ExternalVariables, return nil, nil, errors.New("workload does not have any containers to convert into a compose service") } - -// resourceRefRegex extracts the resource ID from the resource reference: '${resources.RESOURCE_ID}' -var resourceRefRegex = regexp.MustCompile(`\${resources\.(.+)}`) - -// resourcesMap is an internal utility type to group some helper methods. -type resourcesMap map[string]score.ResourceSpec - -// listResourcesVars lists all available environment variables based on the declared resources properties. -func (r resourcesMap) listVars() map[string]interface{} { - var vars = make(map[string]interface{}) - for resName, res := range r { - for propName, prop := range res.Properties { - var envVar string - switch res.Type { - case "environment": - envVar = strings.ToUpper(propName) - default: - envVar = strings.ToUpper(fmt.Sprintf("%s_%s", resName, propName)) - } - - envVar = strings.Replace(envVar, "-", "_", -1) - envVar = strings.Replace(envVar, ".", "_", -1) - - vars[envVar] = prop.Default - } - } - return vars -} - -// mapResourceVar maps resources properties references. -// Returns an empty string if the reference can't be resolved. -func (r resourcesMap) mapVar(ref string) string { - if ref == "$" { - return ref - } - - var segments = strings.SplitN(ref, ".", 3) - if segments[0] != "resources" || len(segments) != 3 { - return "" - } - - var resName = segments[1] - var propName = segments[2] - if res, ok := r[resName]; ok { - if prop, ok := res.Properties[propName]; ok { - var envVar string - switch res.Type { - case "environment": - envVar = strings.ToUpper(propName) - default: - envVar = strings.ToUpper(fmt.Sprintf("%s_%s", resName, propName)) - } - - envVar = strings.Replace(envVar, "-", "_", -1) - envVar = strings.Replace(envVar, ".", "_", -1) - - if prop.Default != nil { - envVar += fmt.Sprintf("-%v", prop.Default) - } else if prop.Required { - envVar += "?err" - } - - return fmt.Sprintf("${%s}", envVar) - } - } - - return "" -} diff --git a/internal/compose/convert_test.go b/internal/compose/convert_test.go index 187ad35..faa89fa 100644 --- a/internal/compose/convert_test.go +++ b/internal/compose/convert_test.go @@ -175,14 +175,14 @@ func TestScoreConvert(t *testing.T) { }, }, Vars: ExternalVariables{ - "DEBUG": false, + "DEBUG": "false", "LOGS_LEVEL": "WARN", "APP_DB_HOST": "localhost", - "APP_DB_PORT": 5432, - "APP_DB_NAME": nil, - "APP_DB_USER_NAME": nil, - "APP_DB_PASSWORD": nil, - "DNS_DOMAIN": nil, + "APP_DB_PORT": "5432", + "APP_DB_NAME": "", + "APP_DB_USER_NAME": "", + "APP_DB_PASSWORD": "", + "DNS_DOMAIN": "", }, }, diff --git a/internal/compose/templates.go b/internal/compose/templates.go new file mode 100644 index 0000000..88aa09a --- /dev/null +++ b/internal/compose/templates.go @@ -0,0 +1,121 @@ +/* +Apache Score +Copyright 2022 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). +*/ +package compose + +import ( + "fmt" + "log" + "os" + "regexp" + "strings" + + "github.com/mitchellh/mapstructure" + + score "github.com/score-spec/score-go/types" +) + +// templatesContext ia an utility type that provides a context for '${...}' templates substitution +type templatesContext map[string]string + +// buildContext initializes a new templatesContext instance +func buildContext(metadata score.WorkloadMeta, resources score.ResourcesSpecs) (templatesContext, error) { + var ctx = make(map[string]string) + + var metadataMap = make(map[string]interface{}) + if decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + TagName: "json", + Result: &metadataMap, + }); err != nil { + return nil, err + } else { + decoder.Decode(metadata) + for key, val := range metadataMap { + var ref = fmt.Sprintf("metadata.%s", key) + if _, exists := ctx[ref]; exists { + return nil, fmt.Errorf("ambiguous property reference '%s'", ref) + } + ctx[ref] = fmt.Sprintf("%v", val) + } + } + + for resName, res := range resources { + ctx[fmt.Sprintf("resources.%s", resName)] = resName + + for propName, prop := range res.Properties { + var ref = fmt.Sprintf("resources.%s.%s", resName, propName) + + var envVar string + switch res.Type { + case "environment": + envVar = strings.ToUpper(propName) + default: + envVar = strings.ToUpper(fmt.Sprintf("%s_%s", resName, propName)) + } + + envVar = strings.Replace(envVar, "-", "_", -1) + envVar = strings.Replace(envVar, ".", "_", -1) + + if prop.Default != nil { + envVar += fmt.Sprintf("-%v", prop.Default) + } else if prop.Required { + envVar += "?err" + } + + ctx[ref] = fmt.Sprintf("${%s}", envVar) + } + } + + return ctx, nil +} + +// Substitute replaces all matching '${...}' templates in a source string +func (context templatesContext) Substitute(src string) string { + return os.Expand(src, context.mapVar) +} + +// MapVar replaces objects and properties references with corresponding values +// Returns an empty string if the reference can't be resolved +func (context templatesContext) mapVar(ref string) string { + if ref == "" { + return "" + } + + // NOTE: os.Expand(..) would invoke a callback function with "$" as an argument for escaped sequences. + // "$${abc}" is treated as "$$" pattern and "{abc}" static text. + // The first segment (pattern) would trigger a callback function call. + // By returning "$" value we would ensure that escaped sequences would remain in the source text. + // For example "$${abc}" would result in "${abc}" after os.Expand(..) call. + if ref == "$" { + return ref + } + + if res, ok := context[ref]; ok { + return res + } + + log.Printf("Warning: Can not resolve '%s'. Resource or property is not declared.", ref) + return "" +} + +// composeEnvVarReferencePattern defines the rule for compose environment variable references +// Possible documented references for compose v3.5: +// - ${ENV_VAR} +// - ${ENV_VAR?err} +// - ${ENV_VAR-default} +var envVarPattern = regexp.MustCompile(`\$\{(\w+)(?:\-(.+?)|\?.+)?\}$`) + +// ListEnvVars reports all environment variables used by templatesContext +func (context templatesContext) ListEnvVars() map[string]interface{} { + var vars = make(map[string]interface{}) + for _, ref := range context { + if matches := envVarPattern.FindStringSubmatch(ref); len(matches) == 3 { + vars[matches[1]] = matches[2] + } + } + return vars +} diff --git a/internal/compose/templates_test.go b/internal/compose/templates_test.go new file mode 100644 index 0000000..d07cce8 --- /dev/null +++ b/internal/compose/templates_test.go @@ -0,0 +1,145 @@ +/* +Apache Score +Copyright 2022 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). +*/ +package compose + +import ( + "testing" + + score "github.com/score-spec/score-go/types" + assert "github.com/stretchr/testify/assert" +) + +func TestBuildContext(t *testing.T) { + var meta = score.WorkloadMeta{ + Name: "test-name", + } + + var resources = score.ResourcesSpecs{ + "env": score.ResourceSpec{ + Type: "environment", + Properties: map[string]score.ResourcePropertySpec{ + "DEBUG": {Required: false, Default: true}, + }, + }, + "db": score.ResourceSpec{ + Type: "postgres", + Properties: map[string]score.ResourcePropertySpec{ + "host": {Required: true, Default: "."}, + "port": {Required: true, Default: "5342"}, + "name": {Required: true}, + }, + }, + } + + context, err := buildContext(meta, resources) + assert.NoError(t, err) + + assert.Equal(t, templatesContext{ + "metadata.name": "test-name", + + "resources.env": "env", + "resources.env.DEBUG": "${DEBUG-true}", + + "resources.db": "db", + "resources.db.host": "${DB_HOST-.}", + "resources.db.port": "${DB_PORT-5342}", + "resources.db.name": "${DB_NAME?err}", + }, context) +} + +func TestMapVar(t *testing.T) { + var context = templatesContext{ + "metadata.name": "test-name", + + "resources.env": "env", + "resources.env.DEBUG": "${DEBUG-true}", + + "resources.db": "db", + "resources.db.host": "${DB_HOST-.}", + "resources.db.port": "${DB_PORT-5342}", + "resources.db.name": "${DB_NAME?err}", + } + + assert.Equal(t, "", context.mapVar("")) + assert.Equal(t, "$", context.mapVar("$")) + + assert.Equal(t, "test-name", context.mapVar("metadata.name")) + assert.Equal(t, "", context.mapVar("metadata.name.nil")) + assert.Equal(t, "", context.mapVar("metadata.nil")) + + assert.Equal(t, "${DEBUG-true}", context.mapVar("resources.env.DEBUG")) + + assert.Equal(t, "db", context.mapVar("resources.db")) + assert.Equal(t, "${DB_HOST-.}", context.mapVar("resources.db.host")) + assert.Equal(t, "${DB_PORT-5342}", context.mapVar("resources.db.port")) + assert.Equal(t, "${DB_NAME?err}", context.mapVar("resources.db.name")) + assert.Equal(t, "", context.mapVar("resources.db.name.nil")) + assert.Equal(t, "", context.mapVar("resources.db.nil")) + assert.Equal(t, "", context.mapVar("resources.nil")) + assert.Equal(t, "", context.mapVar("nil.db.name")) +} + +func TestSubstitute(t *testing.T) { + var context = templatesContext{ + "metadata.name": "test-name", + + "resources.env": "env", + "resources.env.DEBUG": "${DEBUG-true}", + + "resources.db": "db", + "resources.db.host": "${DB_HOST-.}", + "resources.db.port": "${DB_PORT-5342}", + "resources.db.name": "${DB_NAME?err}", + } + + assert.Equal(t, "", context.Substitute("")) + assert.Equal(t, "abc", context.Substitute("abc")) + assert.Equal(t, "abc $ abc", context.Substitute("abc $$ abc")) + assert.Equal(t, "${abc}", context.Substitute("$${abc}")) + + assert.Equal(t, "The name is 'test-name'", context.Substitute("The name is '${metadata.name}'")) + assert.Equal(t, "The name is ''", context.Substitute("The name is '${metadata.nil}'")) + + assert.Equal(t, "resources.env.DEBUG", context.Substitute("resources.env.DEBUG")) + + assert.Equal(t, "db", context.Substitute("${resources.db}")) + assert.Equal(t, + "postgresql://:@${DB_HOST-.}:${DB_PORT-5342}/${DB_NAME?err}", + context.Substitute("postgresql://${resources.db.user}:${resources.db.password}@${resources.db.host}:${resources.db.port}/${resources.db.name}")) +} + +func TestEnvVarPattern(t *testing.T) { + assert.Equal(t, []string(nil), envVarPattern.FindStringSubmatch("")) + assert.Equal(t, []string(nil), envVarPattern.FindStringSubmatch("ENV_VAR")) + assert.Equal(t, []string(nil), envVarPattern.FindStringSubmatch("${}")) + + assert.Equal(t, []string{"${ENV_VAR}", "ENV_VAR", ""}, envVarPattern.FindStringSubmatch("${ENV_VAR}")) + assert.Equal(t, []string{"${ENV_VAR?err}", "ENV_VAR", ""}, envVarPattern.FindStringSubmatch("${ENV_VAR?err}")) + assert.Equal(t, []string{"${ENV_VAR-default}", "ENV_VAR", "default"}, envVarPattern.FindStringSubmatch("${ENV_VAR-default}")) +} + +func TestListEnvVars(t *testing.T) { + var context = templatesContext{ + "metadata.name": "test-name", + + "resources.env": "env", + "resources.env.DEBUG": "${DEBUG-true}", + + "resources.db": "db", + "resources.db.host": "${DB_HOST-.}", + "resources.db.port": "${DB_PORT-5342}", + "resources.db.name": "${DB_NAME?err}", + } + + assert.Equal(t, map[string]interface{}{ + "DEBUG": "true", + "DB_HOST": ".", + "DB_PORT": "5342", + "DB_NAME": "", + }, context.ListEnvVars()) +}