Skip to content

Commit

Permalink
run interpolation after merge, but for required attributes
Browse files Browse the repository at this point in the history 
Signed-off-by: Nicolas De Loof <[email protected]>
  • Loading branch information
@ndeloof
ndeloof committed Jul 8, 2024
1 parent f4d8eb8 commit d7d9f94
Show file tree
Hide file tree
Showing 7 changed files with 106 additions and 45 deletions.
2 changes: 1 addition & 1 deletion cli/options_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ func TestProjectWithDotEnv(t *testing.T) {
"compose-with-variables.yaml",
}, WithName("my_project"), WithEnvFiles(), WithDotEnv)
assert.NilError(t, err)
p, err := ProjectFromOptions(context.TODO(), opts)
p, err := opts.LoadProject(context.TODO())
assert.NilError(t, err)
service, err := p.GetService("simple")
assert.NilError(t, err)
Expand Down
1 change: 0 additions & 1 deletion cli/testdata/env-file/compose-with-env-files.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: "3"
services:
simple:
image: nginx
Expand Down
21 changes: 21 additions & 0 deletions loader/extends.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,11 @@ import (
"path/filepath"

"github.com/compose-spec/compose-go/v2/consts"
"github.com/compose-spec/compose-go/v2/interpolation"
"github.com/compose-spec/compose-go/v2/override"
"github.com/compose-spec/compose-go/v2/paths"
"github.com/compose-spec/compose-go/v2/template"
"github.com/compose-spec/compose-go/v2/transform"
"github.com/compose-spec/compose-go/v2/types"
)

Expand Down Expand Up @@ -68,10 +71,22 @@ func applyServiceExtends(ctx context.Context, name string, services map[string]a
)
switch v := extends.(type) {
case map[string]any:
if opts.Interpolate != nil {
v, err = interpolation.Interpolate(v, *opts.Interpolate)
if err != nil {
return nil, err
}
}
ref = v["service"].(string)
file = v["file"]
opts.ProcessEvent("extends", v)
case string:
if opts.Interpolate != nil {
v, err = opts.Interpolate.Substitute(v, template.Mapping(opts.Interpolate.LookupValue))
if err != nil {
return nil, err
}
}
ref = v
opts.ProcessEvent("extends", map[string]any{"service": ref})
}
Expand Down Expand Up @@ -175,6 +190,12 @@ func getExtendsBaseFromFile(
)
}

// Attempt to make a canonical model so ResolveRelativePaths can operate on source:target short syntaxes
source, err = transform.Canonical(source, true)
if err != nil {
return nil, nil, err
}

var remotes []paths.RemoteResource
for _, loader := range opts.RemoteResourceLoaders() {
remotes = append(remotes, loader.Accept)
Expand Down
14 changes: 12 additions & 2 deletions loader/include.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
)

// loadIncludeConfig parse the required config from raw yaml
func loadIncludeConfig(source any) ([]types.IncludeConfig, error) {
func loadIncludeConfig(source any, options *Options) ([]types.IncludeConfig, error) {
if source == nil {
return nil, nil
}
Expand All @@ -45,13 +45,23 @@ func loadIncludeConfig(source any) ([]types.IncludeConfig, error) {
}
}
}
if options.Interpolate != nil {
for i, config := range configs {
interpolated, err := interp.Interpolate(config.(map[string]any), *options.Interpolate)
if err != nil {
return nil, err
}
configs[i] = interpolated
}
}

var requires []types.IncludeConfig
err := Transform(source, &requires)
return requires, err
}

func ApplyInclude(ctx context.Context, workingDir string, environment types.Mapping, model map[string]any, options *Options, included []string) error {
includeConfig, err := loadIncludeConfig(model["include"])
includeConfig, err := loadIncludeConfig(model["include"], options)
if err != nil {
return err
}
Expand Down
39 changes: 18 additions & 21 deletions loader/loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,23 @@ func loadYamlModel(ctx context.Context, config types.ConfigDetails, opts *Option
}
}

if opts.Interpolate != nil && !opts.SkipInterpolation {
dict, err = interp.Interpolate(dict, *opts.Interpolate)
if err != nil {
return nil, err
}
}

dict, err = transform.Canonical(dict, opts.SkipInterpolation)
if err != nil {
return nil, err
}

dict, err = override.EnforceUnicity(dict)
if err != nil {
return nil, err
}

if !opts.SkipDefaultValues {
dict, err = transform.SetDefaultValues(dict)
if err != nil {
Expand Down Expand Up @@ -415,13 +432,6 @@ func loadYamlFile(ctx context.Context, file types.ConfigFile, opts *Options, wor
return errors.New("Top-level object must be a mapping")
}

if opts.Interpolate != nil && !opts.SkipInterpolation {
cfg, err = interp.Interpolate(cfg, *opts.Interpolate)
if err != nil {
return err
}
}

fixEmptyNotNull(cfg)

if !opts.SkipExtends {
Expand Down Expand Up @@ -450,11 +460,6 @@ func loadYamlFile(ctx context.Context, file types.ConfigFile, opts *Options, wor
return err
}

dict, err = override.EnforceUnicity(dict)
if err != nil {
return err
}

if !opts.SkipValidation {
if err := schema.Validate(dict); err != nil {
return fmt.Errorf("validating %s: %w", file.Filename, err)
Expand All @@ -464,15 +469,7 @@ func loadYamlFile(ctx context.Context, file types.ConfigFile, opts *Options, wor
delete(dict, "version")
}
}

dict, err = transform.Canonical(dict, opts.SkipInterpolation)
if err != nil {
return err
}

// Canonical transformation can reveal duplicates, typically as ports can be a range and conflict with an override
dict, err = override.EnforceUnicity(dict)
return err
return nil
}

var processor PostProcessor
Expand Down
40 changes: 40 additions & 0 deletions loader/loader_yaml_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,46 @@ services:
})
}

func TestParseYAMLFilesInterpolateAfterMerge(t *testing.T) {
model, err := loadYamlModel(
context.TODO(), types.ConfigDetails{
ConfigFiles: []types.ConfigFile{
{
Filename: "test.yaml",
Content: []byte(`
services:
test:
image: foo
environment:
my_env: ${my_env?my_env must be set}
`),
},
{
Filename: "override.yaml",
Content: []byte(`
services:
test:
image: bar
environment:
my_env: ${my_env:-default}
`),
},
},
}, &Options{}, &cycleTracker{}, nil,
)
assert.NilError(t, err)
assert.DeepEqual(
t, model, map[string]interface{}{
"services": map[string]interface{}{
"test": map[string]interface{}{
"image": "bar",
"environment": []any{"my_env=${my_env:-default}"},
},
},
},
)
}

func TestParseYAMLFilesMergeOverride(t *testing.T) {
model, err := loadYamlModel(context.TODO(), types.ConfigDetails{
ConfigFiles: []types.ConfigFile{
Expand Down
34 changes: 14 additions & 20 deletions schema/compose-spec.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,8 @@
},
"additionalProperties": false
},
"cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"cap_add": {"type": "array", "items": {"type": "string"}},
"cap_drop": {"type": "array", "items": {"type": "string"}},
"cgroup": {"type": "string", "enum": ["host", "private"]},
"cgroup_parent": {"type": "string"},
"command": {"$ref": "#/definitions/command"},
Expand Down Expand Up @@ -215,9 +215,9 @@
]
},
"device_cgroup_rules": {"$ref": "#/definitions/list_of_strings"},
"devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"devices": {"type": "array", "items": {"type": "string"}},
"dns": {"$ref": "#/definitions/string_or_list"},
"dns_opt": {"type": "array","items": {"type": "string"}, "uniqueItems": true},
"dns_opt": {"type": "array","items": {"type": "string"}},
"dns_search": {"$ref": "#/definitions/string_or_list"},
"domainname": {"type": "string"},
"entrypoint": {"$ref": "#/definitions/command"},
Expand All @@ -229,8 +229,7 @@
"items": {
"type": ["string", "number"],
"format": "expose"
},
"uniqueItems": true
}
},
"extends": {
"oneOf": [
Expand All @@ -247,14 +246,13 @@
}
]
},
"external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"external_links": {"type": "array", "items": {"type": "string"}},
"extra_hosts": {"$ref": "#/definitions/list_or_dict"},
"group_add": {
"type": "array",
"items": {
"type": ["string", "number"]
},
"uniqueItems": true
}
},
"healthcheck": {"$ref": "#/definitions/healthcheck"},
"hostname": {"type": "string"},
Expand All @@ -263,7 +261,7 @@
"ipc": {"type": "string"},
"isolation": {"type": "string"},
"labels": {"$ref": "#/definitions/list_or_dict"},
"links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"links": {"type": "array", "items": {"type": "string"}},
"logging": {
"type": "object",

Expand Down Expand Up @@ -349,8 +347,7 @@
"patternProperties": {"^x-": {}}
}
]
},
"uniqueItems": true
}
},
"privileged": {"type": ["boolean", "string"]},
"profiles": {"$ref": "#/definitions/list_of_strings"},
Expand All @@ -365,7 +362,7 @@
"scale": {
"type": ["integer", "string"]
},
"security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"security_opt": {"type": "array", "items": {"type": "string"}},
"shm_size": {"type": ["number", "string"]},
"secrets": {"$ref": "#/definitions/service_config_or_secret"},
"sysctls": {"$ref": "#/definitions/list_or_dict"},
Expand Down Expand Up @@ -431,13 +428,11 @@
"patternProperties": {"^x-": {}}
}
]
},
"uniqueItems": true
}
},
"volumes_from": {
"type": "array",
"items": {"type": "string"},
"uniqueItems": true
"items": {"type": "string"}
},
"working_dir": {"type": "string"}
},
Expand Down Expand Up @@ -832,8 +827,7 @@

"list_of_strings": {
"type": "array",
"items": {"type": "string"},
"uniqueItems": true
"items": {"type": "string"}
},

"list_or_dict": {
Expand All @@ -847,7 +841,7 @@
},
"additionalProperties": false
},
{"type": "array", "items": {"type": "string"}, "uniqueItems": true}
{"type": "array", "items": {"type": "string"}}
]
},

Expand Down

0 comments on commit d7d9f94

Please sign in to comment.