From c3942e54c4874263d766dcc897e305482724fa32 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 27 Jul 2024 03:02:33 -0600 Subject: [PATCH] remove support for json as input config file --- validator/README.md | 25 ++--- validator/main.go | 121 ++++----------------- validator/shelltests/json_replacement.json | 1 - validator/shelltests/json_replacement.test | 3 - 4 files changed, 29 insertions(+), 121 deletions(-) delete mode 100644 validator/shelltests/json_replacement.json delete mode 100644 validator/shelltests/json_replacement.test diff --git a/validator/README.md b/validator/README.md index e9411b4..86e33a2 100644 --- a/validator/README.md +++ b/validator/README.md @@ -1,7 +1,6 @@ ## OpenTelemetry SDK Configuration Validator -This application will replace environment variables in values of valid yaml or -json files, following the rules of [file configuration environment variable +This application will replace environment variables in values of valid yaml files, following the rules of [file configuration environment variable substitution](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/file-configuration.md#environment-variable-substitution), before validating that result against the [OpenTelemetry SDK Configuration schema](https://github.com/open-telemetry/opentelemetry-configuration/). @@ -24,11 +23,11 @@ $ make validator-docker ### Usage -The command `otel_config_validator` takes one argument, the path to the yaml or -json configuration file and optionally the path to a file to output the -configuration after environment variable expansion and validation has been done. -The format (json or yaml) of the output is based on the extension (`.json` or -`yml`/`.yaml`) of the output file name. +The command `otel_config_validator` takes one argument, the path to the yaml +file and optionally the path to a file to output the configuration after +environment variable expansion and validation has been done. The format (json or +yaml) of the output is based on the extension (`.json` or `yml`/`.yaml`) of the +output file name. ``` $ ./otel_config_validator -o out.json ../examples/kitchen-sink.yaml @@ -46,15 +45,9 @@ $ docker run -v $(pwd):/opt/otel_config_validator otel_config_validator:current With the above docker command the output file, `out.yaml`, will be owned by `root:root` but be readable by any user. -Environment variable substitution is supported with the syntax `${VARIABLE}`. -Default values are supported in the form `${VARIABLE:-default}`. - -In the case of json input only strings can be the result of substitution. To -ensure only values are replaced the input must be parsed as valid json or yaml -and in the case of json a value like `${VARIABLE}` will always have to be double -quoted as `"${VARIABLE}"` so will always remain double quoted. If you need to -substitute in a boolean, integer or float please use yaml for the input -configuration file. +Environment variable substitution is supported with the syntax `${VARIABLE}` or +`${env:VARIABLE}`. Default values are supported in the form +`${VARIABLE:-default}`. ### Testing diff --git a/validator/main.go b/validator/main.go index fcb83e1..51e1f3a 100644 --- a/validator/main.go +++ b/validator/main.go @@ -129,54 +129,35 @@ func decodeFile(configFile string, outfileExt string) (interface{}, []byte) { var j []byte var d []byte var u interface{} - var v interface{} var jsonInterface interface{} - - data, err := os.ReadFile(configFile) + + y := decodeYaml(configFile) + replaceYamlVariables(&y) + + err := y.Decode(&u) if err != nil { log.Fatal(err) } - ext := filepath.Ext(configFile) - if isYamlExt(ext) { - y := decodeYaml(configFile) - replaceYamlVariables(&y) - - err = y.Decode(&u) + if len(y.Content) > 0 { + d, err = yaml.Marshal(&y.Content[0]) if err != nil { log.Fatal(err) } - - if len(y.Content) > 0 { - d, err = yaml.Marshal(&y.Content[0]) - if err != nil { - log.Fatal(err) - } - - j, err = k8syaml.YAMLToJSON(d) - if err != nil { - log.Fatalf("Error encoding yaml to json for validation: %+v", err) - } - if err := json.Unmarshal(j, &jsonInterface); err != nil { - log.Fatalf("Invalid json file from yaml file %s: %+v", configFile, err) - } + j, err = k8syaml.YAMLToJSON(d) + if err != nil { + log.Fatalf("Error encoding yaml to json for validation: %+v", err) } - toWrite := convertYamlNode(y, outfileExt) - - return jsonInterface, toWrite - } - - if err := json.Unmarshal(data, &v); err != nil { - log.Fatalf("Invalid json file %s: %#v", configFile, err) + if err := json.Unmarshal(j, &jsonInterface); err != nil { + log.Fatalf("Invalid json from yaml file %s: %+v", configFile, err) + } } - expandedConfig, b := replaceJsonVariables(v) + toWrite := convertYamlNode(y, outfileExt) - toWrite := convertJsonBytes(b, outfileExt) - - return expandedConfig, toWrite + return jsonInterface, toWrite } func isYamlExt(ext string) bool { @@ -205,18 +186,9 @@ func convertYamlNode(n yaml.Node, ext string) []byte { return d } -func convertJsonBytes(jsonBytes []byte, ext string) []byte { - if isYamlExt(ext) { - b, _ := k8syaml.JSONToYAML(jsonBytes) - return b - } - - return jsonBytes -} - func decodeYaml(file string) yaml.Node { var node yaml.Node - + body, err := os.ReadFile(file) if err != nil { log.Fatalf("Failed to read configuration file %s: %v", file, err) @@ -319,59 +291,6 @@ func handleScalarNode(n *yaml.Node) { } -// json variable replacement is basic as the json value that -// is an environment variable will always be quoted, so only -// strings are supported -func replaceJsonVariables(c interface{}) (interface{}, []byte) { - expandedConfig := make(map[string]any) - - m, _ := c.(map[string]any) - for k := range m { - val := expandJsonValues(m[k]) - expandedConfig[k] = val - } - b, err := json.Marshal(expandedConfig) - if err != nil { - log.Fatalf("json.Marshal: %+v", err) - } - - y, err := k8syaml.JSONToYAML(b) - if err != nil { - log.Fatalf("Error converting json result to yaml: %+v", err) - } - - return expandedConfig, y -} - -func expandJsonValues(value interface{}) any { - switch v := value.(type) { - case string: - if !strings.Contains(v, "${") || !strings.Contains(v, "}") { - return v - } - - return expandString(v) - case []any: - l := []any{} - for _, e := range v { - newElement := expandJsonValues(e) - l = append(l, newElement) - } - return l - case map[string]any: - newMap := make(map[string]any) - - for k, v := range v { - updated := expandJsonValues(v) - newMap[k] = updated - } - - return newMap - } - - return value -} - // Replace environment variables, like ${EXAMPLE}, with their value. // This does not use `os.ExpandVars` in order to support defaults // for missing variables, ${VAR:-default} @@ -392,11 +311,11 @@ func expandString(s string) string { // iterates over a string to find all environment variables // returns a map of variables to strings or nil func findAllVars(s string) map[string]interface{} { - var envVar string + var envVar string var newValue interface{} var isSet bool var substr string - + result := make(map[string]interface{}) lenS := len(s) @@ -416,8 +335,8 @@ func findAllVars(s string) map[string]interface{} { } else { envVar = substr[openIndex+6 : closeIndex] } - - fullEnvVar := substr[openIndex : closeIndex+1] + + fullEnvVar := substr[openIndex : closeIndex+1] maybeDefaultIndex := strings.Index(envVar, ":-") diff --git a/validator/shelltests/json_replacement.json b/validator/shelltests/json_replacement.json deleted file mode 100644 index 1f34b6c..0000000 --- a/validator/shelltests/json_replacement.json +++ /dev/null @@ -1 +0,0 @@ -{"attribute_limits":{"attribute_count_limit":128,"attribute_value_length_limit":512},"disabled":false,"file_format":"0.1","resource":{"attributes":{"service.name":"${OTEL_SERVICE_NAME:-unknown_service}"}}} diff --git a/validator/shelltests/json_replacement.test b/validator/shelltests/json_replacement.test deleted file mode 100644 index 1756d16..0000000 --- a/validator/shelltests/json_replacement.test +++ /dev/null @@ -1,3 +0,0 @@ -./otel_config_validator -o out/out.json shelltests/json_replacement.json ->>> ->>>= 0