From 413b276b5b9dc82949c3fda3d1f1cd65ea18f23f Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 28 May 2024 12:25:48 -0600 Subject: [PATCH 01/30] add tool for config file validation and envvar replacement --- .gitignore | 6 +- validator/Dockerfile | 21 ++ validator/Dockerfile.shelltest | 15 ++ validator/LICENSE | 201 +++++++++++++++++ validator/Makefile | 18 ++ validator/README.md | 52 +++++ validator/go.mod | 16 ++ validator/go.sum | 15 ++ validator/main.go | 241 +++++++++++++++++++++ validator/main_test.go | 51 +++++ validator/shelltests/help.test | 15 ++ validator/shelltests/json_out.test | 8 + validator/shelltests/missing_arg.test | 4 + validator/shelltests/missing_ext.test | 4 + validator/shelltests/missing_required.test | 5 + validator/shelltests/missing_required.yaml | 1 + validator/shelltests/test.yaml | 1 + validator/shelltests/yaml_out.test | 8 + 18 files changed, 681 insertions(+), 1 deletion(-) create mode 100644 validator/Dockerfile create mode 100644 validator/Dockerfile.shelltest create mode 100644 validator/LICENSE create mode 100644 validator/Makefile create mode 100644 validator/README.md create mode 100644 validator/go.mod create mode 100644 validator/go.sum create mode 100644 validator/main.go create mode 100644 validator/main_test.go create mode 100644 validator/shelltests/help.test create mode 100644 validator/shelltests/json_out.test create mode 100644 validator/shelltests/missing_arg.test create mode 100644 validator/shelltests/missing_ext.test create mode 100644 validator/shelltests/missing_required.test create mode 100644 validator/shelltests/missing_required.yaml create mode 100644 validator/shelltests/test.yaml create mode 100644 validator/shelltests/yaml_out.test diff --git a/.gitignore b/.gitignore index 24e60ca..72bb05d 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,8 @@ node_modules/ package-lock.json # Output directory after applying env substitution -out \ No newline at end of file +out + +# validator binary +otel_config_validator +validator/schema diff --git a/validator/Dockerfile b/validator/Dockerfile new file mode 100644 index 0000000..d4eecbc --- /dev/null +++ b/validator/Dockerfile @@ -0,0 +1,21 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.22 AS build + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux go build -o /otel_config_validator + +FROM gcr.io/distroless/base-debian12 + +WORKDIR / + +COPY --from=build /otel_config_validator /otel_config_validator + +USER nonroot:nonroot + +ENTRYPOINT ["/otel_config_validator"] diff --git a/validator/Dockerfile.shelltest b/validator/Dockerfile.shelltest new file mode 100644 index 0000000..4810ff3 --- /dev/null +++ b/validator/Dockerfile.shelltest @@ -0,0 +1,15 @@ +FROM ubuntu:22.04 + +RUN + +RUN DEBIAN_FRONTEND=noninteractive \ + apt-get update \ + && apt-get install -y software-properties-common \ + && apt-add-repository ppa:rmescandon/yq \ + && apt-get update \ + && apt-get install -y shelltestrunner jq yq \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root + +ENTRYPOINT ["shelltest"] diff --git a/validator/LICENSE b/validator/LICENSE new file mode 100644 index 0000000..f49a4e1 --- /dev/null +++ b/validator/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/validator/Makefile b/validator/Makefile new file mode 100644 index 0000000..6fcb084 --- /dev/null +++ b/validator/Makefile @@ -0,0 +1,18 @@ +ROOT_DIR := $(realpath $(shell dirname $(firstword $(MAKEFILE_LIST)))) +SCHEMA_DIR := ${ROOT_DIR}/../schema +CURRENT_GIT_REF := $(shell git rev-parse --short HEAD) +DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF} +DOCKER_BUILD_ARGS := -t otel_config_validator:${DOCKER_IMAGE_TAG} + +all: build + +copy-schema: + cp -R ${SCHEMA_DIR} ${ROOT_DIR}/ + +build: copy-schema + go build + +docker: copy-schema + docker build . ${DOCKER_BUILD_ARGS} + +.PHONY: all copy-schema build docker diff --git a/validator/README.md b/validator/README.md new file mode 100644 index 0000000..3bad07e --- /dev/null +++ b/validator/README.md @@ -0,0 +1,52 @@ +## OpenTelemetry SDK Configuration Validator + +This application will validate a yaml or json file against the [OpenTelemetry +SDK Configuration schema](https://github.com/open-telemetry/opentelemetry-configuration/). + +### Build + +The `schema` directory is required to be in the directory of the Go file that +embeds it so a `go build` alone will fail, instead run `make` which will copy +the schema: + +``` +$ make +``` + +Same is true for building the docker image: + +``` +$ make docker +``` + +### Usage + +The command `otel_config_validator` takes one argument, the path to the yaml or +json configuration file and optionally the path to a file to output the +configuration after environment variable expansion and validation has been done. +The format (json or yaml) of the output is based on the extension (`.json` or +`yml`/`.yaml`) of the output file name. + +``` +$ ./otel_config_validator -o out.json ../examples/kitchen-sink.yaml +``` + +Environment variable substitution is supported with the syntax `${VARIABLE}`. +Default values are supported in the form `${VARIABLE:default}`. + +### Testing + +Run the Go unit tests: + +``` +$ go test . +``` + +Running the tests of the compiled CLI requires +[shelltest](https://github.com/simonmichael/shelltestrunner), +[jq](https://github.com/jqlang/jq/) and [yq](https://github.com/mikefarah/yq): + +``` +$ shelltest -c --diff --all shelltests/*.test +``` + diff --git a/validator/go.mod b/validator/go.mod new file mode 100644 index 0000000..cb008d4 --- /dev/null +++ b/validator/go.mod @@ -0,0 +1,16 @@ +module otel_config_validator + +go 1.22.0 + +require ( + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 + github.com/stretchr/testify v1.8.4 + github.com/urfave/cli/v3 v3.0.0-alpha9 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect +) diff --git a/validator/go.sum b/validator/go.sum new file mode 100644 index 0000000..bbf394f --- /dev/null +++ b/validator/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/urfave/cli/v3 v3.0.0-alpha9 h1:P0RMy5fQm1AslQS+XCmy9UknDXctOmG/q/FZkUFnJSo= +github.com/urfave/cli/v3 v3.0.0-alpha9/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/validator/main.go b/validator/main.go new file mode 100644 index 0000000..86dd646 --- /dev/null +++ b/validator/main.go @@ -0,0 +1,241 @@ +package main + +import ( + "bytes" + "context" + "embed" + "encoding/json" + "log" + "net/url" + "os" + "path/filepath" + "strings" + + jsonschema "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/urfave/cli/v3" + yaml "gopkg.in/yaml.v3" +) + +//go:embed schema/* +var schemaFS embed.FS + +func main() { + log.SetFlags(0) + + cmd := &cli.Command{ + Name: "otel_config_validator", + Usage: "Validate a configuration file against the OpenTelemetry Configuration Schema", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "output", + Aliases: []string{"o"}, + OnlyOnce: true, + Usage: "optionally where to output the configuration (as json or yaml) after variable expansion and validation", + }, + }, + Action: runAction(), + } + + if err := cmd.Run(context.Background(), os.Args); err != nil { + log.Fatal(err) + } +} + +func runAction() func(ctx context.Context, cmd *cli.Command) error { + return func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() < 1 { + log.Fatalf("Must pass a configuration filename") + } else { + configFilePath := cmd.Args().Get(0) + + jsonConfig := validateConfiguration(configFilePath) + + if o := cmd.String("output"); o != "" { + jsonToFile(jsonConfig, o) + } + } + return nil + } +} + +func validateConfiguration(configFile string) interface{} { + schemaFiles, err := schemaFS.ReadDir("schema") + if err != nil { + log.Fatal(err) + } + + c := jsonschema.NewCompiler() + + for _, file := range schemaFiles { + schemaURL, err := url.JoinPath("https://opentelemetry.io/otelconfig/", file.Name()) + schema, err := schemaFS.ReadFile(filepath.Join("schema", file.Name())) + if err != nil { + log.Fatal(err) + } + + if err := c.AddResource(schemaURL, bytes.NewReader(schema)); err != nil { + log.Fatal(err) + } + } + + schema, err := c.Compile("https://opentelemetry.io/otelconfig/opentelemetry_configuration.json") + if err != nil { + log.Fatalf("%#v", err) + } + + v := decodeFile(configFile) + expandedConfig := replaceVariables(v) + + if err = schema.Validate(expandedConfig); err != nil { + if ve, ok := err.(*jsonschema.ValidationError); ok { + log.Fatalf("%#v", ve) + } else { + log.Fatalf("%#v", err) + } + } + + return expandedConfig +} + +func decodeFile(file string) interface{} { + data, err := os.ReadFile(file) + if err != nil { + log.Fatal(err) + } + + ext := filepath.Ext(file) + if ext == ".yaml" || ext == ".yml" { + return decodeYAML(file) + } + + var v interface{} + if err := json.Unmarshal(data, &v); err != nil { + log.Fatalf("Invalid json file %s: %#v", file, err) + } + + return v +} + +func decodeYAML(file string) interface{} { + var v interface{} + + body, err := os.ReadFile(file) + if err != nil { + log.Fatalf("Failed to read configuration file %s: %v", file, err) + } + + reader := bytes.NewReader(body) + dec := yaml.NewDecoder(reader) + + if err := dec.Decode(&v); err != nil { + log.Fatalf("Invalid yaml file %s: %v", file, err) + } + + return v +} + +func jsonToFile(j interface{}, outFile string) { + ext := filepath.Ext(outFile) + if ext == ".yaml" || ext == ".yml" { + yamlString, err := yaml.Marshal(j) + err = os.WriteFile(outFile, yamlString, 0644) + if err != nil { + log.Fatalf("Unable to write output file: %v", err) + } + + err = os.WriteFile(outFile, yamlString, 0644) + if err != nil { + log.Fatalf("Unable to write output file: %v", err) + } + } else if ext == ".json" { + jsonString, err := json.MarshalIndent(j, "", " ") + if err != nil { + log.Fatalf("Unable to convert to json: %v", err) + } + + err = os.WriteFile(outFile, jsonString, 0644) + if err != nil { + log.Fatalf("Unable to write output file: %v", err) + } + } else { + log.Fatalf("Unknown extension on output file %v", outFile) + } +} + +func replaceVariables(c interface{}) interface{} { + expandedConfig := make(map[string]any) + m, _ := c.(map[string]any) + for k := range m { + val := expandValues(m[k]) + expandedConfig[k] = val + } + + return expandedConfig +} + +func expandValues(value any) any { + switch v := value.(type) { + case string: + if !strings.Contains(v, "${") || !strings.Contains(v, "}") { + return v + } + + return expandString(v) + case []any: + l := []any{} + for _, e := range v { + newElement := expandValues(e) + l = append(l, newElement) + } + return l + case map[string]any: + newMap := make(map[string]any) + + for k, v := range v { + updated := expandValues(v) + newMap[k] = updated + } + + return newMap + } + + return value +} + +// Replace environment variables ${EXAMPLE} with their value and continue to +// try replacing variables until there are no more, meaning ${EXAMPLE} could +// contain another variable ${ANOTHER_VARIABLE}. But stop after 100 iterations +// to prevent an infinite loop. +// This does not use os.ExpandVars in order to support defaults like ${VAR:default} +func expandString(s string) string { + result := s + for i := 0; i < 100; i++ { + if !strings.Contains(result, "${") || !strings.Contains(result, "}") { + break + } + + closeIndex := strings.Index(result, "}") + openIndex := strings.LastIndex(result[:closeIndex+1], "${") + + fullEnvVar := result[openIndex : closeIndex+1] + envVar := result[openIndex+2 : closeIndex] + + maybeDefaultIndex := strings.Index(envVar, ":-") + + var newValue string + if maybeDefaultIndex != -1 { + d := envVar[maybeDefaultIndex+2:] + envVar = envVar[:maybeDefaultIndex] + newValue = os.Getenv(envVar) + if strings.EqualFold(newValue, "") { + newValue = d + } + } else { + newValue = os.Getenv(envVar) + } + + result = strings.ReplaceAll(result, fullEnvVar, newValue) + } + + return result +} diff --git a/validator/main_test.go b/validator/main_test.go new file mode 100644 index 0000000..84b25a2 --- /dev/null +++ b/validator/main_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "os" + "strings" + "testing" +) + +func TestExpandString(t *testing.T) { + e1 := "VARIABLE1" + v1 := "${VARIABLE2}" + + e2 := "VARIABLE2" + v2 := "value2" + + e3 := "VARIABLE3" + v3 := "VARIABLE1" + + os.Setenv(e1, v1) + os.Setenv(e2, v2) + os.Setenv(e3, v3) + + t.Cleanup(func() { + os.Unsetenv(e1) + }) + + s := expandString("${VARIABLE1} World") + if !strings.Contains(s, v2) { + t.Errorf("String \"%v\" does not contain value %v", s, v2) + } + + s = expandString("${VARIABLE1}${VARIABLE2}") + if !strings.Contains(s, "value2value2") { + t.Errorf("String \"%v\" does not contain value %v", s, v2) + } + + s = expandString("${${VARIABLE3}}") + if !strings.Contains(s, v2) { + t.Errorf("String \"%v\" does not contain value %v", s, v2) + } + + s = expandString("${VARIABLE2") + if !strings.Contains(s, "${VARIABLE2") { + t.Errorf("String \"%v\" should still contains ${VARIABLE2", s) + } + + s = expandString("${UNDEFINED:-mydefault}") + if !strings.EqualFold(s, "mydefault") { + t.Errorf("String \"%v\" should be mydefault", s) + } +} diff --git a/validator/shelltests/help.test b/validator/shelltests/help.test new file mode 100644 index 0000000..e1948d2 --- /dev/null +++ b/validator/shelltests/help.test @@ -0,0 +1,15 @@ +./otel_config_validator --help +>>> +NAME: + otel_config_validator - Validate a configuration file against the OpenTelemetry Configuration Schema + +USAGE: + otel_config_validator [global options] [command [command options]] [arguments...] + +COMMANDS: + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --output value, -o value optionally where to output the configuration (as json or yaml) after variable expansion and validation + --help, -h show help (default: false) +>>>= 0 diff --git a/validator/shelltests/json_out.test b/validator/shelltests/json_out.test new file mode 100644 index 0000000..44ba3e4 --- /dev/null +++ b/validator/shelltests/json_out.test @@ -0,0 +1,8 @@ +FILE_FORMAT=0.1 ./otel_config_validator -o out.json shelltests/test.yaml +>>> +>>>= 0 + +jq '.file_format' out.json +>>> +"0.1" +>>>= 0 \ No newline at end of file diff --git a/validator/shelltests/missing_arg.test b/validator/shelltests/missing_arg.test new file mode 100644 index 0000000..0ca83ba --- /dev/null +++ b/validator/shelltests/missing_arg.test @@ -0,0 +1,4 @@ +./otel_config_validator -o out.json +>>>2 +Must pass a configuration filename +>>>= 1 diff --git a/validator/shelltests/missing_ext.test b/validator/shelltests/missing_ext.test new file mode 100644 index 0000000..fa81e8d --- /dev/null +++ b/validator/shelltests/missing_ext.test @@ -0,0 +1,4 @@ +./otel_config_validator -o out shelltests/test.yaml +>>>2 +Unknown extension on output file out +>>>= 1 diff --git a/validator/shelltests/missing_required.test b/validator/shelltests/missing_required.test new file mode 100644 index 0000000..95ca337 --- /dev/null +++ b/validator/shelltests/missing_required.test @@ -0,0 +1,5 @@ +./otel_config_validator shelltests/missing_required.yaml +>>>2 +[I#] [S#] doesn't validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json# + [I#] [S#/required] missing properties: 'file_format' +>>>= 1 diff --git a/validator/shelltests/missing_required.yaml b/validator/shelltests/missing_required.yaml new file mode 100644 index 0000000..555fe92 --- /dev/null +++ b/validator/shelltests/missing_required.yaml @@ -0,0 +1 @@ +missing_required: file_format diff --git a/validator/shelltests/test.yaml b/validator/shelltests/test.yaml new file mode 100644 index 0000000..4fe6208 --- /dev/null +++ b/validator/shelltests/test.yaml @@ -0,0 +1 @@ +file_format: ${FILE_FORMAT} diff --git a/validator/shelltests/yaml_out.test b/validator/shelltests/yaml_out.test new file mode 100644 index 0000000..db5c467 --- /dev/null +++ b/validator/shelltests/yaml_out.test @@ -0,0 +1,8 @@ +FILE_FORMAT=0.1 ./otel_config_validator -o out.yaml shelltests/test.yaml +>>> +>>>= 0 + +yq eval '.file_format' out.yaml +>>> +0.1 +>>>= 0 \ No newline at end of file From bc1439303732188bdeccb2a23fb29e9b7544286c Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 20 Jul 2024 05:43:24 -0600 Subject: [PATCH 02/30] don't replace inserted vars and support different defaults --- validator/main.go | 50 +++++++++++++++------ validator/main_test.go | 29 ++++++++---- validator/shelltests/missing_ext.test | 2 +- validator/shelltests/multiple_defaults.yaml | 1 + validator/shelltests/test.yaml | 2 +- 5 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 validator/shelltests/multiple_defaults.yaml diff --git a/validator/main.go b/validator/main.go index 86dd646..0f7a911 100644 --- a/validator/main.go +++ b/validator/main.go @@ -68,6 +68,9 @@ func validateConfiguration(configFile string) interface{} { for _, file := range schemaFiles { schemaURL, err := url.JoinPath("https://opentelemetry.io/otelconfig/", file.Name()) + if err != nil { + log.Fatal(err) + } schema, err := schemaFS.ReadFile(filepath.Join("schema", file.Name())) if err != nil { log.Fatal(err) @@ -138,6 +141,9 @@ func jsonToFile(j interface{}, outFile string) { ext := filepath.Ext(outFile) if ext == ".yaml" || ext == ".yml" { yamlString, err := yaml.Marshal(j) + if err != nil { + log.Fatalf("Unable to decode yaml: %v", err) + } err = os.WriteFile(outFile, yamlString, 0644) if err != nil { log.Fatalf("Unable to write output file: %v", err) @@ -202,23 +208,37 @@ func expandValues(value any) any { return value } -// Replace environment variables ${EXAMPLE} with their value and continue to -// try replacing variables until there are no more, meaning ${EXAMPLE} could -// contain another variable ${ANOTHER_VARIABLE}. But stop after 100 iterations -// to prevent an infinite loop. -// This does not use os.ExpandVars in order to support defaults like ${VAR:default} -func expandString(s string) string { +// Replace environment variables, like ${EXAMPLE}, with their value. +// This does not use `os.ExpandVars` in order to support defaults +// for missing variables, ${VAR:-default} +func expandString(s string) string { result := s - for i := 0; i < 100; i++ { - if !strings.Contains(result, "${") || !strings.Contains(result, "}") { - break + + allVars := findAllVars(s) + for k, v := range allVars { + result = strings.ReplaceAll(result, k, v) + } + + return result +} + +func findAllVars(s string) map[string]string { + result := make(map[string]string) + lenS := len(s) + + var substr string + for i := 0; i < lenS; { + substr = s[i:lenS] + + if !strings.Contains(substr, "${") || !strings.Contains(substr, "}") { + return result } - closeIndex := strings.Index(result, "}") - openIndex := strings.LastIndex(result[:closeIndex+1], "${") + closeIndex := strings.Index(substr, "}") + openIndex := strings.LastIndex(substr[:closeIndex+1], "${") - fullEnvVar := result[openIndex : closeIndex+1] - envVar := result[openIndex+2 : closeIndex] + fullEnvVar := substr[openIndex : closeIndex+1] + envVar := substr[openIndex+2 : closeIndex] maybeDefaultIndex := strings.Index(envVar, ":-") @@ -234,7 +254,9 @@ func expandString(s string) string { newValue = os.Getenv(envVar) } - result = strings.ReplaceAll(result, fullEnvVar, newValue) + result[fullEnvVar] = newValue + + i += closeIndex + 1 } return result diff --git a/validator/main_test.go b/validator/main_test.go index 84b25a2..8377eb0 100644 --- a/validator/main_test.go +++ b/validator/main_test.go @@ -22,30 +22,43 @@ func TestExpandString(t *testing.T) { t.Cleanup(func() { os.Unsetenv(e1) + os.Unsetenv(e2) + os.Unsetenv(e3) }) + // variables only replaced once so result should be the string "${VARIABLE2} World" s := expandString("${VARIABLE1} World") - if !strings.Contains(s, v2) { - t.Errorf("String \"%v\" does not contain value %v", s, v2) + expected := "${VARIABLE2} World" + if !strings.EqualFold(s, expected) { + t.Errorf("String \"%v\" does not contain value %v", s, expected) } - s = expandString("${VARIABLE1}${VARIABLE2}") - if !strings.Contains(s, "value2value2") { - t.Errorf("String \"%v\" does not contain value %v", s, v2) + s = expandString("${VARIABLE2}") + if !strings.EqualFold(s, v2) { + t.Errorf("String \"%v\" does not equal %v", s, v2) } + // variables nested in a variable declaration s = expandString("${${VARIABLE3}}") - if !strings.Contains(s, v2) { - t.Errorf("String \"%v\" does not contain value %v", s, v2) + if !strings.Contains(s, "${VARIABLE1}") && !strings.Contains(s, "value2") { + t.Errorf("String \"%v\" does not contain value %v", s, v3) } + // variable with no ending bracket s = expandString("${VARIABLE2") - if !strings.Contains(s, "${VARIABLE2") { + if !strings.EqualFold(s, "${VARIABLE2") { t.Errorf("String \"%v\" should still contains ${VARIABLE2", s) } + // replace undefined variable with default s = expandString("${UNDEFINED:-mydefault}") if !strings.EqualFold(s, "mydefault") { t.Errorf("String \"%v\" should be mydefault", s) } + + // replace 2 undefined variables with their particular defaults + s = expandString("${UNDEFINED:-firstdefault} ${UNDEFINED:-seconddefault}") + if !strings.EqualFold(s, "firstdefault seconddefault") { + t.Errorf("String \"%s\" should be \"firstdefault seconddefault\"", s) + } } diff --git a/validator/shelltests/missing_ext.test b/validator/shelltests/missing_ext.test index fa81e8d..2bf201f 100644 --- a/validator/shelltests/missing_ext.test +++ b/validator/shelltests/missing_ext.test @@ -1,4 +1,4 @@ -./otel_config_validator -o out shelltests/test.yaml +./otel_config_validator -o out shelltests/multiple_defaults.yaml >>>2 Unknown extension on output file out >>>= 1 diff --git a/validator/shelltests/multiple_defaults.yaml b/validator/shelltests/multiple_defaults.yaml new file mode 100644 index 0000000..c849ffc --- /dev/null +++ b/validator/shelltests/multiple_defaults.yaml @@ -0,0 +1 @@ +file_format: ${FILE_FORMAT}${BLANK:-}${FILE_FORMAT:-0.1} diff --git a/validator/shelltests/test.yaml b/validator/shelltests/test.yaml index 4fe6208..ccc7074 100644 --- a/validator/shelltests/test.yaml +++ b/validator/shelltests/test.yaml @@ -1 +1 @@ -file_format: ${FILE_FORMAT} +file_format: ${FILE_FORMAT}${BLANK:-} From 53deccdd7816b425407df0a93b73c16a1a44dee5 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 20 Jul 2024 07:06:05 -0600 Subject: [PATCH 03/30] remove subdir LICENSE of validator --- validator/LICENSE | 201 ---------------------------------------------- 1 file changed, 201 deletions(-) delete mode 100644 validator/LICENSE diff --git a/validator/LICENSE b/validator/LICENSE deleted file mode 100644 index f49a4e1..0000000 --- a/validator/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file From 30b35a4a87c964abc6bff9e88004b8feef95fe59 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 20 Jul 2024 09:07:13 -0600 Subject: [PATCH 04/30] include validator makefile in top level makefile --- Makefile | 2 ++ validator/Makefile | 16 +++++++--------- validator/README.md | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 808aacd..d042e82 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ SCHEMA_FILES := $(shell find . -path './schema/*.json' -exec basename {} \; | so EXAMPLE_FILES := $(shell find . -path './examples/*.yaml' -exec basename {} \; | sort) $(shell mkdir -p out) +include validator/Makefile + .PHONY: all all: install-tools compile-schema validate-examples diff --git a/validator/Makefile b/validator/Makefile index 6fcb084..93da665 100644 --- a/validator/Makefile +++ b/validator/Makefile @@ -1,18 +1,16 @@ -ROOT_DIR := $(realpath $(shell dirname $(firstword $(MAKEFILE_LIST)))) +ROOT_DIR := $(realpath $(shell dirname $(lastword $(MAKEFILE_LIST)))) SCHEMA_DIR := ${ROOT_DIR}/../schema CURRENT_GIT_REF := $(shell git rev-parse --short HEAD) DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF} DOCKER_BUILD_ARGS := -t otel_config_validator:${DOCKER_IMAGE_TAG} -all: build - -copy-schema: +validator-copy-schema: cp -R ${SCHEMA_DIR} ${ROOT_DIR}/ -build: copy-schema - go build +validator: validator-copy-schema + go build -C ${ROOT_DIR} ${ROOT_DIR} -docker: copy-schema - docker build . ${DOCKER_BUILD_ARGS} +validator-docker: validator-copy-schema + docker build . ${ROOT_DIR}/${DOCKER_BUILD_ARGS} -.PHONY: all copy-schema build docker +.PHONY: validator-copy-schema validator validator-docker diff --git a/validator/README.md b/validator/README.md index 3bad07e..8f30596 100644 --- a/validator/README.md +++ b/validator/README.md @@ -10,13 +10,13 @@ embeds it so a `go build` alone will fail, instead run `make` which will copy the schema: ``` -$ make +$ make validator ``` Same is true for building the docker image: ``` -$ make docker +$ make validator-docker ``` ### Usage From a468c11dc43c0e0a472a308fc694e557df995d3d Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 20 Jul 2024 09:07:48 -0600 Subject: [PATCH 05/30] add github action to test validator cli --- .github/workflows/validator-tests.yaml | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/validator-tests.yaml diff --git a/.github/workflows/validator-tests.yaml b/.github/workflows/validator-tests.yaml new file mode 100644 index 0000000..25565a4 --- /dev/null +++ b/.github/workflows/validator-tests.yaml @@ -0,0 +1,36 @@ +name: Validator Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + check-schema: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '~1.22' + + - name: Build + run: make validator + + - name: Test + run: | + cd validator + go test . + + - name: Run ShellTests + run: | + cd validator + + apt update + apt install shelltestrunner + + shelltest -c --diff --all shelltests/*.test From 15fd4579315d7c567bb67e177a7474e7d641b938 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 20 Jul 2024 11:55:03 -0600 Subject: [PATCH 06/30] add validation of examples to makefile for validator cli --- validator/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/validator/Makefile b/validator/Makefile index 93da665..c1dd5de 100644 --- a/validator/Makefile +++ b/validator/Makefile @@ -3,6 +3,7 @@ SCHEMA_DIR := ${ROOT_DIR}/../schema CURRENT_GIT_REF := $(shell git rev-parse --short HEAD) DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF} DOCKER_BUILD_ARGS := -t otel_config_validator:${DOCKER_IMAGE_TAG} +validate_file = echo $(file) ; $(shell ${ROOT_DIR}/otel_config_validator -o out.json $(file)) validator-copy-schema: cp -R ${SCHEMA_DIR} ${ROOT_DIR}/ @@ -13,4 +14,7 @@ validator: validator-copy-schema validator-docker: validator-copy-schema docker build . ${ROOT_DIR}/${DOCKER_BUILD_ARGS} +validator-validate-examples: validator + $(foreach file, $(wildcard $(ROOT_DIR)/../examples/*.yaml), $(validate_file)) + .PHONY: validator-copy-schema validator validator-docker From b7018e982b77f14b23294ff9af8100946f4863e7 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 22 Jul 2024 18:16:37 -0600 Subject: [PATCH 07/30] fix typing of replaced variables in yaml configs In the case of yaml we can have an unquoted environment variable that when replaced keeps the appropriate type. Like ${IM_TRUE} becomes `true` and not `"true"` as it would be in the case of json where the string would be quoted `"${IM_TURE}"` in the first place. --- validator/go.mod | 1 + validator/go.sum | 3 + validator/main.go | 305 ++++++++++++++++----- validator/shelltests/json_replacement.json | 1 + validator/shelltests/json_replacement.test | 3 + validator/shelltests/missing_required.test | 3 +- validator/shelltests/no_expected_key.test | 2 + validator/shelltests/no_expected_key.yaml | 1 + validator/shelltests/string_for_int.test | 4 + validator/shelltests/string_for_int.yaml | 18 ++ validator/shelltests/test.yaml | 2 +- validator/shelltests/yaml_out.test | 4 +- 12 files changed, 272 insertions(+), 75 deletions(-) create mode 100644 validator/shelltests/json_replacement.json create mode 100644 validator/shelltests/json_replacement.test create mode 100644 validator/shelltests/no_expected_key.test create mode 100644 validator/shelltests/no_expected_key.yaml create mode 100644 validator/shelltests/string_for_int.test create mode 100644 validator/shelltests/string_for_int.yaml diff --git a/validator/go.mod b/validator/go.mod index cb008d4..039bbf5 100644 --- a/validator/go.mod +++ b/validator/go.mod @@ -13,4 +13,5 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/validator/go.sum b/validator/go.sum index bbf394f..bbcefb5 100644 --- a/validator/go.sum +++ b/validator/go.sum @@ -1,5 +1,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -13,3 +14,5 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/validator/main.go b/validator/main.go index 0f7a911..fac1d3d 100644 --- a/validator/main.go +++ b/validator/main.go @@ -5,15 +5,17 @@ import ( "context" "embed" "encoding/json" + jsonschema "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/urfave/cli/v3" + yaml "gopkg.in/yaml.v3" + "log" "net/url" "os" "path/filepath" + k8syaml "sigs.k8s.io/yaml" + "strconv" "strings" - - jsonschema "github.com/santhosh-tekuri/jsonschema/v5" - "github.com/urfave/cli/v3" - yaml "gopkg.in/yaml.v3" ) //go:embed schema/* @@ -48,17 +50,41 @@ func runAction() func(ctx context.Context, cmd *cli.Command) error { } else { configFilePath := cmd.Args().Get(0) - jsonConfig := validateConfiguration(configFilePath) + outFile := cmd.String("output") + ext := validateOutputOption(outFile) + + yaml := validateConfiguration(configFilePath, ext) - if o := cmd.String("output"); o != "" { - jsonToFile(jsonConfig, o) + if outFile != "" { + toFile(yaml, outFile) } + + return nil } + return nil } } -func validateConfiguration(configFile string) interface{} { +func validateOutputOption(outFile string) string { + if outFile != "" { + ext := filepath.Ext(outFile) + if !isYamlExt(ext) && !isJsonExt(ext) { + log.Fatalf("Unknown extension on output file %v", outFile) + } + + err := os.MkdirAll(filepath.Dir(outFile), 0700) + if err != nil { + log.Fatalf("Unable to create dir for output file %s: %+v", outFile, err) + } + + return ext + } + + return "" +} + +func validateConfiguration(configFile string, outfileExt string) []byte { schemaFiles, err := schemaFS.ReadDir("schema") if err != nil { log.Fatal(err) @@ -86,100 +112,191 @@ func validateConfiguration(configFile string) interface{} { log.Fatalf("%#v", err) } - v := decodeFile(configFile) - expandedConfig := replaceVariables(v) + expandedConfig, outFile := decodeFile(configFile, outfileExt) if err = schema.Validate(expandedConfig); err != nil { if ve, ok := err.(*jsonschema.ValidationError); ok { - log.Fatalf("%#v", ve) + log.Fatalf("%+v", ve) } else { - log.Fatalf("%#v", err) + log.Fatalf("%+v", err) } } - return expandedConfig + return outFile } -func decodeFile(file string) interface{} { - data, err := os.ReadFile(file) +func decodeFile(configFile string, outfileExt string) (interface{}, []byte) { + data, err := os.ReadFile(configFile) if err != nil { log.Fatal(err) } - ext := filepath.Ext(file) - if ext == ".yaml" || ext == ".yml" { - return decodeYAML(file) + var j []byte + var d []byte + + ext := filepath.Ext(configFile) + if isYamlExt(ext) { + y := decodeYaml(configFile) + replaceYamlVariables(&y) + + var u interface{} + err = y.Decode(&u) + if err != nil { + log.Fatal(err) + } + + var jsonInterface interface{} + if len(y.Content) > 0 { + d, err = yaml.Marshal(&y.Content[0]) + if err != nil { + log.Fatal(err) + } + + j, err = k8syaml.YAMLToJSON(d) + if err != nil { + log.Fatalf("Error encoding yaml to json for validation: %+v", err) + } + + if err := json.Unmarshal(j, &jsonInterface); err != nil { + log.Fatalf("Invalid json file from yaml file %s: %+v", configFile, err) + } + } + + toWrite := convertYamlNode(y, outfileExt) + + return jsonInterface, toWrite } var v interface{} if err := json.Unmarshal(data, &v); err != nil { - log.Fatalf("Invalid json file %s: %#v", file, err) + log.Fatalf("Invalid json file %s: %#v", configFile, err) } - return v + expandedConfig, b := replaceJsonVariables(v) + + toWrite := convertJsonBytes(b, outfileExt) + + return expandedConfig, toWrite } -func decodeYAML(file string) interface{} { - var v interface{} +func isYamlExt(ext string) bool { + return ext == ".yaml" || ext == ".yml" +} + +func isJsonExt(ext string) bool { + return ext == ".json" +} + +func convertYamlNode(n yaml.Node, ext string) []byte { + d, err := yaml.Marshal(&n.Content[0]) + if err != nil { + log.Fatalf("Error marshalling yaml: %+v", err) + } + + if isJsonExt(ext) { + j, err := k8syaml.YAMLToJSON(d) + if err != nil { + log.Fatalf("Error converting yaml to json: %+v", err) + } + + return j + } + + return d +} + +func convertJsonBytes(jsonBytes []byte, ext string) []byte { + if isYamlExt(ext) { + b, _ := k8syaml.JSONToYAML(jsonBytes) + return b + } + return jsonBytes +} + +func decodeYaml(file string) yaml.Node { body, err := os.ReadFile(file) if err != nil { log.Fatalf("Failed to read configuration file %s: %v", file, err) } - reader := bytes.NewReader(body) - dec := yaml.NewDecoder(reader) - - if err := dec.Decode(&v); err != nil { - log.Fatalf("Invalid yaml file %s: %v", file, err) + var node yaml.Node + if err := yaml.Unmarshal([]byte(body), &node); err != nil { + log.Fatalf("Unable to parse config file %s: %+v", file, err) } - return v + return node } -func jsonToFile(j interface{}, outFile string) { - ext := filepath.Ext(outFile) - if ext == ".yaml" || ext == ".yml" { - yamlString, err := yaml.Marshal(j) - if err != nil { - log.Fatalf("Unable to decode yaml: %v", err) - } - err = os.WriteFile(outFile, yamlString, 0644) - if err != nil { - log.Fatalf("Unable to write output file: %v", err) - } +func toFile(jsonBytes []byte, outFile string) { + err := os.WriteFile(outFile, jsonBytes, 0644) + if err != nil { + log.Fatalf("Unable to write output file: %v", err) + } +} - err = os.WriteFile(outFile, yamlString, 0644) - if err != nil { - log.Fatalf("Unable to write output file: %v", err) - } - } else if ext == ".json" { - jsonString, err := json.MarshalIndent(j, "", " ") - if err != nil { - log.Fatalf("Unable to convert to json: %v", err) +func replaceYamlVariables(y *yaml.Node) interface{} { + if y.Kind == yaml.DocumentNode { + l := len(y.Content) + for i := 0; i < l; i++ { + n := y.Content[i] + handleNode(n) } + } - err = os.WriteFile(outFile, jsonString, 0644) - if err != nil { - log.Fatalf("Unable to write output file: %v", err) - } - } else { - log.Fatalf("Unknown extension on output file %v", outFile) + return y +} + +func handleNode(n *yaml.Node) { + if n.Kind == yaml.MappingNode { + handleMappingNode(n) + } else if n.Kind == yaml.SequenceNode { + handleSequenceNode(n) + } else if n.Kind == yaml.ScalarNode { + handleScalarNode(n) + } +} + +func handleMappingNode(c *yaml.Node) { + lenMap := len(c.Content) + + // map is a flat list. every other element is a value + for j := 1; j < lenMap; j += 2 { + s := c.Content[j] + handleNode(s) } } -func replaceVariables(c interface{}) interface{} { +func handleSequenceNode(c *yaml.Node) { + lenMap := len(c.Content) + for j := 0; j < lenMap; j++ { + s := c.Content[j] + handleNode(s) + } +} + +func replaceJsonVariables(c interface{}) (interface{}, []byte) { expandedConfig := make(map[string]any) + m, _ := c.(map[string]any) for k := range m { - val := expandValues(m[k]) + val := expandJsonValues(m[k]) expandedConfig[k] = val } + b, err := json.Marshal(expandedConfig) + if err != nil { + log.Fatalf("json.Marshal: %+v", err) + } - return expandedConfig + y, err := k8syaml.JSONToYAML(b) + if err != nil { + log.Fatalf("Error converting json result to yaml: %+v", err) + } + + return expandedConfig, y } -func expandValues(value any) any { +func expandJsonValues(value interface{}) any { switch v := value.(type) { case string: if !strings.Contains(v, "${") || !strings.Contains(v, "}") { @@ -190,7 +307,7 @@ func expandValues(value any) any { case []any: l := []any{} for _, e := range v { - newElement := expandValues(e) + newElement := expandJsonValues(e) l = append(l, newElement) } return l @@ -198,7 +315,7 @@ func expandValues(value any) any { newMap := make(map[string]any) for k, v := range v { - updated := expandValues(v) + updated := expandJsonValues(v) newMap[k] = updated } @@ -208,22 +325,66 @@ func expandValues(value any) any { return value } +func handleScalarNode(n *yaml.Node) { + // only a string type can have variable replacement + // even if it can turn into another type + if n.Tag == "!!str" { + v := n.Value + if !strings.Contains(v, "${") || !strings.Contains(v, "}") { + return + } + + newValue := expandString(v) + n.Value = newValue + + // Otel Configuration File Spec defines supported values + // for converting environment variables to types as being + // booleans, integers and floats. Here we check if the + // value cleanly parses to one of those types. If it does + // and the value isn't quoted style then we set the Style to + // TaggedStyle and set the tag to the appropriate type. But + // only if the style is already TaggedStyle or no style. + // Anything else means it is explicitly a string. + if n.Style == 0 || n.Style == yaml.TaggedStyle { + if newValue == "" { + // be sure tag and style are empty for a null value so we get + // quoted empty string + n.Style = 0 + n.Tag = "" + } else if _, err := strconv.ParseBool(newValue); err == nil { + n.Style = yaml.TaggedStyle + n.Tag = "!!bool" + } else if _, err := strconv.Atoi(newValue); err == nil { + n.Style = yaml.TaggedStyle + n.Tag = "!!int" + } else if _, err := strconv.ParseFloat(newValue, 64); err == nil { + n.Style = yaml.TaggedStyle + n.Tag = "!!float" + } + } + } + +} + // Replace environment variables, like ${EXAMPLE}, with their value. // This does not use `os.ExpandVars` in order to support defaults // for missing variables, ${VAR:-default} -func expandString(s string) string { - result := s - +func expandString(s string) string { allVars := findAllVars(s) for k, v := range allVars { - result = strings.ReplaceAll(result, k, v) + if v == nil { + s = strings.ReplaceAll(s, k, "") + } else { + return v.(string) + } + } - return result + return s } -func findAllVars(s string) map[string]string { - result := make(map[string]string) +func findAllVars(s string) map[string]interface{} { + result := make(map[string]interface{}) lenS := len(s) var substr string @@ -242,16 +403,20 @@ func findAllVars(s string) map[string]string { maybeDefaultIndex := strings.Index(envVar, ":-") - var newValue string + var newValue interface{} + var isSet bool if maybeDefaultIndex != -1 { d := envVar[maybeDefaultIndex+2:] envVar = envVar[:maybeDefaultIndex] - newValue = os.Getenv(envVar) - if strings.EqualFold(newValue, "") { + newValue, isSet = os.LookupEnv(envVar) + if !isSet { newValue = d } } else { - newValue = os.Getenv(envVar) + newValue, isSet = os.LookupEnv(envVar) + if !isSet { + newValue = nil + } } result[fullEnvVar] = newValue diff --git a/validator/shelltests/json_replacement.json b/validator/shelltests/json_replacement.json new file mode 100644 index 0000000..1f34b6c --- /dev/null +++ b/validator/shelltests/json_replacement.json @@ -0,0 +1 @@ +{"attribute_limits":{"attribute_count_limit":128,"attribute_value_length_limit":512},"disabled":false,"file_format":"0.1","resource":{"attributes":{"service.name":"${OTEL_SERVICE_NAME:-unknown_service}"}}} diff --git a/validator/shelltests/json_replacement.test b/validator/shelltests/json_replacement.test new file mode 100644 index 0000000..fdb4180 --- /dev/null +++ b/validator/shelltests/json_replacement.test @@ -0,0 +1,3 @@ +./otel_config_validator -o out.json shelltests/json_replacement.json +>>> +>>>= 0 diff --git a/validator/shelltests/missing_required.test b/validator/shelltests/missing_required.test index 95ca337..a41d573 100644 --- a/validator/shelltests/missing_required.test +++ b/validator/shelltests/missing_required.test @@ -1,5 +1,4 @@ ./otel_config_validator shelltests/missing_required.yaml >>>2 -[I#] [S#] doesn't validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json# - [I#] [S#/required] missing properties: 'file_format' +jsonschema: '' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#/required: missing properties: 'file_format' >>>= 1 diff --git a/validator/shelltests/no_expected_key.test b/validator/shelltests/no_expected_key.test new file mode 100644 index 0000000..f8fcec5 --- /dev/null +++ b/validator/shelltests/no_expected_key.test @@ -0,0 +1,2 @@ +FILE_FORMAT=0.1 ./otel_config_validator shelltests/no_expected_key.yaml +>>>= 1 diff --git a/validator/shelltests/no_expected_key.yaml b/validator/shelltests/no_expected_key.yaml new file mode 100644 index 0000000..f5965cf --- /dev/null +++ b/validator/shelltests/no_expected_key.yaml @@ -0,0 +1 @@ +file_format: "${FILE_FORMAT}"${BLANK:-} diff --git a/validator/shelltests/string_for_int.test b/validator/shelltests/string_for_int.test new file mode 100644 index 0000000..a03c193 --- /dev/null +++ b/validator/shelltests/string_for_int.test @@ -0,0 +1,4 @@ +./otel_config_validator -o out.json shelltests/string_for_int.yaml +>>>2 +jsonschema: '/attribute_limits/attribute_value_length_limit' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#/properties/attribute_limits/$ref/properties/attribute_value_length_limit/type: expected integer or null, but got string +>>>= 1 diff --git a/validator/shelltests/string_for_int.yaml b/validator/shelltests/string_for_int.yaml new file mode 100644 index 0000000..8aec2c1 --- /dev/null +++ b/validator/shelltests/string_for_int.yaml @@ -0,0 +1,18 @@ +# The file format version +file_format: "0.1" + +# Configure if the SDK is disabled or not. +disabled: ${OTEL_SDK_DISABLED:-false} + +# Configure resource for all signals. +resource: + # Configure resource attributes. + attributes: + # Configure `service.name` resource attribute + service.name: "${OTEL_SERVICE_NAME:-unknown_service}" + +attribute_limits: + # Configure max attribute value size. + attribute_value_length_limit: ${OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT:-"hello"} + # Configure max attribute count. + attribute_count_limit: ${OTEL_ATTRIBUTE_COUNT_LIMIT:-128} diff --git a/validator/shelltests/test.yaml b/validator/shelltests/test.yaml index ccc7074..76d4aa4 100644 --- a/validator/shelltests/test.yaml +++ b/validator/shelltests/test.yaml @@ -1 +1 @@ -file_format: ${FILE_FORMAT}${BLANK:-} +file_format: "${FILE_FORMAT}" diff --git a/validator/shelltests/yaml_out.test b/validator/shelltests/yaml_out.test index db5c467..d729269 100644 --- a/validator/shelltests/yaml_out.test +++ b/validator/shelltests/yaml_out.test @@ -2,7 +2,7 @@ FILE_FORMAT=0.1 ./otel_config_validator -o out.yaml shelltests/test.yaml >>> >>>= 0 -yq eval '.file_format' out.yaml +cat out.yaml >>> -0.1 +file_format: "0.1" >>>= 0 \ No newline at end of file From 61503b6d48575d3e4c8c57824ae38cae362a8d8e Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 04:25:40 -0600 Subject: [PATCH 08/30] add test of examples with otel_config_validator cli --- validator/Makefile | 11 +++++++---- validator/shelltests/examples.test | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 validator/shelltests/examples.test diff --git a/validator/Makefile b/validator/Makefile index c1dd5de..49584e8 100644 --- a/validator/Makefile +++ b/validator/Makefile @@ -3,7 +3,8 @@ SCHEMA_DIR := ${ROOT_DIR}/../schema CURRENT_GIT_REF := $(shell git rev-parse --short HEAD) DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF} DOCKER_BUILD_ARGS := -t otel_config_validator:${DOCKER_IMAGE_TAG} -validate_file = echo $(file) ; $(shell ${ROOT_DIR}/otel_config_validator -o out.json $(file)) +EXAMPLE_FILES := $(shell find ${ROOT_DIR}/../examples -name "*.yaml" -exec basename {} \;) +$(shell mkdir -p out) validator-copy-schema: cp -R ${SCHEMA_DIR} ${ROOT_DIR}/ @@ -14,7 +15,9 @@ validator: validator-copy-schema validator-docker: validator-copy-schema docker build . ${ROOT_DIR}/${DOCKER_BUILD_ARGS} -validator-validate-examples: validator - $(foreach file, $(wildcard $(ROOT_DIR)/../examples/*.yaml), $(validate_file)) +validator-validate-examples: + @for f in $(EXAMPLE_FILES); do \ + ./otel_config_validator -o ${ROOT_DIR}/out/$$f ${ROOT_DIR}/../examples/$$f ; \ + done -.PHONY: validator-copy-schema validator validator-docker +.PHONY: validator-validate-examples validator-copy-schema validator validator-docker diff --git a/validator/shelltests/examples.test b/validator/shelltests/examples.test new file mode 100644 index 0000000..63c2bab --- /dev/null +++ b/validator/shelltests/examples.test @@ -0,0 +1,2 @@ +make validator-validate-examples +>>>= 0 \ No newline at end of file From 0c38c527138d94fa34375c12ae31436e8b8c1b19 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 04:45:00 -0600 Subject: [PATCH 09/30] shelltests output to out dir --- validator/shelltests/json_out.test | 4 ++-- validator/shelltests/json_replacement.test | 2 +- validator/shelltests/missing_arg.test | 2 +- validator/shelltests/string_for_int.test | 2 +- validator/shelltests/yaml_out.test | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/validator/shelltests/json_out.test b/validator/shelltests/json_out.test index 44ba3e4..975977e 100644 --- a/validator/shelltests/json_out.test +++ b/validator/shelltests/json_out.test @@ -1,8 +1,8 @@ -FILE_FORMAT=0.1 ./otel_config_validator -o out.json shelltests/test.yaml +FILE_FORMAT=0.1 ./otel_config_validator -o out/out.json shelltests/test.yaml >>> >>>= 0 -jq '.file_format' out.json +jq '.file_format' out/out.json >>> "0.1" >>>= 0 \ No newline at end of file diff --git a/validator/shelltests/json_replacement.test b/validator/shelltests/json_replacement.test index fdb4180..1756d16 100644 --- a/validator/shelltests/json_replacement.test +++ b/validator/shelltests/json_replacement.test @@ -1,3 +1,3 @@ -./otel_config_validator -o out.json shelltests/json_replacement.json +./otel_config_validator -o out/out.json shelltests/json_replacement.json >>> >>>= 0 diff --git a/validator/shelltests/missing_arg.test b/validator/shelltests/missing_arg.test index 0ca83ba..f387fec 100644 --- a/validator/shelltests/missing_arg.test +++ b/validator/shelltests/missing_arg.test @@ -1,4 +1,4 @@ -./otel_config_validator -o out.json +./otel_config_validator -o out/out.json >>>2 Must pass a configuration filename >>>= 1 diff --git a/validator/shelltests/string_for_int.test b/validator/shelltests/string_for_int.test index a03c193..c34b689 100644 --- a/validator/shelltests/string_for_int.test +++ b/validator/shelltests/string_for_int.test @@ -1,4 +1,4 @@ -./otel_config_validator -o out.json shelltests/string_for_int.yaml +./otel_config_validator -o out/out.json shelltests/string_for_int.yaml >>>2 jsonschema: '/attribute_limits/attribute_value_length_limit' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#/properties/attribute_limits/$ref/properties/attribute_value_length_limit/type: expected integer or null, but got string >>>= 1 diff --git a/validator/shelltests/yaml_out.test b/validator/shelltests/yaml_out.test index d729269..a89c4a2 100644 --- a/validator/shelltests/yaml_out.test +++ b/validator/shelltests/yaml_out.test @@ -1,8 +1,8 @@ -FILE_FORMAT=0.1 ./otel_config_validator -o out.yaml shelltests/test.yaml +FILE_FORMAT=0.1 ./otel_config_validator -o out/out.yaml shelltests/test.yaml >>> >>>= 0 -cat out.yaml +cat out/out.yaml >>> file_format: "0.1" >>>= 0 \ No newline at end of file From 1ba621412613fb787db72cb03d90839a25f0d3a0 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 05:13:10 -0600 Subject: [PATCH 10/30] fix replacement of multiple variables in a single value --- validator/main.go | 93 +++++++++++++++++++++++------------------- validator/main_test.go | 4 +- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/validator/main.go b/validator/main.go index fac1d3d..e8d76cf 100644 --- a/validator/main.go +++ b/validator/main.go @@ -67,7 +67,7 @@ func runAction() func(ctx context.Context, cmd *cli.Command) error { } func validateOutputOption(outFile string) string { - if outFile != "" { + if outFile != "" { ext := filepath.Ext(outFile) if !isYamlExt(ext) && !isJsonExt(ext) { log.Fatalf("Unknown extension on output file %v", outFile) @@ -257,6 +257,7 @@ func handleNode(n *yaml.Node) { } } +// loop into all elements of a yaml map func handleMappingNode(c *yaml.Node) { lenMap := len(c.Content) @@ -267,6 +268,7 @@ func handleMappingNode(c *yaml.Node) { } } +// loop into all values of a yaml sequence func handleSequenceNode(c *yaml.Node) { lenMap := len(c.Content) for j := 0; j < lenMap; j++ { @@ -275,6 +277,50 @@ func handleSequenceNode(c *yaml.Node) { } } +func handleScalarNode(n *yaml.Node) { + // only a string type can have variable replacement + // even if it can turn into another type + if n.Tag == "!!str" { + v := n.Value + if !strings.Contains(v, "${") || !strings.Contains(v, "}") { + return + } + + newValue := expandString(v) + n.Value = newValue + + // Otel Configuration File Spec defines supported values + // for converting environment variables to types as being + // booleans, integers and floats. Here we check if the + // value cleanly parses to one of those types. If it does + // and the value isn't quoted style then we set the Style to + // TaggedStyle and set the tag to the appropriate type. But + // only if the style is already TaggedStyle or no style. + // Anything else means it is explicitly a string. + if n.Style == 0 || n.Style == yaml.TaggedStyle { + if newValue == "" { + // be sure tag and style are empty for a null value so we get + // quoted empty string + n.Style = 0 + n.Tag = "" + } else if _, err := strconv.ParseBool(newValue); err == nil { + n.Style = yaml.TaggedStyle + n.Tag = "!!bool" + } else if _, err := strconv.Atoi(newValue); err == nil { + n.Style = yaml.TaggedStyle + n.Tag = "!!int" + } else if _, err := strconv.ParseFloat(newValue, 64); err == nil { + n.Style = yaml.TaggedStyle + n.Tag = "!!float" + } + } + } + +} + +// json variable replacement is basic as the json value that +// is an environment variable will always be quoted, so only +// strings are supported func replaceJsonVariables(c interface{}) (interface{}, []byte) { expandedConfig := make(map[string]any) @@ -325,47 +371,6 @@ func expandJsonValues(value interface{}) any { return value } -func handleScalarNode(n *yaml.Node) { - // only a string type can have variable replacement - // even if it can turn into another type - if n.Tag == "!!str" { - v := n.Value - if !strings.Contains(v, "${") || !strings.Contains(v, "}") { - return - } - - newValue := expandString(v) - n.Value = newValue - - // Otel Configuration File Spec defines supported values - // for converting environment variables to types as being - // booleans, integers and floats. Here we check if the - // value cleanly parses to one of those types. If it does - // and the value isn't quoted style then we set the Style to - // TaggedStyle and set the tag to the appropriate type. But - // only if the style is already TaggedStyle or no style. - // Anything else means it is explicitly a string. - if n.Style == 0 || n.Style == yaml.TaggedStyle { - if newValue == "" { - // be sure tag and style are empty for a null value so we get - // quoted empty string - n.Style = 0 - n.Tag = "" - } else if _, err := strconv.ParseBool(newValue); err == nil { - n.Style = yaml.TaggedStyle - n.Tag = "!!bool" - } else if _, err := strconv.Atoi(newValue); err == nil { - n.Style = yaml.TaggedStyle - n.Tag = "!!int" - } else if _, err := strconv.ParseFloat(newValue, 64); err == nil { - n.Style = yaml.TaggedStyle - n.Tag = "!!float" - } - } - } - -} - // Replace environment variables, like ${EXAMPLE}, with their value. // This does not use `os.ExpandVars` in order to support defaults // for missing variables, ${VAR:-default} @@ -375,7 +380,7 @@ func expandString(s string) string { if v == nil { s = strings.ReplaceAll(s, k, "") } else { - return v.(string) + s = strings.ReplaceAll(s, k, v.(string)) } } @@ -383,6 +388,8 @@ func expandString(s string) string { return s } +// iterates over a string to find all environment variables +// returns a map of variables to strings or nil func findAllVars(s string) map[string]interface{} { result := make(map[string]interface{}) lenS := len(s) diff --git a/validator/main_test.go b/validator/main_test.go index 8377eb0..d7f7ee6 100644 --- a/validator/main_test.go +++ b/validator/main_test.go @@ -30,7 +30,7 @@ func TestExpandString(t *testing.T) { s := expandString("${VARIABLE1} World") expected := "${VARIABLE2} World" if !strings.EqualFold(s, expected) { - t.Errorf("String \"%v\" does not contain value %v", s, expected) + t.Errorf("String \"%v\" is not equal to \"%v\"", s, expected) } s = expandString("${VARIABLE2}") @@ -41,7 +41,7 @@ func TestExpandString(t *testing.T) { // variables nested in a variable declaration s = expandString("${${VARIABLE3}}") if !strings.Contains(s, "${VARIABLE1}") && !strings.Contains(s, "value2") { - t.Errorf("String \"%v\" does not contain value %v", s, v3) + t.Errorf("String \"%v\" is not equal to \"${%v}\"", s, v3) } // variable with no ending bracket From fd51c259e863913c47b52371890ff5734fa625bd Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 05:13:46 -0600 Subject: [PATCH 11/30] fix up validator makefile and add docker build to github action --- .github/workflows/validator-tests.yaml | 5 +++++ validator/Makefile | 11 ++++++----- validator/README.md | 18 ++++++++++++++---- validator/shelltests/examples.test | 5 +++++ 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/validator-tests.yaml b/.github/workflows/validator-tests.yaml index 25565a4..d54c8ce 100644 --- a/.github/workflows/validator-tests.yaml +++ b/.github/workflows/validator-tests.yaml @@ -18,6 +18,7 @@ jobs: with: go-version: '~1.22' + # TODO: Publish releases - name: Build run: make validator @@ -34,3 +35,7 @@ jobs: apt install shelltestrunner shelltest -c --diff --all shelltests/*.test + + # TODO: Push this to registry on release + - name: Build Docker Image + run: make validator-docker-image diff --git a/validator/Makefile b/validator/Makefile index 49584e8..1911c7c 100644 --- a/validator/Makefile +++ b/validator/Makefile @@ -2,7 +2,7 @@ ROOT_DIR := $(realpath $(shell dirname $(lastword $(MAKEFILE_LIST)))) SCHEMA_DIR := ${ROOT_DIR}/../schema CURRENT_GIT_REF := $(shell git rev-parse --short HEAD) DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF} -DOCKER_BUILD_ARGS := -t otel_config_validator:${DOCKER_IMAGE_TAG} +DOCKER_BUILD_ARGS := -f ${ROOT_DIR}/Dockerfile -t otel_config_validator:${DOCKER_IMAGE_TAG} EXAMPLE_FILES := $(shell find ${ROOT_DIR}/../examples -name "*.yaml" -exec basename {} \;) $(shell mkdir -p out) @@ -12,12 +12,13 @@ validator-copy-schema: validator: validator-copy-schema go build -C ${ROOT_DIR} ${ROOT_DIR} -validator-docker: validator-copy-schema - docker build . ${ROOT_DIR}/${DOCKER_BUILD_ARGS} +validator-docker-image: validator-copy-schema + docker build ${DOCKER_BUILD_ARGS} ${ROOT_DIR} validator-validate-examples: @for f in $(EXAMPLE_FILES); do \ - ./otel_config_validator -o ${ROOT_DIR}/out/$$f ${ROOT_DIR}/../examples/$$f ; \ + echo "Validating" $$f ; \ + ${ROOT_DIR}/otel_config_validator -o ${ROOT_DIR}/out/$$f ${ROOT_DIR}/../examples/$$f ; \ done -.PHONY: validator-validate-examples validator-copy-schema validator validator-docker +.PHONY: validator-validate-examples validator-copy-schema validator validator-docker-image diff --git a/validator/README.md b/validator/README.md index 8f30596..c2039d7 100644 --- a/validator/README.md +++ b/validator/README.md @@ -1,12 +1,15 @@ ## OpenTelemetry SDK Configuration Validator -This application will validate a yaml or json file against the [OpenTelemetry -SDK Configuration schema](https://github.com/open-telemetry/opentelemetry-configuration/). +This application will replace environment variables in values of valid yaml or +json files, following the rules of [file configuration environment variable +substitution](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/file-configuration.md#environment-variable-substitution), +before validating that result against the [OpenTelemetry SDK Configuration +schema](https://github.com/open-telemetry/opentelemetry-configuration/). ### Build The `schema` directory is required to be in the directory of the Go file that -embeds it so a `go build` alone will fail, instead run `make` which will copy +embeds it, so a `go build` alone will fail, instead run `make` which will copy the schema: ``` @@ -32,7 +35,14 @@ $ ./otel_config_validator -o out.json ../examples/kitchen-sink.yaml ``` Environment variable substitution is supported with the syntax `${VARIABLE}`. -Default values are supported in the form `${VARIABLE:default}`. +Default values are supported in the form `${VARIABLE:-default}`. + +In the case of json input only strings can be the result of substitution. To +ensure only values are replaced the input must be parsed as valid json or yaml +and in the case of json a value like `${VARIABLE}` will always have to be double +quoted as `"${VARIABLE}"` so will always remain double quoted. If you need to +substitute in a boolean, integer or float please use yaml for the input +configuration file. ### Testing diff --git a/validator/shelltests/examples.test b/validator/shelltests/examples.test index 63c2bab..dbcb652 100644 --- a/validator/shelltests/examples.test +++ b/validator/shelltests/examples.test @@ -1,2 +1,7 @@ make validator-validate-examples +>>> +Validating anchors.yaml +Validating kitchen-sink.yaml +Validating sdk-config.yaml +Validating sdk-migration-config.yaml >>>= 0 \ No newline at end of file From a42e1cab15f24485f9ff575a39562dfebe7b6bb6 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 05:29:05 -0600 Subject: [PATCH 12/30] support ${env:VAR} syntax for env var substitution --- validator/main.go | 45 ++++++++++++++++----------- validator/main_test.go | 11 +++++++ validator/shelltests/env_prefix.test | 3 ++ validator/shelltests/env_prefix.yaml | 1 + validator/shelltests/missing_arg.test | 2 +- 5 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 validator/shelltests/env_prefix.test create mode 100644 validator/shelltests/env_prefix.yaml diff --git a/validator/main.go b/validator/main.go index e8d76cf..92f0f1c 100644 --- a/validator/main.go +++ b/validator/main.go @@ -46,7 +46,7 @@ func main() { func runAction() func(ctx context.Context, cmd *cli.Command) error { return func(ctx context.Context, cmd *cli.Command) error { if cmd.Args().Len() < 1 { - log.Fatalf("Must pass a configuration filename") + log.Fatalf("Error: Must pass a configuration filename") } else { configFilePath := cmd.Args().Get(0) @@ -126,26 +126,27 @@ func validateConfiguration(configFile string, outfileExt string) []byte { } func decodeFile(configFile string, outfileExt string) (interface{}, []byte) { + var j []byte + var d []byte + var u interface{} + var v interface{} + var jsonInterface interface{} + data, err := os.ReadFile(configFile) if err != nil { log.Fatal(err) } - var j []byte - var d []byte - ext := filepath.Ext(configFile) if isYamlExt(ext) { y := decodeYaml(configFile) replaceYamlVariables(&y) - - var u interface{} + err = y.Decode(&u) if err != nil { log.Fatal(err) } - - var jsonInterface interface{} + if len(y.Content) > 0 { d, err = yaml.Marshal(&y.Content[0]) if err != nil { @@ -166,8 +167,7 @@ func decodeFile(configFile string, outfileExt string) (interface{}, []byte) { return jsonInterface, toWrite } - - var v interface{} + if err := json.Unmarshal(data, &v); err != nil { log.Fatalf("Invalid json file %s: %#v", configFile, err) } @@ -215,12 +215,13 @@ func convertJsonBytes(jsonBytes []byte, ext string) []byte { } func decodeYaml(file string) yaml.Node { + var node yaml.Node + body, err := os.ReadFile(file) if err != nil { log.Fatalf("Failed to read configuration file %s: %v", file, err) } - var node yaml.Node if err := yaml.Unmarshal([]byte(body), &node); err != nil { log.Fatalf("Unable to parse config file %s: %+v", file, err) } @@ -391,27 +392,35 @@ func expandString(s string) string { // iterates over a string to find all environment variables // returns a map of variables to strings or nil func findAllVars(s string) map[string]interface{} { + var envVar string + var newValue interface{} + var isSet bool + var substr string + result := make(map[string]interface{}) lenS := len(s) - var substr string for i := 0; i < lenS; { substr = s[i:lenS] if !strings.Contains(substr, "${") || !strings.Contains(substr, "}") { + // no more environment variables in string return result } closeIndex := strings.Index(substr, "}") - openIndex := strings.LastIndex(substr[:closeIndex+1], "${") - - fullEnvVar := substr[openIndex : closeIndex+1] - envVar := substr[openIndex+2 : closeIndex] + openIndex := strings.LastIndex(substr[:closeIndex+1], "${env:") + if openIndex == -1 { + openIndex = strings.LastIndex(substr[:closeIndex+1], "${") + envVar = substr[openIndex+2 : closeIndex] + } else { + envVar = substr[openIndex+6 : closeIndex] + } + + fullEnvVar := substr[openIndex : closeIndex+1] maybeDefaultIndex := strings.Index(envVar, ":-") - var newValue interface{} - var isSet bool if maybeDefaultIndex != -1 { d := envVar[maybeDefaultIndex+2:] envVar = envVar[:maybeDefaultIndex] diff --git a/validator/main_test.go b/validator/main_test.go index d7f7ee6..f2874bd 100644 --- a/validator/main_test.go +++ b/validator/main_test.go @@ -38,6 +38,11 @@ func TestExpandString(t *testing.T) { t.Errorf("String \"%v\" does not equal %v", s, v2) } + s = expandString("${env:VARIABLE2}") + if !strings.EqualFold(s, v2) { + t.Errorf("String \"%v\" does not equal %v", s, v2) + } + // variables nested in a variable declaration s = expandString("${${VARIABLE3}}") if !strings.Contains(s, "${VARIABLE1}") && !strings.Contains(s, "value2") { @@ -56,6 +61,12 @@ func TestExpandString(t *testing.T) { t.Errorf("String \"%v\" should be mydefault", s) } + // replace undefined variable with default + s = expandString("${env:UNDEFINED:-mydefault}") + if !strings.EqualFold(s, "mydefault") { + t.Errorf("String \"%v\" should be mydefault", s) + } + // replace 2 undefined variables with their particular defaults s = expandString("${UNDEFINED:-firstdefault} ${UNDEFINED:-seconddefault}") if !strings.EqualFold(s, "firstdefault seconddefault") { diff --git a/validator/shelltests/env_prefix.test b/validator/shelltests/env_prefix.test new file mode 100644 index 0000000..be1659a --- /dev/null +++ b/validator/shelltests/env_prefix.test @@ -0,0 +1,3 @@ +./otel_config_validator -o out/out.json shelltests/env_prefix.yaml +>>> +>>>= 0 diff --git a/validator/shelltests/env_prefix.yaml b/validator/shelltests/env_prefix.yaml new file mode 100644 index 0000000..1de5286 --- /dev/null +++ b/validator/shelltests/env_prefix.yaml @@ -0,0 +1 @@ +file_format: "${env:FILE_FORMAT}" diff --git a/validator/shelltests/missing_arg.test b/validator/shelltests/missing_arg.test index f387fec..1af62af 100644 --- a/validator/shelltests/missing_arg.test +++ b/validator/shelltests/missing_arg.test @@ -1,4 +1,4 @@ ./otel_config_validator -o out/out.json >>>2 -Must pass a configuration filename +Error: Must pass a configuration filename >>>= 1 From 5f6836545a067ba0850f9e6c61fbfb0a5fd642b5 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 05:46:48 -0600 Subject: [PATCH 13/30] support integer written as hex with 0x prefix --- validator/main.go | 2 +- validator/shelltests/hex_integer.test | 8 ++++++++ validator/shelltests/hex_integer.yaml | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 validator/shelltests/hex_integer.test create mode 100644 validator/shelltests/hex_integer.yaml diff --git a/validator/main.go b/validator/main.go index 92f0f1c..fcb83e1 100644 --- a/validator/main.go +++ b/validator/main.go @@ -307,7 +307,7 @@ func handleScalarNode(n *yaml.Node) { } else if _, err := strconv.ParseBool(newValue); err == nil { n.Style = yaml.TaggedStyle n.Tag = "!!bool" - } else if _, err := strconv.Atoi(newValue); err == nil { + } else if _, err := strconv.ParseInt(newValue, 0, 64); err == nil { n.Style = yaml.TaggedStyle n.Tag = "!!int" } else if _, err := strconv.ParseFloat(newValue, 64); err == nil { diff --git a/validator/shelltests/hex_integer.test b/validator/shelltests/hex_integer.test new file mode 100644 index 0000000..56a8868 --- /dev/null +++ b/validator/shelltests/hex_integer.test @@ -0,0 +1,8 @@ +OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT="0xdeadbeef" ./otel_config_validator -o out/hex_integer.json shelltests/hex_integer.yaml +>>> +>>>= 0 + +jq '.attribute_limits.attribute_value_length_limit' out/hex_integer.json +>>> +3735928559 +>>>= 0 \ No newline at end of file diff --git a/validator/shelltests/hex_integer.yaml b/validator/shelltests/hex_integer.yaml new file mode 100644 index 0000000..b742f2c --- /dev/null +++ b/validator/shelltests/hex_integer.yaml @@ -0,0 +1,18 @@ +# The file format version +file_format: "0.1" + +# Configure if the SDK is disabled or not. +disabled: ${OTEL_SDK_DISABLED:-false} + +# Configure resource for all signals. +resource: + # Configure resource attributes. + attributes: + # Configure `service.name` resource attribute + service.name: "${OTEL_SERVICE_NAME:-unknown_service}" + +attribute_limits: + # Configure max attribute value size. + attribute_value_length_limit: ${OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT} + # Configure max attribute count. + attribute_count_limit: ${OTEL_ATTRIBUTE_COUNT_LIMIT:-128} From c32dc71681e794b6a0859c24e1be6140ea295e3e Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 06:26:37 -0600 Subject: [PATCH 14/30] tag docker image with :current and add example to readme --- validator/Dockerfile | 4 +--- validator/Makefile | 2 +- validator/README.md | 12 ++++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/validator/Dockerfile b/validator/Dockerfile index d4eecbc..e29f2ac 100644 --- a/validator/Dockerfile +++ b/validator/Dockerfile @@ -12,10 +12,8 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o /otel_config_validator FROM gcr.io/distroless/base-debian12 -WORKDIR / +WORKDIR /opt/otel_config_validator COPY --from=build /otel_config_validator /otel_config_validator -USER nonroot:nonroot - ENTRYPOINT ["/otel_config_validator"] diff --git a/validator/Makefile b/validator/Makefile index 1911c7c..d805d9e 100644 --- a/validator/Makefile +++ b/validator/Makefile @@ -2,7 +2,7 @@ ROOT_DIR := $(realpath $(shell dirname $(lastword $(MAKEFILE_LIST)))) SCHEMA_DIR := ${ROOT_DIR}/../schema CURRENT_GIT_REF := $(shell git rev-parse --short HEAD) DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF} -DOCKER_BUILD_ARGS := -f ${ROOT_DIR}/Dockerfile -t otel_config_validator:${DOCKER_IMAGE_TAG} +DOCKER_BUILD_ARGS := -f ${ROOT_DIR}/Dockerfile -t otel_config_validator:${DOCKER_IMAGE_TAG} -t otel_config_validator:current EXAMPLE_FILES := $(shell find ${ROOT_DIR}/../examples -name "*.yaml" -exec basename {} \;) $(shell mkdir -p out) diff --git a/validator/README.md b/validator/README.md index c2039d7..e9411b4 100644 --- a/validator/README.md +++ b/validator/README.md @@ -34,6 +34,18 @@ The format (json or yaml) of the output is based on the extension (`.json` or $ ./otel_config_validator -o out.json ../examples/kitchen-sink.yaml ``` +The docker image creates a directory `/opt/otel_config_validator` and sets it to +the `WORKDIR`, meaning you can mount your current directory to +`/opt/otel_config_validator` and then read/write from `./` in the arguments to +`docker run`: + +``` +$ docker run -v $(pwd):/opt/otel_config_validator otel_config_validator:current -o out.yaml examples/kitchen-sink.yaml +``` + +With the above docker command the output file, `out.yaml`, will be owned by +`root:root` but be readable by any user. + Environment variable substitution is supported with the syntax `${VARIABLE}`. Default values are supported in the form `${VARIABLE:-default}`. From 1bdb2bf86cb7b6ec31f17398d7b9a07b1ef0162d Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 06:29:39 -0600 Subject: [PATCH 15/30] docker: add gomod and gobuild cache to buildx --- validator/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/validator/Dockerfile b/validator/Dockerfile index e29f2ac..80208df 100644 --- a/validator/Dockerfile +++ b/validator/Dockerfile @@ -3,12 +3,16 @@ FROM golang:1.22 AS build WORKDIR /app +RUN go env -w GOMODCACHE=/root/.cache/go-mod +RUN go env -w GOCACHE=/root/.cache/go-build + COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -o /otel_config_validator +RUN --mount=type=cache,target=/root/.cache/go-build,id=cache-go-build --mount=type=cache,target=/root/.cache/go-mod,id=cache-go-mod \ + CGO_ENABLED=0 GOOS=linux go build -o /otel_config_validator FROM gcr.io/distroless/base-debian12 From e0b5940dcb47dee66707fae4542515bcc35edec2 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 06:43:21 -0600 Subject: [PATCH 16/30] github action: sudo apt-get commands in validator tests --- .github/workflows/validator-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validator-tests.yaml b/.github/workflows/validator-tests.yaml index d54c8ce..0b4ae6d 100644 --- a/.github/workflows/validator-tests.yaml +++ b/.github/workflows/validator-tests.yaml @@ -31,8 +31,8 @@ jobs: run: | cd validator - apt update - apt install shelltestrunner + sudo apt-get update + sudo apt-get install shelltestrunner shelltest -c --diff --all shelltests/*.test From 79cbf676bc99c6afc71bcaf5d8c54dabdc0a82a4 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 06:55:11 -0600 Subject: [PATCH 17/30] add sort of validator example files in makefile so shelltests pass --- validator/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/Makefile b/validator/Makefile index d805d9e..37490bb 100644 --- a/validator/Makefile +++ b/validator/Makefile @@ -3,7 +3,7 @@ SCHEMA_DIR := ${ROOT_DIR}/../schema CURRENT_GIT_REF := $(shell git rev-parse --short HEAD) DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF} DOCKER_BUILD_ARGS := -f ${ROOT_DIR}/Dockerfile -t otel_config_validator:${DOCKER_IMAGE_TAG} -t otel_config_validator:current -EXAMPLE_FILES := $(shell find ${ROOT_DIR}/../examples -name "*.yaml" -exec basename {} \;) +EXAMPLE_FILES := $(shell find ${ROOT_DIR}/../examples -name "*.yaml" -exec basename {} \; | sort) $(shell mkdir -p out) validator-copy-schema: From 95975aaa9efdbe3082aa70f7154f530bc4e3faf9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 27 Jul 2024 03:02:33 -0600 Subject: [PATCH 18/30] remove support for json as input config file --- validator/README.md | 25 ++--- validator/main.go | 121 ++++----------------- validator/shelltests/json_replacement.json | 1 - validator/shelltests/json_replacement.test | 3 - 4 files changed, 29 insertions(+), 121 deletions(-) delete mode 100644 validator/shelltests/json_replacement.json delete mode 100644 validator/shelltests/json_replacement.test diff --git a/validator/README.md b/validator/README.md index e9411b4..86e33a2 100644 --- a/validator/README.md +++ b/validator/README.md @@ -1,7 +1,6 @@ ## OpenTelemetry SDK Configuration Validator -This application will replace environment variables in values of valid yaml or -json files, following the rules of [file configuration environment variable +This application will replace environment variables in values of valid yaml files, following the rules of [file configuration environment variable substitution](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/file-configuration.md#environment-variable-substitution), before validating that result against the [OpenTelemetry SDK Configuration schema](https://github.com/open-telemetry/opentelemetry-configuration/). @@ -24,11 +23,11 @@ $ make validator-docker ### Usage -The command `otel_config_validator` takes one argument, the path to the yaml or -json configuration file and optionally the path to a file to output the -configuration after environment variable expansion and validation has been done. -The format (json or yaml) of the output is based on the extension (`.json` or -`yml`/`.yaml`) of the output file name. +The command `otel_config_validator` takes one argument, the path to the yaml +file and optionally the path to a file to output the configuration after +environment variable expansion and validation has been done. The format (json or +yaml) of the output is based on the extension (`.json` or `yml`/`.yaml`) of the +output file name. ``` $ ./otel_config_validator -o out.json ../examples/kitchen-sink.yaml @@ -46,15 +45,9 @@ $ docker run -v $(pwd):/opt/otel_config_validator otel_config_validator:current With the above docker command the output file, `out.yaml`, will be owned by `root:root` but be readable by any user. -Environment variable substitution is supported with the syntax `${VARIABLE}`. -Default values are supported in the form `${VARIABLE:-default}`. - -In the case of json input only strings can be the result of substitution. To -ensure only values are replaced the input must be parsed as valid json or yaml -and in the case of json a value like `${VARIABLE}` will always have to be double -quoted as `"${VARIABLE}"` so will always remain double quoted. If you need to -substitute in a boolean, integer or float please use yaml for the input -configuration file. +Environment variable substitution is supported with the syntax `${VARIABLE}` or +`${env:VARIABLE}`. Default values are supported in the form +`${VARIABLE:-default}`. ### Testing diff --git a/validator/main.go b/validator/main.go index fcb83e1..51e1f3a 100644 --- a/validator/main.go +++ b/validator/main.go @@ -129,54 +129,35 @@ func decodeFile(configFile string, outfileExt string) (interface{}, []byte) { var j []byte var d []byte var u interface{} - var v interface{} var jsonInterface interface{} - - data, err := os.ReadFile(configFile) + + y := decodeYaml(configFile) + replaceYamlVariables(&y) + + err := y.Decode(&u) if err != nil { log.Fatal(err) } - ext := filepath.Ext(configFile) - if isYamlExt(ext) { - y := decodeYaml(configFile) - replaceYamlVariables(&y) - - err = y.Decode(&u) + if len(y.Content) > 0 { + d, err = yaml.Marshal(&y.Content[0]) if err != nil { log.Fatal(err) } - - if len(y.Content) > 0 { - d, err = yaml.Marshal(&y.Content[0]) - if err != nil { - log.Fatal(err) - } - - j, err = k8syaml.YAMLToJSON(d) - if err != nil { - log.Fatalf("Error encoding yaml to json for validation: %+v", err) - } - if err := json.Unmarshal(j, &jsonInterface); err != nil { - log.Fatalf("Invalid json file from yaml file %s: %+v", configFile, err) - } + j, err = k8syaml.YAMLToJSON(d) + if err != nil { + log.Fatalf("Error encoding yaml to json for validation: %+v", err) } - toWrite := convertYamlNode(y, outfileExt) - - return jsonInterface, toWrite - } - - if err := json.Unmarshal(data, &v); err != nil { - log.Fatalf("Invalid json file %s: %#v", configFile, err) + if err := json.Unmarshal(j, &jsonInterface); err != nil { + log.Fatalf("Invalid json from yaml file %s: %+v", configFile, err) + } } - expandedConfig, b := replaceJsonVariables(v) + toWrite := convertYamlNode(y, outfileExt) - toWrite := convertJsonBytes(b, outfileExt) - - return expandedConfig, toWrite + return jsonInterface, toWrite } func isYamlExt(ext string) bool { @@ -205,18 +186,9 @@ func convertYamlNode(n yaml.Node, ext string) []byte { return d } -func convertJsonBytes(jsonBytes []byte, ext string) []byte { - if isYamlExt(ext) { - b, _ := k8syaml.JSONToYAML(jsonBytes) - return b - } - - return jsonBytes -} - func decodeYaml(file string) yaml.Node { var node yaml.Node - + body, err := os.ReadFile(file) if err != nil { log.Fatalf("Failed to read configuration file %s: %v", file, err) @@ -319,59 +291,6 @@ func handleScalarNode(n *yaml.Node) { } -// json variable replacement is basic as the json value that -// is an environment variable will always be quoted, so only -// strings are supported -func replaceJsonVariables(c interface{}) (interface{}, []byte) { - expandedConfig := make(map[string]any) - - m, _ := c.(map[string]any) - for k := range m { - val := expandJsonValues(m[k]) - expandedConfig[k] = val - } - b, err := json.Marshal(expandedConfig) - if err != nil { - log.Fatalf("json.Marshal: %+v", err) - } - - y, err := k8syaml.JSONToYAML(b) - if err != nil { - log.Fatalf("Error converting json result to yaml: %+v", err) - } - - return expandedConfig, y -} - -func expandJsonValues(value interface{}) any { - switch v := value.(type) { - case string: - if !strings.Contains(v, "${") || !strings.Contains(v, "}") { - return v - } - - return expandString(v) - case []any: - l := []any{} - for _, e := range v { - newElement := expandJsonValues(e) - l = append(l, newElement) - } - return l - case map[string]any: - newMap := make(map[string]any) - - for k, v := range v { - updated := expandJsonValues(v) - newMap[k] = updated - } - - return newMap - } - - return value -} - // Replace environment variables, like ${EXAMPLE}, with their value. // This does not use `os.ExpandVars` in order to support defaults // for missing variables, ${VAR:-default} @@ -392,11 +311,11 @@ func expandString(s string) string { // iterates over a string to find all environment variables // returns a map of variables to strings or nil func findAllVars(s string) map[string]interface{} { - var envVar string + var envVar string var newValue interface{} var isSet bool var substr string - + result := make(map[string]interface{}) lenS := len(s) @@ -416,8 +335,8 @@ func findAllVars(s string) map[string]interface{} { } else { envVar = substr[openIndex+6 : closeIndex] } - - fullEnvVar := substr[openIndex : closeIndex+1] + + fullEnvVar := substr[openIndex : closeIndex+1] maybeDefaultIndex := strings.Index(envVar, ":-") diff --git a/validator/shelltests/json_replacement.json b/validator/shelltests/json_replacement.json deleted file mode 100644 index 1f34b6c..0000000 --- a/validator/shelltests/json_replacement.json +++ /dev/null @@ -1 +0,0 @@ -{"attribute_limits":{"attribute_count_limit":128,"attribute_value_length_limit":512},"disabled":false,"file_format":"0.1","resource":{"attributes":{"service.name":"${OTEL_SERVICE_NAME:-unknown_service}"}}} diff --git a/validator/shelltests/json_replacement.test b/validator/shelltests/json_replacement.test deleted file mode 100644 index 1756d16..0000000 --- a/validator/shelltests/json_replacement.test +++ /dev/null @@ -1,3 +0,0 @@ -./otel_config_validator -o out/out.json shelltests/json_replacement.json ->>> ->>>= 0 From d15ea0811a2140784d35a70c3107bc408b260134 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 23 Jul 2024 16:29:39 -0600 Subject: [PATCH 19/30] add binary and docker image publishing --- .github/workflows/release.yml | 281 ++++++++++++++++++++++++ .github/workflows/validator-docker.yaml | 63 ++++++ dist.toml | 27 +++ 3 files changed, 371 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/validator-docker.yaml create mode 100644 dist.toml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a00892e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,281 @@ +# Copyright 2022-2024, axodotdev +# SPDX-License-Identifier: MIT or Apache-2.0 +# +# CI that: +# +# * checks for a Git Tag that looks like a release +# * builds artifacts with cargo-dist (archives, installers, hashes) +# * uploads those artifacts to temporary workflow zip +# * on success, uploads the artifacts to a GitHub Release +# +# Note that the GitHub Release will be created with a generated +# title/body based on your changelogs. + +name: Release +permissions: + "contents": "write" + +# This task will run whenever you push a git tag that looks like a version +# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. +# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where +# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION +# must be a Cargo-style SemVer Version (must have at least major.minor.patch). +# +# If PACKAGE_NAME is specified, then the announcement will be for that +# package (erroring out if it doesn't have the given version or isn't cargo-dist-able). +# +# If PACKAGE_NAME isn't specified, then the announcement will be for all +# (cargo-dist-able) packages in the workspace with that version (this mode is +# intended for workspaces with only one dist-able package, or with all dist-able +# packages versioned/released in lockstep). +# +# If you push multiple tags at once, separate instances of this workflow will +# spin up, creating an independent announcement for each one. However, GitHub +# will hard limit this to 3 tags per commit, as it will assume more tags is a +# mistake. +# +# If there's a prerelease-style suffix to the version, then the release(s) +# will be marked as a prerelease. +on: + pull_request: + push: + tags: + - '**[0-9]+.[0-9]+.[0-9]+*' + +jobs: + # Run 'cargo dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: "ubuntu-20.04" + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ !github.event.pull_request && github.ref_name || '' }} + tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} + publishing: ${{ !github.event.pull_request }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + # we specify bash to get pipefail; it guards against the `curl` command + # failing. otherwise `sh` won't catch that `curl` returned non-0 + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.19.1/cargo-dist-installer.sh | sh" + - name: Cache cargo-dist + uses: actions/upload-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/cargo-dist + # sure would be cool if github gave us proper conditionals... + # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible + # functionality based on whether this is a pull_request, and whether it's from a fork. + # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* + # but also really annoying to build CI around when it needs secrets to work right.) + - id: plan + run: | + cargo dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "cargo dist ran successfully" + cat plan-dist-manifest.json + echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v4 + with: + name: artifacts-plan-dist-manifest + path: plan-dist-manifest.json + + # Build and packages all the platform-specific things + build-local-artifacts: + name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) + # Let the initial task tell us to not run (currently very blunt) + needs: + - plan + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + strategy: + fail-fast: false + # Target platforms/runners are computed by cargo-dist in create-release. + # Each member of the matrix has the following arguments: + # + # - runner: the github runner + # - dist-args: cli flags to pass to cargo dist + # - install-dist: expression to run to install cargo-dist on the runner + # + # Typically there will be: + # - 1 "global" task that builds universal installers + # - N "local" tasks that build each platform's binaries and platform-specific installers + matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} + runs-on: ${{ matrix.runner }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + steps: + - name: enable windows longpaths + run: | + git config --global core.longpaths true + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + run: ${{ matrix.install_dist }} + # Get the dist-manifest + - name: Fetch local artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - name: Build artifacts + run: | + # Actually do builds and make zips and whatnot + cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "cargo dist ran successfully" + - id: cargo-dist + name: Post-build + # We force bash here just because github makes it really hard to get values up + # to "real" actions without writing to env-vars, and writing to env-vars has + # inconsistent syntax between shell and powershell. + shell: bash + run: | + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v4 + with: + name: artifacts-build-local-${{ join(matrix.targets, '_') }} + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + # Build and package all the platform-agnostic(ish) things + build-global-artifacts: + needs: + - plan + - build-local-artifacts + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cached cargo-dist + uses: actions/download-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/cargo-dist + # Get all the local artifacts for the global tasks to use (for e.g. checksums) + - name: Fetch local artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: cargo-dist + shell: bash + run: | + cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "cargo dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v4 + with: + name: artifacts-build-global + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + # Determines if we should publish/announce + host: + needs: + - plan + - build-local-artifacts + - build-global-artifacts + # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-20.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cached cargo-dist + uses: actions/download-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/cargo-dist + # Fetch artifacts from scratch-storage + - name: Fetch artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: host + shell: bash + run: | + cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v4 + with: + # Overwrite the previous copy + name: artifacts-dist-manifest + path: dist-manifest.json + # Create a GitHub Release while uploading all files to it + - name: "Download GitHub Artifacts" + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: artifacts + merge-multiple: true + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create GitHub Release + env: + PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}" + ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}" + ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}" + RELEASE_COMMIT: "${{ github.sha }}" + run: | + # Write and read notes from a file to avoid quoting breaking things + echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt + + gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/* + + announce: + needs: + - plan + - host + # use "always() && ..." to allow us to wait for all publish jobs while + # still allowing individual publish jobs to skip themselves (for prereleases). + # "host" however must run to completion, no skipping allowed! + if: ${{ always() && needs.host.result == 'success' }} + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive diff --git a/.github/workflows/validator-docker.yaml b/.github/workflows/validator-docker.yaml new file mode 100644 index 0000000..b134704 --- /dev/null +++ b/.github/workflows/validator-docker.yaml @@ -0,0 +1,63 @@ +name: Validator Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + release: + types: [published] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + attestations: write + id-token: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=sha-,suffix=,format=short + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Copy schema files + run: make validator-copy-schema + + - name: Build and push + id: push + uses: docker/build-push-action@v6 + with: + context: ./validator/ + cache-from: type=gha + cache-to: type=gha,mode=max + file: ./validator/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 diff --git a/dist.toml b/dist.toml new file mode 100644 index 0000000..47a82bb --- /dev/null +++ b/dist.toml @@ -0,0 +1,27 @@ +[package] +name = "otel_config_validator" +description = "Releaser for otel_config_validator" +version = "0.2.0" +license = "Apache-2.0" +repository = "https://github.com/open-telemetry/opentelemetry-configuration" +homepage = "https://opentelemetry.io/" +binaries = ["validator/otel_config_validator"] +build-command = ["make", "validator"] + +# Config for 'cargo dist' +[dist] +# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.19.1" +# CI backends to support +ci = "github" +# The installers to generate for each app +installers = ["shell", "powershell"] +# Target platforms to build apps for (Rust target-triple syntax) +targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "x86_64-pc-windows-msvc"] +# Path that installers should place binaries in +install-path = ["$OTEL_CONFIG_VALIDATOR_HOME/bin", "~/.otel_config_validator/bin"] +# Publish jobs to run in CI +pr-run-mode = "plan" +# Whether to install an updater program +install-updater = false + From c71308bef9a160ac8e704a019f75cda2d00df636 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 16 Aug 2024 06:05:29 -0600 Subject: [PATCH 20/30] add schema option to validator cli --- validator/main.go | 84 +++++++++++++++---- .../opentelemetry_configuration.json | 1 + validator/shelltests/help.test | 1 + validator/shelltests/schema_option.test | 4 + validator/shelltests/schema_option.yaml | 1 + 5 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 validator/shelltests/false_schema/opentelemetry_configuration.json create mode 100644 validator/shelltests/schema_option.test create mode 100644 validator/shelltests/schema_option.yaml diff --git a/validator/main.go b/validator/main.go index 51e1f3a..536762f 100644 --- a/validator/main.go +++ b/validator/main.go @@ -34,6 +34,12 @@ func main() { OnlyOnce: true, Usage: "optionally where to output the configuration (as json or yaml) after variable expansion and validation", }, + &cli.StringFlag{ + Name: "schema", + Aliases: []string{"s"}, + OnlyOnce: true, + Usage: "directory containing the json schema to use instead of the compiled in version", + }, }, Action: runAction(), } @@ -48,14 +54,23 @@ func runAction() func(ctx context.Context, cmd *cli.Command) error { if cmd.Args().Len() < 1 { log.Fatalf("Error: Must pass a configuration filename") } else { + isOutputSet := cmd.IsSet("output") configFilePath := cmd.Args().Get(0) + //make schemaDir able to be nil if not set + var schemaDir *string + schemaDir = nil + if cmd.IsSet("schema") { + tmp := cmd.String("schema") + schemaDir = &tmp + } + outFile := cmd.String("output") - ext := validateOutputOption(outFile) + ext := validateOutputOption(isOutputSet, outFile) - yaml := validateConfiguration(configFilePath, ext) + yaml := validateConfiguration(configFilePath, ext, schemaDir) - if outFile != "" { + if isOutputSet { toFile(yaml, outFile) } @@ -66,31 +81,55 @@ func runAction() func(ctx context.Context, cmd *cli.Command) error { } } -func validateOutputOption(outFile string) string { - if outFile != "" { - ext := filepath.Ext(outFile) - if !isYamlExt(ext) && !isJsonExt(ext) { - log.Fatalf("Unknown extension on output file %v", outFile) - } +func validateOutputOption(isOutputSet bool, outFile string) string { + if isOutputSet { + if outFile != "" { + ext := filepath.Ext(outFile) + if !isYamlExt(ext) && !isJsonExt(ext) { + log.Fatalf("Unknown extension on output file %v", outFile) + } - err := os.MkdirAll(filepath.Dir(outFile), 0700) - if err != nil { - log.Fatalf("Unable to create dir for output file %s: %+v", outFile, err) - } + err := os.MkdirAll(filepath.Dir(outFile), 0700) + if err != nil { + log.Fatalf("Unable to create dir for output file %s: %+v", outFile, err) + } - return ext + return ext + } else { + log.Fatal("Output can not be an empty string") + } } return "" } -func validateConfiguration(configFile string, outfileExt string) []byte { - schemaFiles, err := schemaFS.ReadDir("schema") +func add_resources_from_dir(c *jsonschema.Compiler, schemaDir string) { + schemaFiles, err :=os.ReadDir(schemaDir) if err != nil { log.Fatal(err) } - c := jsonschema.NewCompiler() + for _, file := range schemaFiles { + schemaURL, err := url.JoinPath("https://opentelemetry.io/otelconfig/", file.Name()) + if err != nil { + log.Fatal(err) + } + schema, err := os.ReadFile(filepath.Join(schemaDir, file.Name())) + if err != nil { + log.Fatal(err) + } + + if err := c.AddResource(schemaURL, bytes.NewReader(schema)); err != nil { + log.Fatal(err) + } + } +} + +func add_resources_from_embed(c *jsonschema.Compiler) { + schemaFiles, err :=schemaFS.ReadDir("schema") + if err != nil { + log.Fatal(err) + } for _, file := range schemaFiles { schemaURL, err := url.JoinPath("https://opentelemetry.io/otelconfig/", file.Name()) @@ -107,6 +146,17 @@ func validateConfiguration(configFile string, outfileExt string) []byte { } } +} + + +func validateConfiguration(configFile string, outfileExt string, schemaDir *string) []byte { + c := jsonschema.NewCompiler() + if schemaDir != nil { + add_resources_from_dir(c, *schemaDir) + } else { + add_resources_from_embed(c) + } + schema, err := c.Compile("https://opentelemetry.io/otelconfig/opentelemetry_configuration.json") if err != nil { log.Fatalf("%#v", err) diff --git a/validator/shelltests/false_schema/opentelemetry_configuration.json b/validator/shelltests/false_schema/opentelemetry_configuration.json new file mode 100644 index 0000000..c508d53 --- /dev/null +++ b/validator/shelltests/false_schema/opentelemetry_configuration.json @@ -0,0 +1 @@ +false diff --git a/validator/shelltests/help.test b/validator/shelltests/help.test index e1948d2..e5740fe 100644 --- a/validator/shelltests/help.test +++ b/validator/shelltests/help.test @@ -11,5 +11,6 @@ COMMANDS: GLOBAL OPTIONS: --output value, -o value optionally where to output the configuration (as json or yaml) after variable expansion and validation + --schema value, -s value directory containing the json schema to use instead of the compiled in version --help, -h show help (default: false) >>>= 0 diff --git a/validator/shelltests/schema_option.test b/validator/shelltests/schema_option.test new file mode 100644 index 0000000..1d086a8 --- /dev/null +++ b/validator/shelltests/schema_option.test @@ -0,0 +1,4 @@ +./otel_config_validator -s shelltests/false_schema shelltests/schema_option.yaml +>>>2 +jsonschema: '' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#: not allowed +>>>= 1 diff --git a/validator/shelltests/schema_option.yaml b/validator/shelltests/schema_option.yaml new file mode 100644 index 0000000..3ca1c82 --- /dev/null +++ b/validator/shelltests/schema_option.yaml @@ -0,0 +1 @@ +file_format: "0.1" From 772de78fb47a5d856d909fad3a2df71e734a8c2c Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 20 Aug 2024 15:28:13 -0600 Subject: [PATCH 21/30] validator: disconnect version of cli from schema version --- dist.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dist.toml b/dist.toml index 47a82bb..612ed86 100644 --- a/dist.toml +++ b/dist.toml @@ -1,7 +1,7 @@ [package] name = "otel_config_validator" description = "Releaser for otel_config_validator" -version = "0.2.0" +version = "0.1.0" license = "Apache-2.0" repository = "https://github.com/open-telemetry/opentelemetry-configuration" homepage = "https://opentelemetry.io/" @@ -19,7 +19,7 @@ installers = ["shell", "powershell"] # Target platforms to build apps for (Rust target-triple syntax) targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "x86_64-pc-windows-msvc"] # Path that installers should place binaries in -install-path = ["$OTEL_CONFIG_VALIDATOR_HOME/bin", "~/.otel_config_validator/bin"] +install-path = ["$OTEL_CONFIG_VALIDATOR_HOME/bin", "~/.local/bin"] # Publish jobs to run in CI pr-run-mode = "plan" # Whether to install an updater program From bfd845a8cbba5fde56a58aec7a4425177facb3c8 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 20 Aug 2024 15:49:04 -0600 Subject: [PATCH 22/30] validator: update readme docs for -s/--schema option --- validator/README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/validator/README.md b/validator/README.md index 86e33a2..f4c88e7 100644 --- a/validator/README.md +++ b/validator/README.md @@ -1,8 +1,10 @@ ## OpenTelemetry SDK Configuration Validator -This application will replace environment variables in values of valid yaml files, following the rules of [file configuration environment variable +This application will replace environment variables in values of valid yaml +files, following the rules of [file configuration environment variable substitution](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/file-configuration.md#environment-variable-substitution), -before validating that result against the [OpenTelemetry SDK Configuration +before validating that result against a version of the [OpenTelemetry SDK +Configuration schema](https://github.com/open-telemetry/opentelemetry-configuration/). ### Build @@ -21,6 +23,10 @@ Same is true for building the docker image: $ make validator-docker ``` +The schema in the `schema` directory is the default that will be used when +running the validator. To use a different schema a directory containing the +schema can be passed with `-s`/`--schema`. + ### Usage The command `otel_config_validator` takes one argument, the path to the yaml @@ -49,6 +55,15 @@ Environment variable substitution is supported with the syntax `${VARIABLE}` or `${env:VARIABLE}`. Default values are supported in the form `${VARIABLE:-default}`. +#### Using a Different Schema Version + +To use a version of of the schema other than the one builtin to the +`otel_config_validator` executable pass the `-s ` option: + +``` +$ ./otel_config_validator -o out.json -s .../some/path/opentelemetry-configuration-0.1.0/ ../examples/kitchen-sink.yaml +``` + ### Testing Run the Go unit tests: From cb25f5c2da9d8e9adba4255cd8772f96b3c9c542 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 20 Aug 2024 16:04:55 -0600 Subject: [PATCH 23/30] validator: add tag prefix to seperate cli tags from schema tags --- .../workflows/{release.yml => validator-release.yml} | 2 +- dist.toml | 2 ++ validator/README.md | 10 ++++++++++ validator/main.go | 1 + 4 files changed, 14 insertions(+), 1 deletion(-) rename .github/workflows/{release.yml => validator-release.yml} (99%) diff --git a/.github/workflows/release.yml b/.github/workflows/validator-release.yml similarity index 99% rename from .github/workflows/release.yml rename to .github/workflows/validator-release.yml index a00892e..626ce80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/validator-release.yml @@ -40,7 +40,7 @@ on: pull_request: push: tags: - - '**[0-9]+.[0-9]+.[0-9]+*' + - 'validator**[0-9]+.[0-9]+.[0-9]+*' jobs: # Run 'cargo dist plan' (or host) to determine what tasks we need to do diff --git a/dist.toml b/dist.toml index 612ed86..5dcc2cd 100644 --- a/dist.toml +++ b/dist.toml @@ -22,6 +22,8 @@ targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux- install-path = ["$OTEL_CONFIG_VALIDATOR_HOME/bin", "~/.local/bin"] # Publish jobs to run in CI pr-run-mode = "plan" +# A prefix git tags must include for cargo-dist to care about them +tag-namespace = "validator" # Whether to install an updater program install-updater = false diff --git a/validator/README.md b/validator/README.md index f4c88e7..2e9b936 100644 --- a/validator/README.md +++ b/validator/README.md @@ -80,3 +80,13 @@ Running the tests of the compiled CLI requires $ shelltest -c --diff --all shelltests/*.test ``` +### Releasing + +To release a new version of `otel_config_validator` the version in `dist.toml` +and the `Version` property of `cli.Comamnd` in `main.go` must be bumped. Next, a +tag prefixed with `validator-` must be created and pushed to the repository, for +example `validator-0.1.0`. Then, the `cargo-dist` Github Action will create a +Github release, build binaries for multiple platforms and publish them to the +new release. + +Docker image are published on merge to `main`. diff --git a/validator/main.go b/validator/main.go index 536762f..8d4fa5f 100644 --- a/validator/main.go +++ b/validator/main.go @@ -27,6 +27,7 @@ func main() { cmd := &cli.Command{ Name: "otel_config_validator", Usage: "Validate a configuration file against the OpenTelemetry Configuration Schema", + Version: "0.1.0", Flags: []cli.Flag{ &cli.StringFlag{ Name: "output", From 429b6ef8c7c1660edd8954883d7bc94631216ba2 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 20 Aug 2024 16:20:18 -0600 Subject: [PATCH 24/30] validator: fix test for help output for validator --- validator/shelltests/help.test | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/validator/shelltests/help.test b/validator/shelltests/help.test index e5740fe..e6c14d0 100644 --- a/validator/shelltests/help.test +++ b/validator/shelltests/help.test @@ -6,6 +6,9 @@ NAME: USAGE: otel_config_validator [global options] [command [command options]] [arguments...] +VERSION: + 0.1.0 + COMMANDS: help, h Shows a list of commands or help for one command @@ -13,4 +16,5 @@ GLOBAL OPTIONS: --output value, -o value optionally where to output the configuration (as json or yaml) after variable expansion and validation --schema value, -s value directory containing the json schema to use instead of the compiled in version --help, -h show help (default: false) + --version, -v print the version (default: false) >>>= 0 From 58c6e5aeb25f1870b672406f93eb1c2cf6952346 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 22 Aug 2024 05:16:16 -0600 Subject: [PATCH 25/30] validator: add to readme opening that a json/yaml file can be written --- validator/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validator/README.md b/validator/README.md index 2e9b936..4f21c7a 100644 --- a/validator/README.md +++ b/validator/README.md @@ -5,7 +5,8 @@ files, following the rules of [file configuration environment variable substitution](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/file-configuration.md#environment-variable-substitution), before validating that result against a version of the [OpenTelemetry SDK Configuration -schema](https://github.com/open-telemetry/opentelemetry-configuration/). +schema](https://github.com/open-telemetry/opentelemetry-configuration/) and +optionally outputting to a yaml or json file.. ### Build From a886bceb1ae1a7063fe42480029290b133aad45d Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 22 Aug 2024 05:18:48 -0600 Subject: [PATCH 26/30] validator readme: link to otel-spec when describing var subs --- validator/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator/README.md b/validator/README.md index 4f21c7a..fcc6b5a 100644 --- a/validator/README.md +++ b/validator/README.md @@ -54,7 +54,9 @@ With the above docker command the output file, `out.yaml`, will be owned by Environment variable substitution is supported with the syntax `${VARIABLE}` or `${env:VARIABLE}`. Default values are supported in the form -`${VARIABLE:-default}`. +`${VARIABLE:-default}`. The full specification for variable substitution can be +found in the +[opentelemetry-specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/file-configuration.md#environment-variable-substitution). #### Using a Different Schema Version From f34e91177a922f3082bfafad913cf3cc33128d4e Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 22 Aug 2024 05:35:29 -0600 Subject: [PATCH 27/30] validator: add note that Go 1.20+ is required to build --- validator/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/validator/README.md b/validator/README.md index fcc6b5a..12f4761 100644 --- a/validator/README.md +++ b/validator/README.md @@ -6,10 +6,14 @@ substitution](https://github.com/open-telemetry/opentelemetry-specification/blob before validating that result against a version of the [OpenTelemetry SDK Configuration schema](https://github.com/open-telemetry/opentelemetry-configuration/) and -optionally outputting to a yaml or json file.. +optionally outputting to a yaml or json file. ### Build +Requirements: + +- Go 1.20 or above + The `schema` directory is required to be in the directory of the Go file that embeds it, so a `go build` alone will fail, instead run `make` which will copy the schema: From aacbdcfb9ccbc2b8ffe883a3090b343693bfa312 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 22 Aug 2024 05:38:33 -0600 Subject: [PATCH 28/30] validator: fix readme reference to makefile target for docker Co-authored-by: jack-berg <34418638+jack-berg@users.noreply.github.com> --- validator/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/README.md b/validator/README.md index 12f4761..780123b 100644 --- a/validator/README.md +++ b/validator/README.md @@ -25,7 +25,7 @@ $ make validator Same is true for building the docker image: ``` -$ make validator-docker +$ make validator-docker-image ``` The schema in the `schema` directory is the default that will be used when From c3b2f7cf85758085f3cdaa452debd6fb9b488e80 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 22 Aug 2024 12:39:34 -0600 Subject: [PATCH 29/30] validator: use shelltest docker image to run shelltests To support this change the runner now copies the built cli from the builder stage of the main Dockerfile and the Dockerfile.shelltest was removed. To support running in both local environments and Docker the shelltests depend on the cli being in the path and the Docker image adds `/` to the PATH, where it copes the binary from the first stage, and the makefile run of the otel_config_validator on the examples will add the validator dir to the path at the end so it is only used if `/otel_config_validator` isn't in the PATH. --- .github/workflows/validator-tests.yaml | 10 +--------- validator/Dockerfile | 21 ++++++++++++++++++++- validator/Dockerfile.shelltest | 15 --------------- validator/Makefile | 22 +++++++++++++++------- validator/README.md | 6 ++++-- validator/shelltests/env_prefix.test | 2 +- validator/shelltests/help.test | 2 +- validator/shelltests/hex_integer.test | 2 +- validator/shelltests/json_out.test | 2 +- validator/shelltests/missing_arg.test | 2 +- validator/shelltests/missing_ext.test | 2 +- validator/shelltests/missing_required.test | 2 +- validator/shelltests/no_expected_key.test | 2 +- validator/shelltests/schema_option.test | 2 +- validator/shelltests/string_for_int.test | 2 +- validator/shelltests/yaml_out.test | 2 +- 16 files changed, 51 insertions(+), 45 deletions(-) delete mode 100644 validator/Dockerfile.shelltest diff --git a/.github/workflows/validator-tests.yaml b/.github/workflows/validator-tests.yaml index 0b4ae6d..21b335c 100644 --- a/.github/workflows/validator-tests.yaml +++ b/.github/workflows/validator-tests.yaml @@ -29,13 +29,5 @@ jobs: - name: Run ShellTests run: | - cd validator - - sudo apt-get update - sudo apt-get install shelltestrunner - - shelltest -c --diff --all shelltests/*.test + make validator-run-shelltests - # TODO: Push this to registry on release - - name: Build Docker Image - run: make validator-docker-image diff --git a/validator/Dockerfile b/validator/Dockerfile index 80208df..4d419d8 100644 --- a/validator/Dockerfile +++ b/validator/Dockerfile @@ -14,10 +14,29 @@ COPY . . RUN --mount=type=cache,target=/root/.cache/go-build,id=cache-go-build --mount=type=cache,target=/root/.cache/go-mod,id=cache-go-mod \ CGO_ENABLED=0 GOOS=linux go build -o /otel_config_validator -FROM gcr.io/distroless/base-debian12 +FROM gcr.io/distroless/base-debian12 AS releaser WORKDIR /opt/otel_config_validator COPY --from=build /otel_config_validator /otel_config_validator ENTRYPOINT ["/otel_config_validator"] + +FROM ubuntu:22.04 AS shelltest + +RUN DEBIAN_FRONTEND=noninteractive \ + apt-get update \ + && apt-get install -y software-properties-common \ + && apt-add-repository ppa:rmescandon/yq \ + && apt-get update \ + && apt-get install -y shelltestrunner jq yq make \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root/validator + +COPY --from=build /otel_config_validator /otel_config_validator + +# add otel_config_validator to the path +ENV PATH=/:$PATH + +ENTRYPOINT ["shelltest"] diff --git a/validator/Dockerfile.shelltest b/validator/Dockerfile.shelltest deleted file mode 100644 index 4810ff3..0000000 --- a/validator/Dockerfile.shelltest +++ /dev/null @@ -1,15 +0,0 @@ -FROM ubuntu:22.04 - -RUN - -RUN DEBIAN_FRONTEND=noninteractive \ - apt-get update \ - && apt-get install -y software-properties-common \ - && apt-add-repository ppa:rmescandon/yq \ - && apt-get update \ - && apt-get install -y shelltestrunner jq yq \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /root - -ENTRYPOINT ["shelltest"] diff --git a/validator/Makefile b/validator/Makefile index 37490bb..23f8ec9 100644 --- a/validator/Makefile +++ b/validator/Makefile @@ -1,8 +1,10 @@ -ROOT_DIR := $(realpath $(shell dirname $(lastword $(MAKEFILE_LIST)))) -SCHEMA_DIR := ${ROOT_DIR}/../schema -CURRENT_GIT_REF := $(shell git rev-parse --short HEAD) -DOCKER_IMAGE_TAG := ${CURRENT_GIT_REF} -DOCKER_BUILD_ARGS := -f ${ROOT_DIR}/Dockerfile -t otel_config_validator:${DOCKER_IMAGE_TAG} -t otel_config_validator:current +ROOT_DIR :=$(realpath $(shell dirname $(lastword $(MAKEFILE_LIST)))) +PARENT_DIR :=$(realpath ${ROOT_DIR}/../) +SCHEMA_DIR :=${ROOT_DIR}/../schema +CURRENT_GIT_REF :=$(shell git rev-parse --short HEAD) +DOCKER_IMAGE_TAG :=${CURRENT_GIT_REF} +DOCKER_BUILD_ARGS :=-f ${ROOT_DIR}/Dockerfile -t otel_config_validator:${DOCKER_IMAGE_TAG} -t otel_config_validator:current +DOCKER_SHELLTEST_BUILD_ARGS :=-f ${ROOT_DIR}/Dockerfile --target shelltest -t shelltest:${CURRENT_GIT_REF} -t shelltest:current EXAMPLE_FILES := $(shell find ${ROOT_DIR}/../examples -name "*.yaml" -exec basename {} \; | sort) $(shell mkdir -p out) @@ -13,12 +15,18 @@ validator: validator-copy-schema go build -C ${ROOT_DIR} ${ROOT_DIR} validator-docker-image: validator-copy-schema - docker build ${DOCKER_BUILD_ARGS} ${ROOT_DIR} + docker build --target releaser ${DOCKER_BUILD_ARGS} ${ROOT_DIR} validator-validate-examples: @for f in $(EXAMPLE_FILES); do \ echo "Validating" $$f ; \ - ${ROOT_DIR}/otel_config_validator -o ${ROOT_DIR}/out/$$f ${ROOT_DIR}/../examples/$$f ; \ + PATH=${PATH}:${ROOT_DIR}/ otel_config_validator -o ${ROOT_DIR}/out/$$f ${ROOT_DIR}/../examples/$$f ; \ done +validator-build-shelltest-image: + docker build ${DOCKER_SHELLTEST_BUILD_ARGS} ${ROOT_DIR} + +validator-run-shelltests: validator-build-shelltest-image + docker run -v ${PARENT_DIR}:/root shelltest:${CURRENT_GIT_REF} -- --plain /root/validator/shelltests + .PHONY: validator-validate-examples validator-copy-schema validator validator-docker-image diff --git a/validator/README.md b/validator/README.md index 780123b..904e57a 100644 --- a/validator/README.md +++ b/validator/README.md @@ -81,10 +81,12 @@ $ go test . Running the tests of the compiled CLI requires [shelltest](https://github.com/simonmichael/shelltestrunner), -[jq](https://github.com/jqlang/jq/) and [yq](https://github.com/mikefarah/yq): +[jq](https://github.com/jqlang/jq/) and [yq](https://github.com/mikefarah/yq) +and setting the `$PATH` to include `otel_config_validator`, but you can just use +the `Makefile` target `validator-run-shelltests` to run them in Docker: ``` -$ shelltest -c --diff --all shelltests/*.test +$ make validator-run-shelltests ``` ### Releasing diff --git a/validator/shelltests/env_prefix.test b/validator/shelltests/env_prefix.test index be1659a..b1e7250 100644 --- a/validator/shelltests/env_prefix.test +++ b/validator/shelltests/env_prefix.test @@ -1,3 +1,3 @@ -./otel_config_validator -o out/out.json shelltests/env_prefix.yaml +otel_config_validator -o out/out.json shelltests/env_prefix.yaml >>> >>>= 0 diff --git a/validator/shelltests/help.test b/validator/shelltests/help.test index e6c14d0..f6f0426 100644 --- a/validator/shelltests/help.test +++ b/validator/shelltests/help.test @@ -1,4 +1,4 @@ -./otel_config_validator --help +otel_config_validator --help >>> NAME: otel_config_validator - Validate a configuration file against the OpenTelemetry Configuration Schema diff --git a/validator/shelltests/hex_integer.test b/validator/shelltests/hex_integer.test index 56a8868..622caaa 100644 --- a/validator/shelltests/hex_integer.test +++ b/validator/shelltests/hex_integer.test @@ -1,4 +1,4 @@ -OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT="0xdeadbeef" ./otel_config_validator -o out/hex_integer.json shelltests/hex_integer.yaml +OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT="0xdeadbeef" otel_config_validator -o out/hex_integer.json shelltests/hex_integer.yaml >>> >>>= 0 diff --git a/validator/shelltests/json_out.test b/validator/shelltests/json_out.test index 975977e..0853416 100644 --- a/validator/shelltests/json_out.test +++ b/validator/shelltests/json_out.test @@ -1,4 +1,4 @@ -FILE_FORMAT=0.1 ./otel_config_validator -o out/out.json shelltests/test.yaml +FILE_FORMAT=0.1 otel_config_validator -o out/out.json shelltests/test.yaml >>> >>>= 0 diff --git a/validator/shelltests/missing_arg.test b/validator/shelltests/missing_arg.test index 1af62af..058c72d 100644 --- a/validator/shelltests/missing_arg.test +++ b/validator/shelltests/missing_arg.test @@ -1,4 +1,4 @@ -./otel_config_validator -o out/out.json +otel_config_validator -o out/out.json >>>2 Error: Must pass a configuration filename >>>= 1 diff --git a/validator/shelltests/missing_ext.test b/validator/shelltests/missing_ext.test index 2bf201f..a758d9b 100644 --- a/validator/shelltests/missing_ext.test +++ b/validator/shelltests/missing_ext.test @@ -1,4 +1,4 @@ -./otel_config_validator -o out shelltests/multiple_defaults.yaml +otel_config_validator -o out shelltests/multiple_defaults.yaml >>>2 Unknown extension on output file out >>>= 1 diff --git a/validator/shelltests/missing_required.test b/validator/shelltests/missing_required.test index a41d573..501cbe1 100644 --- a/validator/shelltests/missing_required.test +++ b/validator/shelltests/missing_required.test @@ -1,4 +1,4 @@ -./otel_config_validator shelltests/missing_required.yaml +otel_config_validator shelltests/missing_required.yaml >>>2 jsonschema: '' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#/required: missing properties: 'file_format' >>>= 1 diff --git a/validator/shelltests/no_expected_key.test b/validator/shelltests/no_expected_key.test index f8fcec5..ac9feeb 100644 --- a/validator/shelltests/no_expected_key.test +++ b/validator/shelltests/no_expected_key.test @@ -1,2 +1,2 @@ -FILE_FORMAT=0.1 ./otel_config_validator shelltests/no_expected_key.yaml +FILE_FORMAT=0.1 otel_config_validator shelltests/no_expected_key.yaml >>>= 1 diff --git a/validator/shelltests/schema_option.test b/validator/shelltests/schema_option.test index 1d086a8..9792bfa 100644 --- a/validator/shelltests/schema_option.test +++ b/validator/shelltests/schema_option.test @@ -1,4 +1,4 @@ -./otel_config_validator -s shelltests/false_schema shelltests/schema_option.yaml +otel_config_validator -s shelltests/false_schema shelltests/schema_option.yaml >>>2 jsonschema: '' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#: not allowed >>>= 1 diff --git a/validator/shelltests/string_for_int.test b/validator/shelltests/string_for_int.test index c34b689..c457788 100644 --- a/validator/shelltests/string_for_int.test +++ b/validator/shelltests/string_for_int.test @@ -1,4 +1,4 @@ -./otel_config_validator -o out/out.json shelltests/string_for_int.yaml +otel_config_validator -o out/out.json shelltests/string_for_int.yaml >>>2 jsonschema: '/attribute_limits/attribute_value_length_limit' does not validate with https://opentelemetry.io/otelconfig/opentelemetry_configuration.json#/properties/attribute_limits/$ref/properties/attribute_value_length_limit/type: expected integer or null, but got string >>>= 1 diff --git a/validator/shelltests/yaml_out.test b/validator/shelltests/yaml_out.test index a89c4a2..ac9b75d 100644 --- a/validator/shelltests/yaml_out.test +++ b/validator/shelltests/yaml_out.test @@ -1,4 +1,4 @@ -FILE_FORMAT=0.1 ./otel_config_validator -o out/out.yaml shelltests/test.yaml +FILE_FORMAT=0.1 otel_config_validator -o out/out.yaml shelltests/test.yaml >>> >>>= 0 From 1f31826f81dc8ef60569b12065a9127af48d1a92 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 30 Aug 2024 11:20:59 -0600 Subject: [PATCH 30/30] validator: update docker builds to use scratch --- validator/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/Dockerfile b/validator/Dockerfile index 4d419d8..b4fd168 100644 --- a/validator/Dockerfile +++ b/validator/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM golang:1.22 AS build +FROM golang:1.22-alpine AS build WORKDIR /app @@ -14,7 +14,7 @@ COPY . . RUN --mount=type=cache,target=/root/.cache/go-build,id=cache-go-build --mount=type=cache,target=/root/.cache/go-mod,id=cache-go-mod \ CGO_ENABLED=0 GOOS=linux go build -o /otel_config_validator -FROM gcr.io/distroless/base-debian12 AS releaser +FROM scratch AS releaser WORKDIR /opt/otel_config_validator