From a59795b6d2c337ca89af287c3a59ba661d07207f Mon Sep 17 00:00:00 2001 From: Kevin van den Broek Date: Mon, 8 Apr 2024 16:43:18 +0200 Subject: [PATCH] Bazel toolchain support This adds the Bazel toolchain support to change the effective Jsonnet compiler. Go-Jsonnet, Jsonnet (cpp) and Jrsonnet (Rust) are implemented. There are some incompatibilities: - Jsonnet (cpp) does not support the -c flag to create nested output directories. - Jrsonnet (Rust) does not support manifest files containing the files that were generated by the Jsonnet program. It's currently only able to write those outputs to stdout. - Jrsonnet text output (e.g. errors) is different compared to the cpp and Go implementations. Due to these incompatibilities there are two addditional attributes on the toolchain: the flags to pass during directory creation and if manifest files are supported. When adding rules_jsonnet as a bzlmod module, a user can now select the preferred compiler by using: ``` jsonnet = use_extension("@rules_jsonnet//jsonnet:extensions.bzl", "jsonnet") jsonnet.compiler(name = "rust") ``` Currently there are three options: `cpp`, `go` and `rust`. Additionally, this fixes CI: The current .bazelci expects a variable ${{ jsonnet_port }} to be defined. However, that isn't the case with the current setup. The current behaviour is an empty ${{ jsonnet_port }} which causes the implementation to fall back to the default (Go), effectively disabling the cpp tests. The CI runs: ``` bazel test //... --extra_toolchains=@rules_jsonnet//jsonnet:go_jsonnet_toolchain bazel test //... --extra_toolchains=@rules_jsonnet//jsonnet:rust_jsonnet_toolchain bazel test //... --extra_toolchains=@rules_jsonnet//jsonnet:cpp_jsonnet_toolchain ``` Lastly, fix docs: The current docs setup only supports a single source file for docs generation (jsonnet.bzl). Now that `toolchains.bzl` is also a file that must be included in the docs generation, we use a helper file `docs.bzl` to aggregate what needs to be documented. --- .bazelci/presubmit.yml | 10 ++- .github/workflows/create_archive_and_notes.sh | 2 + MODULE.bazel | 42 +++++++++ README.md | 39 +++++++-- docs/BUILD.bazel | 2 +- docs/MODULE.bazel | 14 +++ docs/jsonnet.md | 38 ++++---- examples/BUILD | 4 +- examples/MODULE.bazel | 22 +++++ examples/invalid.out | 4 +- examples/other_module/MODULE.bazel | 3 + examples/other_toolchain_module/MODULE.bazel | 13 +++ examples/out_dir.jsonnet | 1 - jsonnet/BUILD | 64 ++++++++++---- jsonnet/docs.bzl | 25 ++++++ jsonnet/extensions.bzl | 68 +++++++++++++++ jsonnet/jsonnet.bzl | 87 +++++++++---------- jsonnet/toolchain.bzl | 46 ++++++++++ 18 files changed, 378 insertions(+), 106 deletions(-) create mode 100644 examples/other_toolchain_module/MODULE.bazel create mode 100644 jsonnet/docs.bzl create mode 100644 jsonnet/extensions.bzl create mode 100644 jsonnet/toolchain.bzl diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index b273d67..a170f84 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -3,6 +3,10 @@ matrix: platform: - ubuntu2204 - macos + toolchain: + - "@rules_jsonnet//jsonnet:go_jsonnet_toolchain" + - "@rules_jsonnet//jsonnet:cpp_jsonnet_toolchain" + - "@rules_jsonnet//jsonnet:rust_jsonnet_toolchain" tasks: rules_jsonnet: @@ -16,8 +20,7 @@ tasks: platform: ${{ platform }} working_directory: docs test_flags: - - --define - - jsonnet_port=${{ jsonnet_port }} + - "--extra_toolchains=${{ toolchain }}" test_targets: - //... @@ -26,7 +29,6 @@ tasks: platform: ${{ platform }} working_directory: examples test_flags: - - --define - - jsonnet_port=${{ jsonnet_port }} + - "--extra_toolchains=${{ toolchain }}" test_targets: - //... diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index aafc1ee..f78af7b 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -58,5 +58,7 @@ jsonnet_go_repositories() load("@google_jsonnet_go//bazel:deps.bzl", "jsonnet_go_dependencies") jsonnet_go_dependencies() + +register_toolchains("@io_bazel_rules_jsonnet//jsonnet:go_jsonnet_toolchain") \`\`\` EOF diff --git a/MODULE.bazel b/MODULE.bazel index 8cf1d2b..a627ffc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -7,3 +7,45 @@ module( bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "jsonnet", version = "0.20.0") bazel_dep(name = "jsonnet_go", version = "0.20.0", repo_name = "google_jsonnet_go") + +bazel_dep(name = "rules_rust", version = "0.41.1") + +jsonnet = use_extension("//jsonnet:extensions.bzl", "jsonnet") +use_repo(jsonnet, "rules_jsonnet_toolchain") +register_toolchains("@rules_jsonnet_toolchain//:toolchain") + +rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") +rust.toolchain( + edition = "2021", + # Nightly version is required to be able to depend on a binary dependency + # with Cargo. + # See https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#artifact-dependencies. + versions = ["nightly/2024-04-05"], + sha256s = { + "2024-04-05/rustc-nightly-x86_64-apple-darwin.tar.xz": "88fd4812f2bb59c70c6f8c5a25f3a8141dce1cd4290d360b53bad6303af899c4", + "2024-04-05/clippy-nightly-x86_64-apple-darwin.tar.xz": "7b217341e2f6199df8ff58589909412700be8ac4c66bc653089dc686f168188a", + "2024-04-05/cargo-nightly-x86_64-apple-darwin.tar.xz": "f5ac56ad3d78d3bfefc970a7ec8e859c0e535fa2f0d3b64e9aebc9b97322806d", + "2024-04-05/llvm-tools-nightly-x86_64-apple-darwin.tar.xz": "c8cc91a0d7c0c28960976cb583edb8421fc1ad2cb85fee07dd6ee67a2ef2f3f4", + "2024-04-05/rust-std-nightly-x86_64-apple-darwin.tar.xz": "f6ed77da316cd88bc3206552135ff08aa41de2bf12846593f42fb67c7bf04894", + }, +) + +use_repo(rust, "rust_toolchains") +register_toolchains("@rust_toolchains//:all") + +crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate") +crate.spec( + package = "jrsonnet", + version = "0.5.0-pre95", + # Binary artifacts can't be depended upon without specifically marking the + # artifact as `bin`. + artifact = "bin", +) + +# Required for rules_rust to generate binary targets for the Jrsonnet crate. +crate.annotation( + crate = "jrsonnet", + gen_binaries = ["jrsonnet"], +) +crate.from_specs() +use_repo(crate, "crates") diff --git a/README.md b/README.md index 1e492cf..8a3376e 100644 --- a/README.md +++ b/README.md @@ -22,17 +22,40 @@ These are build rules for working with [Jsonnet][jsonnet] files with Bazel. To use the Jsonnet rules as part of your Bazel project, please follow the instructions on [the releases page](https://github.com/bazelbuild/rules_jsonnet/releases). -## Jsonnet Port Selection +## Jsonnet Compiler Selection -By default, Bazel will use [the Go port](https://github.com/google/go-jsonnet) of Jsonnet. To use [the C++ port](https://github.com/google/jsonnet) of Jsonnet instead, invoke Bazel with the `--define jsonnet_port=cpp` command-line flag. To select the Go port explicitly, invoke Bazel with the `--define jsonnet_port=go` command-line flag. +By default for Bzlmod, Bazel will use the [Go +compiler](https://github.com/google/go-jsonnet). Note that the +primary development focus of the Jsonnet project is now with the Go compiler. +This repository's support for using the C++ compiler is deprecated, and may be +removed in a future release. -_bazel_ Flag | Jsonnet Port ------------- | ------------ -(none) | Go -`--define jsonnet_port=cpp`| C++ -`--define jsonnet_port=go` | Go +To use [the +C++](https://github.com/google/jsonnet) or +[Rust](https://github.com/CertainLach/jrsonnet) compiler of Jsonnet instead, +register a different compiler: -Note that the primary development focus of the Jsonnet project is now with the Go port. This repository's support for using the C++ port is deprecated, and may be removed in a future release. +| Jsonnet compiler | MODULE.bazel directive | WORKSPACE directive | +| ---------------- | --------------------------------- | -------------------------------------------------------------------------------- | +| Go | `jsonnet.compiler(name = "go")` | `register_toolchains("@io_bazel_rules_jsonnet//jsonnet:go_jsonnet_toolchain")` | +| cpp | `jsonnet.compiler(name = "cpp")` | `register_toolchains("@io_bazel_rules_jsonnet//jsonnet:cpp_jsonnet_toolchain")` | +| Rust | `jsonnet.compiler(name = "rust")` | `register_toolchains("@io_bazel_rules_jsonnet//jsonnet:rust_jsonnet_toolchain")` | + +Note that `WORKSPACE` users must register a toolchain manually, using the table +above as reference. + +### CLI + +Use the `--extra_toolchains` flag to pass the preferred toolchain to the bazel +invocation: + +```bash +bazel build //... --extra_toolchains=@rules_jsonnet//jsonnet:cpp_jsonnet_toolchain + +bazel test //... --extra_toolchains=@rules_jsonnet//jsonnet:rust_jsonnet_toolchain + +bazel run //... --extra_toolchains=@rules_jsonnet//jsonnet:go_jsonnet_toolchain +``` ## jsonnet_library diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 713c582..f565c9e 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -2,7 +2,7 @@ load("@aspect_bazel_lib//lib:docs.bzl", "stardoc_with_diff_test", "update_docs") stardoc_with_diff_test( name = "jsonnet", - bzl_library_target = "@rules_jsonnet//jsonnet:jsonnet", + bzl_library_target = "@rules_jsonnet//jsonnet:docs", ) update_docs( diff --git a/docs/MODULE.bazel b/docs/MODULE.bazel index 2b79f4e..7460e98 100644 --- a/docs/MODULE.bazel +++ b/docs/MODULE.bazel @@ -11,3 +11,17 @@ local_path_override( module_name = "rules_jsonnet", path = "..", ) + +bazel_dep(name = "rules_rust", version = "0.41.1") + +rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") +rust.toolchain( + edition = "2021", + # Nightly version is required to be able to depend on a binary dependency + # with Cargo. + # See https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#artifact-dependencies. + versions = ["nightly/2024-04-05"], +) + +use_repo(rust, "rust_toolchains") +register_toolchains("@rust_toolchains//:all") diff --git a/docs/jsonnet.md b/docs/jsonnet.md index 88df332..ab9ea97 100644 --- a/docs/jsonnet.md +++ b/docs/jsonnet.md @@ -1,6 +1,6 @@ -Jsonnet Rules +# Jsonnet Rules These are build rules for working with [Jsonnet][jsonnet] files with Bazel. @@ -16,7 +16,7 @@ instructions on [the releases page](https://github.com/bazelbuild/rules_jsonnet/ ## jsonnet_library 
-jsonnet_library(name, deps, srcs, data, imports, jsonnet)
+jsonnet_library(name, deps, srcs, data, imports)
Creates a logical set of Jsonnet files. @@ -60,7 +60,6 @@ Example: | srcs | List of `.jsonnet` files that comprises this Jsonnet library | List of labels | optional | `[]` | | data | - | List of labels | optional | `[]` | | imports | List of import `-J` flags to be passed to the `jsonnet` compiler. | List of strings | optional | `[]` | -| jsonnet | A jsonnet binary | Label | optional | `"@rules_jsonnet//jsonnet:jsonnet_tool"` | @@ -70,7 +69,7 @@ Example: 
 jsonnet_to_json(name, deps, src, data, outs, code_vars, ext_code, ext_code_envs, ext_code_file_vars,
                 ext_code_files, ext_str_envs, ext_str_file_vars, ext_str_files, ext_strs, extra_args,
-                imports, jsonnet, multiple_outputs, out_dir, stamp_keys, tla_code, tla_code_envs,
+                imports, multiple_outputs, out_dir, stamp_keys, tla_code, tla_code_envs,
                 tla_code_files, tla_str_envs, tla_str_files, tla_strs, vars, yaml_stream)
@@ -200,7 +199,6 @@ Example: | ext_strs | - | Dictionary: String -> String | optional | `{}` | | extra_args | Additional command line arguments for the Jsonnet interpreter.

For example, setting this argument to `["--string"]` causes the interpreter to manifest the output file(s) as plain text instead of JSON. | List of strings | optional | `[]` | | imports | List of import `-J` flags to be passed to the `jsonnet` compiler. | List of strings | optional | `[]` | -| jsonnet | A jsonnet binary | Label | optional | `"@rules_jsonnet//jsonnet:jsonnet_tool"` | | multiple_outputs | Set to `True` to explicitly enable multiple file output via the `jsonnet -m` flag.

This is used for the case where multiple file output is used but only for generating a single output file. For example:

local foo = import "foo.jsonnet";

{
    "foo.json": foo,
}


This attribute is incompatible with `out_dir`. | Boolean | optional | `False` | | out_dir | Name of the directory where output files are stored.

If the names of output files are not known up front, this option can be used to write all output files to a single directory artifact. Files in this directory cannot be referenced individually.

This attribute is incompatible with `outs` and `multiple_outputs`. | String | optional | `""` | | stamp_keys | - | List of strings | optional | `[]` | @@ -221,7 +219,7 @@ Example: 
 jsonnet_to_json_test(name, deps, src, data, canonicalize_golden, code_vars, error, ext_code,
                      ext_code_envs, ext_code_file_vars, ext_code_files, ext_str_envs,
-                     ext_str_file_vars, ext_str_files, ext_strs, extra_args, golden, imports, jsonnet,
+                     ext_str_file_vars, ext_str_files, ext_strs, extra_args, golden, imports,
                      output_file_contents, regex, stamp_keys, tla_code, tla_code_envs, tla_code_files,
                      tla_str_envs, tla_str_files, tla_strs, vars, yaml_stream)
@@ -348,7 +346,6 @@ Example: | extra_args | Additional command line arguments for the Jsonnet interpreter.

For example, setting this argument to `["--string"]` causes the interpreter to manifest the output file(s) as plain text instead of JSON. | List of strings | optional | `[]` | | golden | The expected (combined stdout and stderr) output to compare to the output of running `jsonnet` on `src`. | Label | optional | `None` | | imports | List of import `-J` flags to be passed to the `jsonnet` compiler. | List of strings | optional | `[]` | -| jsonnet | A jsonnet binary | Label | optional | `"@rules_jsonnet//jsonnet:jsonnet_tool"` | | output_file_contents | - | Boolean | optional | `True` | | regex | Set to 1 if `golden` contains a regex used to match the output of running `jsonnet` on `src`. | Boolean | optional | `False` | | stamp_keys | - | List of strings | optional | `[]` | @@ -362,29 +359,24 @@ Example: | yaml_stream | - | Boolean | optional | `False` | - + -## JsonnetLibraryInfo +## jsonnet_toolchain 
-JsonnetLibraryInfo()
+jsonnet_toolchain(name, compiler, create_directory_flags, manifest_file_support)
+The Jsonnet compiler information. +**ATTRIBUTES** -**FIELDS** - - - - - -## jsonnet_repositories - -
-jsonnet_repositories()
-
- -Adds the external dependencies needed for the Jsonnet rules. +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| compiler | - | Label | optional | `None` | +| create_directory_flags | The flags passed to the Jsonnet compiler when a directory must be created. | List of strings | required | | +| manifest_file_support | If the Jsonnet compiler supports writing the output filenames to a manifest file. | Boolean | required | | diff --git a/examples/BUILD b/examples/BUILD index 105c314..c9c2eab 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -1,11 +1,11 @@ +load("@rules_jsonnet//jsonnet:jsonnet.bzl", "jsonnet_library", "jsonnet_to_json", "jsonnet_to_json_test") + package(default_visibility = ["//visibility:public"]) # This directory contains unit and regression tests that also serve as examples # for jsonnet rules. The BUILD rules should not contain any jsonnet_to_json # rules as this is redundant with jsonnet_to_json_test rules. -load("@rules_jsonnet//jsonnet:jsonnet.bzl", "jsonnet_library", "jsonnet_to_json", "jsonnet_to_json_test") - jsonnet_library( name = "workflow", srcs = ["workflow.libsonnet"], diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel index 7d190ee..cadd8a1 100644 --- a/examples/MODULE.bazel +++ b/examples/MODULE.bazel @@ -9,8 +9,30 @@ local_path_override( path = "..", ) +jsonnet = use_extension("@rules_jsonnet//jsonnet:extensions.bzl", "jsonnet") + bazel_dep(name = "other_module", version = "0.0.0") local_path_override( module_name = "other_module", path = "other_module", ) + +bazel_dep(name = "other_toolchain_module", version = "0.0.0") +local_path_override( + module_name = "other_toolchain_module", + path = "other_toolchain_module", +) + +bazel_dep(name = "rules_rust", version = "0.41.1") + +rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") +rust.toolchain( + edition = "2021", + # Nightly version is required to be able to depend on a binary dependency + # with Cargo. + # See https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#artifact-dependencies. + versions = ["nightly/2024-04-05"], +) + +use_repo(rust, "rust_toolchains") +register_toolchains("@rust_toolchains//:all") diff --git a/examples/invalid.out b/examples/invalid.out index d8e6fb8..7cf13bd 100644 --- a/examples/invalid.out +++ b/examples/invalid.out @@ -1,3 +1 @@ -^RUNTIME ERROR: Foo\. - .*invalid\.jsonnet:15:1-13 ($|\$ - During evaluation ) +^RUNTIME ERROR: Foo.*invalid\.jsonnet:15:1-13* diff --git a/examples/other_module/MODULE.bazel b/examples/other_module/MODULE.bazel index 1287d51..b4d6cdb 100644 --- a/examples/other_module/MODULE.bazel +++ b/examples/other_module/MODULE.bazel @@ -4,3 +4,6 @@ module( ) bazel_dep(name = "rules_jsonnet", version = "0.0.0") + +jsonnet = use_extension("@rules_jsonnet//jsonnet:extensions.bzl", "jsonnet") +jsonnet.compiler(name = "rust") diff --git a/examples/other_toolchain_module/MODULE.bazel b/examples/other_toolchain_module/MODULE.bazel new file mode 100644 index 0000000..6c1898b --- /dev/null +++ b/examples/other_toolchain_module/MODULE.bazel @@ -0,0 +1,13 @@ +# This module 'other_toolchain_module' is here to test multiple modules +# providing different toolchains and the logic in the toolchain conflict +# resolution logic. + +module( + name = "other_toolchain_module", + version = "0.0.0", +) + +bazel_dep(name = "rules_jsonnet", version = "0.0.0") + +jsonnet = use_extension("@rules_jsonnet//jsonnet:extensions.bzl", "jsonnet") +jsonnet.compiler(name = "rust") diff --git a/examples/out_dir.jsonnet b/examples/out_dir.jsonnet index 8551ed0..5421829 100644 --- a/examples/out_dir.jsonnet +++ b/examples/out_dir.jsonnet @@ -1,5 +1,4 @@ { 'hello.txt': 'Hello, Bazel!', 'goodbye.txt': 'Goodbye, Bazel!', - 'nested/nested.txt': 'This file is in a nested directory.', } diff --git a/jsonnet/BUILD b/jsonnet/BUILD index ead6a25..f45f57d 100644 --- a/jsonnet/BUILD +++ b/jsonnet/BUILD @@ -1,6 +1,7 @@ +load(":toolchain.bzl", "jsonnet_toolchain") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -exports_files(["jsonnet.bzl"]) +exports_files(["docs.bzl", "jsonnet.bzl", "toolchain.bzl"]) bzl_library( name = "bzl_srcs", @@ -8,8 +9,12 @@ bzl_library( ) bzl_library( - name = "jsonnet", - srcs = ["jsonnet.bzl"], + name = "docs", + srcs = [ + "docs.bzl", + "jsonnet.bzl", + "toolchain.bzl", + ], visibility = ["//visibility:public"], deps = [ ":bzl_srcs", @@ -25,25 +30,46 @@ py_binary( visibility = ["//visibility:public"], ) -config_setting( - name = "port_cpp", - define_values = { - "jsonnet_port": "cpp", - }, +toolchain_type(name = "toolchain_type") + +jsonnet_toolchain( + name = "rust_jsonnet", + compiler = "@crates//:jrsonnet__jrsonnet", + create_directory_flags = ["-c"], + manifest_file_support = False, +) + +jsonnet_toolchain( + name = "go_jsonnet", + compiler = "@google_jsonnet_go//cmd/jsonnet", + create_directory_flags = ["-c"], + manifest_file_support = True, +) + +jsonnet_toolchain( + name = "cpp_jsonnet", + compiler = "@jsonnet//cmd:jsonnet", + create_directory_flags = [], + manifest_file_support = True, ) -config_setting( - name = "port_go", - define_values = { - "jsonnet_port": "go", - }, +toolchain( + name = "rust_jsonnet_toolchain", + toolchain = ":rust_jsonnet", + toolchain_type = ":toolchain_type", + visibility = ["//visibility:public"], +) + +toolchain( + name = "go_jsonnet_toolchain", + toolchain = ":go_jsonnet", + toolchain_type = ":toolchain_type", + visibility = ["//visibility:public"], ) -alias( - name = "jsonnet_tool", - actual = select({ - "//jsonnet:port_cpp": "@jsonnet//cmd:jsonnet", - "//conditions:default": "@google_jsonnet_go//cmd/jsonnet", - }), +toolchain( + name = "cpp_jsonnet_toolchain", + toolchain = ":cpp_jsonnet", + toolchain_type = ":toolchain_type", visibility = ["//visibility:public"], ) diff --git a/jsonnet/docs.bzl b/jsonnet/docs.bzl new file mode 100644 index 0000000..9b2aa9b --- /dev/null +++ b/jsonnet/docs.bzl @@ -0,0 +1,25 @@ +"""\ +# Jsonnet Rules + +These are build rules for working with [Jsonnet][jsonnet] files with Bazel. + +[jsonnet]: https://jsonnet.org/ + +## Setup + +To use the Jsonnet rules as part of your Bazel project, please follow the +instructions on [the releases page](https://github.com/bazelbuild/rules_jsonnet/releases). +""" + +load("//jsonnet:toolchain.bzl", _jsonnet_toolchain = "jsonnet_toolchain") +load( + "//jsonnet:jsonnet.bzl", + _jsonnet_library = "jsonnet_library", + _jsonnet_to_json = "jsonnet_to_json", + _jsonnet_to_json_test = "jsonnet_to_json_test", +) + +jsonnet_toolchain = _jsonnet_toolchain +jsonnet_library = _jsonnet_library +jsonnet_to_json = _jsonnet_to_json +jsonnet_to_json_test = _jsonnet_to_json_test diff --git a/jsonnet/extensions.bzl b/jsonnet/extensions.bzl new file mode 100644 index 0000000..2cb7a1f --- /dev/null +++ b/jsonnet/extensions.bzl @@ -0,0 +1,68 @@ +def _get_jsonnet_compiler(module_ctx): + """_get_jsonnet_compiler resolves a Jsonnet compiler from the module graph.""" + + modules_with_compiler = [ + module + for module in module_ctx.modules + if module.tags.compiler + ] + + if not modules_with_compiler: + return "go" + + for module in modules_with_compiler: + if len(module.tags.compiler) != 1: + fail( + "Only one compiler can be specified, got: %s" % + [compiler.name for compiler in module.tags.compiler], + ) + + if module.is_root: + return module.tags.compiler[0].name + + compiler_name = modules_with_compiler[0].tags.compiler[0].name + for module in modules_with_compiler: + if module.tags.compiler[0].name != compiler_name: + fail( + "Different compilers specified by different modules, got: %s. " % + [compiler_name, module.tags.compiler[0].name] + + "Specify a compiler in the root module to resolve this.", + ) + + return compiler_name + +def _jsonnet_impl(module_ctx): + _jsonnet_toolchain_repo( + name = "rules_jsonnet_toolchain", + compiler = _get_jsonnet_compiler(module_ctx), + ) + +jsonnet = module_extension( + implementation = _jsonnet_impl, + tag_classes = { + "compiler": tag_class( + attrs = { + "name": attr.string(), + }, + ), + }, +) + +def _jsonnet_toolchain_repo_impl(ctx): + ctx.file( + "BUILD.bazel", + content = """ +alias( + name = "toolchain", + actual = "@io_bazel_rules_jsonnet//jsonnet:%s_jsonnet_toolchain", +) +""" % ctx.attr.compiler, + executable = False, + ) + +_jsonnet_toolchain_repo = repository_rule( + implementation = _jsonnet_toolchain_repo_impl, + attrs = { + "compiler": attr.string(), + }, +) diff --git a/jsonnet/jsonnet.bzl b/jsonnet/jsonnet.bzl index 75a2374..65f13c6 100644 --- a/jsonnet/jsonnet.bzl +++ b/jsonnet/jsonnet.bzl @@ -11,19 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""Jsonnet Rules - -These are build rules for working with [Jsonnet][jsonnet] files with Bazel. - -[jsonnet]: https://jsonnet.org/ - -## Setup - -To use the Jsonnet rules as part of your Bazel project, please follow the -instructions on [the releases page](https://github.com/bazelbuild/rules_jsonnet/releases). -""" - load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//lib:shell.bzl", "shell") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") @@ -98,11 +85,6 @@ def _jsonnet_library_impl(ctx): ), ] -def _jsonnet_toolchain(ctx): - return struct( - jsonnet_path = ctx.executable.jsonnet.path, - ) - def _quote(s): return '"' + s.replace('"', '\\"') + '"' @@ -146,7 +128,7 @@ def _make_stamp_resolve(ext_vars, ctx, relative = True): val = "$(cat %s)" % stamp_file.short_path else: val = "$(cat %s)" % stamp_file.path - stamp_inputs += [stamp_file] + stamp_inputs.append(stamp_file) results[key] = val @@ -161,7 +143,6 @@ def _jsonnet_to_json_impl(ctx): print("'code_vars' attribute is deprecated, please use 'ext_code'.") depinfo = _setup_deps(ctx.attr.deps) - toolchain = _jsonnet_toolchain(ctx) jsonnet_ext_strs = ctx.attr.ext_strs or ctx.attr.vars jsonnet_ext_str_envs = ctx.attr.ext_str_envs jsonnet_ext_code = ctx.attr.ext_code or ctx.attr.code_vars @@ -191,7 +172,7 @@ def _jsonnet_to_json_impl(ctx): command = ( [ "set -e;", - toolchain.jsonnet_path, + ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.compiler.path, ] + ["-J " + shell.quote(im) for im in _get_import_paths(ctx.label, [ctx.file.src], ctx.attr.imports)] + ["-J " + shell.quote(im) for im in depinfo.imports.to_list()] + @@ -233,10 +214,15 @@ def _jsonnet_to_json_impl(ctx): # will write the resulting JSON to stdout, which is redirected into # a single JSON output file. if ctx.attr.out_dir: - output_manifest = ctx.actions.declare_file("_%s_outs.mf" % ctx.label.name) + if ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.manifest_file_support: + output_manifest = ctx.actions.declare_file("_%s_outs.mf" % ctx.label.name) + outputs.append(output_manifest) + command += ["-o", output_manifest.path] + out_dir = ctx.actions.declare_directory(ctx.attr.out_dir) - outputs += [out_dir, output_manifest] - command += [ctx.file.src.path, "-c", "-m", out_dir.path, "-o", output_manifest.path] + outputs.append(out_dir) + command += [ctx.file.src.path, "-m", out_dir.path] + command += ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.create_directory_flags elif len(ctx.attr.outs) > 1 or ctx.attr.multiple_outputs: # Assume that the output directory is the leading part of the # directory name that is shared by all output files. @@ -248,16 +234,20 @@ def _jsonnet_to_json_impl(ctx): if part1 != part2: base_dirname = base_dirname[:i] break - - output_manifest = ctx.actions.declare_file("_%s_outs.mf" % ctx.label.name) - outputs += ctx.outputs.outs + [output_manifest] - command += ["-m", "/".join(base_dirname), ctx.file.src.path, "-o", output_manifest.path] + if ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.manifest_file_support: + output_manifest = ctx.actions.declare_file("_%s_outs.mf" % ctx.label.name) + outputs.append(output_manifest) + command += ["-o", output_manifest.path] + + outputs += ctx.outputs.outs + command += ["-m", "/".join(base_dirname), ctx.file.src.path] + command += ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.create_directory_flags elif len(ctx.attr.outs) > 1: fail("Only one file can be specified in outs if multiple_outputs is " + "not set.") else: compiled_json = ctx.outputs.outs[0] - outputs += [compiled_json] + outputs.append(compiled_json) command += [ctx.file.src.path, "-o", compiled_json.path] transitive_data = depset(transitive = [dep.data_runfiles.files for dep in ctx.attr.deps] + @@ -282,11 +272,10 @@ def _jsonnet_to_json_impl(ctx): depinfo.transitive_sources.to_list() ) - tools = [ctx.executable.jsonnet] - ctx.actions.run_shell( inputs = compile_inputs + stamp_inputs, - tools = tools, + toolchain = Label("//jsonnet:toolchain_type"), + tools = [ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.compiler], outputs = outputs, mnemonic = "Jsonnet", command = " ".join(command), @@ -300,6 +289,11 @@ def _jsonnet_to_json_impl(ctx): runfiles = ctx.runfiles(files = [out_dir]), )] + return [DefaultInfo( + files = depset([]), + runfiles = ctx.runfiles(files = []), + )] + _EXIT_CODE_COMPARE_COMMAND = """ EXIT_CODE=$? EXPECTED_EXIT_CODE=%d @@ -329,6 +323,10 @@ fi """ _REGEX_DIFF_COMMAND = """ +# Needed due to rust-jsonnet, go-jsonnet and cpp-jsonnet producing different +# output (casing, text etc). +shopt -s nocasematch + GOLDEN_REGEX=$(%s %s) if [[ ! "$OUTPUT" =~ $GOLDEN_REGEX ]]; then echo "FAIL (regex mismatch): %s" @@ -342,12 +340,11 @@ fi def _jsonnet_to_json_test_impl(ctx): """Implementation of the jsonnet_to_json_test rule.""" depinfo = _setup_deps(ctx.attr.deps) - toolchain = _jsonnet_toolchain(ctx) golden_files = [] diff_command = "" if ctx.file.golden: - golden_files += [ctx.file.golden] + golden_files.append(ctx.file.golden) # Note that we only run jsonnet to canonicalize the golden output if the # expected return code is 0, and canonicalize_golden was not explicitly disabled. @@ -356,7 +353,7 @@ def _jsonnet_to_json_test_impl(ctx): # For legacy reasons, we also disable canonicalize_golden for yaml_streams. canonicalize = not (ctx.attr.yaml_stream or not ctx.attr.canonicalize_golden) - dump_golden_cmd = (ctx.executable.jsonnet.short_path if ctx.attr.error == 0 and canonicalize else "/bin/cat") + dump_golden_cmd = (ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.compiler.short_path if ctx.attr.error == 0 and canonicalize else "/bin/cat") if ctx.attr.regex: diff_command = _REGEX_DIFF_COMMAND % ( dump_golden_cmd, @@ -391,7 +388,7 @@ def _jsonnet_to_json_test_impl(ctx): other_args = ctx.attr.extra_args + (["-y"] if ctx.attr.yaml_stream else []) jsonnet_command = " ".join( - ["OUTPUT=$(%s" % ctx.executable.jsonnet.short_path] + + ["OUTPUT=$(%s" % ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.compiler.short_path] + ["-J " + shell.quote(im) for im in _get_import_paths(ctx.label, [ctx.file.src], ctx.attr.imports)] + ["-J " + shell.quote(im) for im in depinfo.imports.to_list()] + other_args + @@ -435,7 +432,7 @@ def _jsonnet_to_json_test_impl(ctx): ), ] if diff_command: - command += [diff_command] + command.append(diff_command) ctx.actions.write( output = ctx.outputs.executable, @@ -450,7 +447,11 @@ def _jsonnet_to_json_test_impl(ctx): ) test_inputs = ( - [ctx.file.src, ctx.executable.jsonnet] + golden_files + + [ + ctx.file.src, + ctx.toolchains["//jsonnet:toolchain_type"].jsonnetinfo.compiler, + ] + + golden_files + transitive_data.to_list() + depinfo.transitive_sources.to_list() + jsonnet_ext_str_files + @@ -473,13 +474,6 @@ _jsonnet_common_attrs = { "imports": attr.string_list( doc = "List of import `-J` flags to be passed to the `jsonnet` compiler.", ), - "jsonnet": attr.label( - doc = "A jsonnet binary", - default = Label("//jsonnet:jsonnet_tool"), - cfg = "exec", - executable = True, - allow_single_file = True, - ), "deps": attr.label_list( doc = "List of targets that are required by the `srcs` Jsonnet files.", providers = [JsonnetLibraryInfo], @@ -640,6 +634,7 @@ jsonnet_to_json = rule( attrs = dict(_jsonnet_compile_attrs.items() + _jsonnet_to_json_attrs.items() + _jsonnet_common_attrs.items()), + toolchains = ["//jsonnet:toolchain_type"], doc = """\ Compiles Jsonnet code to JSON. @@ -774,6 +769,7 @@ jsonnet_to_json_test = rule( attrs = dict(_jsonnet_compile_attrs.items() + _jsonnet_to_json_test_attrs.items() + _jsonnet_common_attrs.items()), + toolchains = ["//jsonnet:toolchain_type"], executable = True, test = True, doc = """\ @@ -895,3 +891,4 @@ def jsonnet_repositories(): strip_prefix = "go-jsonnet-0d78479d37eabd9451892dd02be2470145b4d4fa", urls = ["https://github.com/google/go-jsonnet/archive/0d78479d37eabd9451892dd02be2470145b4d4fa.tar.gz"], ) + diff --git a/jsonnet/toolchain.bzl b/jsonnet/toolchain.bzl new file mode 100644 index 0000000..d2bd3cd --- /dev/null +++ b/jsonnet/toolchain.bzl @@ -0,0 +1,46 @@ +JsonnetToolchainInfo = provider( + doc = "Jsonnet toolchain provider", + fields = { + "compiler": "The file to the Jsonnet compiler", + "create_directory_flags": "The flags to pass when creating a directory.", + "manifest_file_support": ( + "If the Jsonnet compiler supports writing the output filenames to a " + + "manifest file." + ), + }, +) + +def _jsonnet_toolchain_impl(ctx): + toolchain_info = platform_common.ToolchainInfo( + jsonnetinfo = JsonnetToolchainInfo( + compiler = ctx.executable.compiler, + create_directory_flags = ctx.attr.create_directory_flags, + manifest_file_support = ctx.attr.manifest_file_support, + ), + ) + return [toolchain_info] + +jsonnet_toolchain = rule( + implementation = _jsonnet_toolchain_impl, + doc = "The Jsonnet compiler information.", + attrs = { + "compiler": attr.label( + executable = True, + cfg = "exec", + ), + "create_directory_flags": attr.string_list( + mandatory = True, + doc = ( + "The flags passed to the Jsonnet compiler when a directory " + + "must be created." + ), + ), + "manifest_file_support": attr.bool( + mandatory = True, + doc = ( + "If the Jsonnet compiler supports writing the output filenames " + + "to a manifest file." + ), + ), + }, +)