From fa60c0c4204c6c449775b06092c54b25a3a2ebc2 Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Tue, 9 May 2023 12:49:42 -0700 Subject: [PATCH] feat: support custom nodejs toolchains (#1053) --- MODULE.bazel | 18 ++++++++ WORKSPACE | 8 ++-- docs/js_binary.md | 10 +++-- js/private/js_binary.bzl | 30 ++++++++++--- js/private/test/BUILD.bazel | 44 ++++++++++++++++++- js/private/test/node-patches/BUILD.bazel | 15 ++++--- .../test/node-patches_legacy/BUILD.bazel | 15 ++++--- 7 files changed, 110 insertions(+), 30 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 3661fc772..1da2b00a3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -47,7 +47,25 @@ node_dev = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node", dev_dep use_repo(node_dev, "nodejs_toolchains") +use_repo(node_dev, "node18_linux_amd64") +use_repo(node_dev, "node18_darwin_arm64") +use_repo(node_dev, "node18_darwin_amd64") +use_repo(node_dev, "node18_linux_arm64") +use_repo(node_dev, "node18_linux_s390x") +use_repo(node_dev, "node18_linux_ppc64le") +use_repo(node_dev, "node18_windows_amd64") + +use_repo(node_dev, "node16_linux_amd64") +use_repo(node_dev, "node16_darwin_arm64") +use_repo(node_dev, "node16_darwin_amd64") +use_repo(node_dev, "node16_linux_arm64") +use_repo(node_dev, "node16_linux_s390x") +use_repo(node_dev, "node16_linux_ppc64le") +use_repo(node_dev, "node16_windows_amd64") + node_dev.toolchain(node_version = "16.14.2") +node_dev.toolchain(name = "node16", node_version = "16.13.1") +node_dev.toolchain(name = "node18", node_version = "18.13.0") ############################################ # npm dependencies used by examples diff --git a/WORKSPACE b/WORKSPACE index 9949b4805..9e9739ac1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -28,13 +28,13 @@ nodejs_register_toolchains( # Alternate toolchains for testing across versions nodejs_register_toolchains( - name = "node14", - node_version = "14.17.1", + name = "node16", + node_version = "16.13.1", ) nodejs_register_toolchains( - name = "node16", - node_version = "16.13.1", + name = "node18", + node_version = "18.13.0", ) load("@bazel_skylib//lib:unittest.bzl", "register_unittest_toolchains") diff --git a/docs/js_binary.md b/docs/js_binary.md index d1d886e3c..a2e6ffe2e 100644 --- a/docs/js_binary.md +++ b/docs/js_binary.md @@ -24,8 +24,8 @@ js_binary( 
 js_binary(name, chdir, copy_data_to_bin, data, enable_runfiles, entry_point, env,
           expected_exit_code, include_declarations, include_npm, include_npm_linked_packages,
-          include_transitive_sources, log_level, no_copy_to_bin, node_options, patch_node_fs,
-          preserve_symlinks_main, unresolved_symlinks_enabled)
+          include_transitive_sources, log_level, no_copy_to_bin, node_options, node_toolchain,
+          patch_node_fs, preserve_symlinks_main, unresolved_symlinks_enabled)
Execute a program in the Node.js runtime. @@ -85,6 +85,7 @@ This rules requires that Bazel was run with | log_level | Set the logging level.

Log from are written to stderr. They will be supressed on success when running as the tool of a js_run_binary when silent_on_success is True. In that case, they will be shown only on a build failure along with the stdout & stderr of the node tool being run. | String | optional | "error" | | no_copy_to_bin | List of files to not copy to the Bazel output tree when copy_data_to_bin is True.

This is useful for exceptional cases where a copy_to_bin is not possible or not suitable for an input file such as a file in an external repository. In most cases, this option is not needed. See copy_data_to_bin docstring for more info. | List of labels | optional | [] | | node_options | Options to pass to the node invocation on the command line.

https://nodejs.org/api/cli.html

These options are passed directly to the node invocation on the command line. Options passed here will take precendence over options passed via the NODE_OPTIONS environment variable. Options passed here are not added to the NODE_OPTIONS environment variable so will not be automatically picked up by child processes that inherit that enviroment variable. | List of strings | optional | [] | +| node_toolchain | The Node.js toolchain to use for this target.

See https://bazelbuild.github.io/rules_nodejs/Toolchains.html

Typically this is left unset so that Bazel automatically selects the right Node.js toolchain for the target platform. See https://bazel.build/extending/toolchains#toolchain-resolution for more information. | Label | optional | None | | patch_node_fs | Patch the to Node.js fs API (https://nodejs.org/api/fs.html) for this node program to prevent the program from following symlinks out of the execroot, runfiles and the sandbox.

When enabled, js_binary patches the Node.js sync and async fs API functions lstat, readlink, realpath, readdir and opendir so that the node program being run cannot resolve symlinks out of the execroot and the runfiles tree. When in the sandbox, these patches prevent the program being run from resolving symlinks out of the sandbox.

When disabled, node programs can leave the execroot, runfiles and sandbox by following symlinks which can lead to non-hermetic behavior. | Boolean | optional | True | | preserve_symlinks_main | When True, the --preserve-symlinks-main flag is passed to node.

This prevents node from following an ESM entry script out of runfiles and the sandbox. This can happen for .mjs ESM entry points where the fs node patches, which guard the runfiles and sandbox, are not applied. See https://github.com/aspect-build/rules_js/issues/362 for more information. Once #362 is resolved, the default for this attribute can be set to False.

This flag was added in Node.js v10.2.0 (released 2018-05-23). If your node toolchain is configured to use a Node.js version older than this you'll need to set this attribute to False.

See https://nodejs.org/api/cli.html#--preserve-symlinks-main for more information. | Boolean | optional | True | | unresolved_symlinks_enabled | Whether unresolved symlinks are enabled in the current build configuration.

These are enabled with the --experimental_allow_unresolved_symlinks flag.

Typical usage of this rule is via a macro which automatically sets this attribute based on a config_setting rule. | Boolean | optional | False | @@ -97,8 +98,8 @@ This rules requires that Bazel was run with 
 js_test(name, chdir, copy_data_to_bin, data, enable_runfiles, entry_point, env, expected_exit_code,
         include_declarations, include_npm, include_npm_linked_packages, include_transitive_sources,
-        log_level, no_copy_to_bin, node_options, patch_node_fs, preserve_symlinks_main,
-        unresolved_symlinks_enabled)
+        log_level, no_copy_to_bin, node_options, node_toolchain, patch_node_fs,
+        preserve_symlinks_main, unresolved_symlinks_enabled)
Identical to js_binary, but usable under `bazel test`. @@ -143,6 +144,7 @@ the contract between Bazel and a test runner. | log_level | Set the logging level.

Log from are written to stderr. They will be supressed on success when running as the tool of a js_run_binary when silent_on_success is True. In that case, they will be shown only on a build failure along with the stdout & stderr of the node tool being run. | String | optional | "error" | | no_copy_to_bin | List of files to not copy to the Bazel output tree when copy_data_to_bin is True.

This is useful for exceptional cases where a copy_to_bin is not possible or not suitable for an input file such as a file in an external repository. In most cases, this option is not needed. See copy_data_to_bin docstring for more info. | List of labels | optional | [] | | node_options | Options to pass to the node invocation on the command line.

https://nodejs.org/api/cli.html

These options are passed directly to the node invocation on the command line. Options passed here will take precendence over options passed via the NODE_OPTIONS environment variable. Options passed here are not added to the NODE_OPTIONS environment variable so will not be automatically picked up by child processes that inherit that enviroment variable. | List of strings | optional | [] | +| node_toolchain | The Node.js toolchain to use for this target.

See https://bazelbuild.github.io/rules_nodejs/Toolchains.html

Typically this is left unset so that Bazel automatically selects the right Node.js toolchain for the target platform. See https://bazel.build/extending/toolchains#toolchain-resolution for more information. | Label | optional | None | | patch_node_fs | Patch the to Node.js fs API (https://nodejs.org/api/fs.html) for this node program to prevent the program from following symlinks out of the execroot, runfiles and the sandbox.

When enabled, js_binary patches the Node.js sync and async fs API functions lstat, readlink, realpath, readdir and opendir so that the node program being run cannot resolve symlinks out of the execroot and the runfiles tree. When in the sandbox, these patches prevent the program being run from resolving symlinks out of the sandbox.

When disabled, node programs can leave the execroot, runfiles and sandbox by following symlinks which can lead to non-hermetic behavior. | Boolean | optional | True | | preserve_symlinks_main | When True, the --preserve-symlinks-main flag is passed to node.

This prevents node from following an ESM entry script out of runfiles and the sandbox. This can happen for .mjs ESM entry points where the fs node patches, which guard the runfiles and sandbox, are not applied. See https://github.com/aspect-build/rules_js/issues/362 for more information. Once #362 is resolved, the default for this attribute can be set to False.

This flag was added in Node.js v10.2.0 (released 2018-05-23). If your node toolchain is configured to use a Node.js version older than this you'll need to set this attribute to False.

See https://nodejs.org/api/cli.html#--preserve-symlinks-main for more information. | Boolean | optional | True | | unresolved_symlinks_enabled | Whether unresolved symlinks are enabled in the current build configuration.

These are enabled with the --experimental_allow_unresolved_symlinks flag.

Typical usage of this rule is via a macro which automatically sets this attribute based on a config_setting rule. | Boolean | optional | False | diff --git a/js/private/js_binary.bzl b/js/private/js_binary.bzl index 07d9cdf4c..d9b679670 100644 --- a/js/private/js_binary.bzl +++ b/js/private/js_binary.bzl @@ -247,6 +247,16 @@ _ATTRS = { # TODO(2.0): make this mandatory so that downstream binary rules that inherit these attributes are required to set it mandatory = False, ), + "node_toolchain": attr.label( + doc = """The Node.js toolchain to use for this target. + + See https://bazelbuild.github.io/rules_nodejs/Toolchains.html + + Typically this is left unset so that Bazel automatically selects the right Node.js toolchain + for the target platform. See https://bazel.build/extending/toolchains#toolchain-resolution + for more information. + """, + ), "_launcher_template": attr.label( default = Label("//js/private:js_binary.sh.tpl"), allow_single_file = True, @@ -309,7 +319,7 @@ def _consistent_label_str(workspace_name, label): label.name, ) -def _bash_launcher(ctx, entry_point_path, log_prefix_rule_set, log_prefix_rule, fixed_args, fixed_env, is_windows, use_legacy_node_patches): +def _bash_launcher(ctx, node_toolchain, entry_point_path, log_prefix_rule_set, log_prefix_rule, fixed_args, fixed_env, is_windows, use_legacy_node_patches): envs = [] for (key, value) in dicts.add(fixed_env, ctx.attr.env).items(): envs.append(_ENV_SET.format( @@ -400,7 +410,7 @@ def _bash_launcher(ctx, entry_point_path, log_prefix_rule_set, log_prefix_rule, npm_path = "" if ctx.attr.include_npm: - npm_path = _target_tool_short_path(ctx.workspace_name, ctx.toolchains["@rules_nodejs//nodejs:toolchain_type"].nodeinfo.npm_path) + npm_path = _target_tool_short_path(ctx.workspace_name, node_toolchain.nodeinfo.npm_path) if is_windows: npm_wrapper = ctx.actions.declare_file("%s_node_bin/npm.bat" % ctx.label.name) ctx.actions.expand_template( @@ -419,7 +429,7 @@ def _bash_launcher(ctx, entry_point_path, log_prefix_rule_set, log_prefix_rule, ) toolchain_files.append(npm_wrapper) - node_path = _target_tool_short_path(ctx.workspace_name, ctx.toolchains["@rules_nodejs//nodejs:toolchain_type"].nodeinfo.target_tool_path) + node_path = _target_tool_short_path(ctx.workspace_name, node_toolchain.nodeinfo.target_tool_path) launcher_subst = { "{{target_label}}": _consistent_label_str(ctx.workspace_name, ctx.label), @@ -457,10 +467,15 @@ def _create_launcher(ctx, log_prefix_rule_set, log_prefix_rule, fixed_args = [], unresolved_symlinks_enabled = ctx.attr.unresolved_symlinks_enabled use_legacy_node_patches = not is_bazel_6 or not unresolved_symlinks_enabled + if ctx.attr.node_toolchain: + node_toolchain = ctx.attr.node_toolchain[platform_common.ToolchainInfo] + else: + node_toolchain = ctx.toolchains["@rules_nodejs//nodejs:toolchain_type"] + if is_windows and not ctx.attr.enable_runfiles: fail("need --enable_runfiles on Windows for to support rules_js") - if ctx.attr.include_npm and not hasattr(ctx.toolchains["@rules_nodejs//nodejs:toolchain_type"].nodeinfo, "npm_files"): + if ctx.attr.include_npm and not hasattr(node_toolchain.nodeinfo, "npm_files"): fail("include_npm requires a minimum @rules_nodejs version of 5.7.0") if DirectoryPathInfo in ctx.attr.entry_point: @@ -475,7 +490,7 @@ def _create_launcher(ctx, log_prefix_rule_set, log_prefix_rule, fixed_args = [], entry_point = ctx.files.entry_point[0] entry_point_path = entry_point.short_path - bash_launcher, toolchain_files = _bash_launcher(ctx, entry_point_path, log_prefix_rule_set, log_prefix_rule, fixed_args, fixed_env, is_windows, use_legacy_node_patches) + bash_launcher, toolchain_files = _bash_launcher(ctx, node_toolchain, entry_point_path, log_prefix_rule_set, log_prefix_rule, fixed_args, fixed_env, is_windows, use_legacy_node_patches) launcher = create_windows_native_launcher_script(ctx, bash_launcher) if is_windows else bash_launcher launcher_files = [bash_launcher] + toolchain_files @@ -483,9 +498,10 @@ def _create_launcher(ctx, log_prefix_rule_set, log_prefix_rule, fixed_args = [], launcher_files.extend(ctx.files._node_patches_legacy_files + [ctx.file._node_patches_legacy]) else: launcher_files.extend(ctx.files._node_patches_files + [ctx.file._node_patches]) - launcher_files.extend(ctx.toolchains["@rules_nodejs//nodejs:toolchain_type"].nodeinfo.tool_files) + + launcher_files.extend(node_toolchain.nodeinfo.tool_files) if ctx.attr.include_npm: - launcher_files.extend(ctx.toolchains["@rules_nodejs//nodejs:toolchain_type"].nodeinfo.npm_files) + launcher_files.extend(node_toolchain.nodeinfo.npm_files) runfiles = gather_runfiles( ctx = ctx, diff --git a/js/private/test/BUILD.bazel b/js/private/test/BUILD.bazel index 2087fe400..58be75f23 100644 --- a/js/private/test/BUILD.bazel +++ b/js/private/test/BUILD.bazel @@ -1,7 +1,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files") load("@aspect_bazel_lib_host//:defs.bzl", "host") -load("//js:defs.bzl", "js_binary", "js_library") +load("//js:defs.bzl", "js_binary", "js_library", "js_test") load(":js_library_test.bzl", "js_library_test_suite") #################################################################################################### @@ -59,3 +59,45 @@ genrule( cmd = "echo '{\"answer\": 42}' > $@", visibility = ["//js/private/test:__subpackages__"], ) + +write_file( + name = "binary_version", + out = "binary_version.js", + content = [""" +if (parseInt(process.version.slice(1)) !== parseInt(process.argv[2])) { + throw new Error(`Expected node version ${parseInt(process.version)}.* but got ${parseInt(process.argv[2])}`) +} +"""], +) + +js_test( + name = "main_default_toolchain", + args = ["16"], + entry_point = "binary_version.js", +) + +js_test( + name = "main_toolchain_16", + args = ["16"], + entry_point = "binary_version.js", + # using the select statement will download toolchains for all three platforms + # you can also just provide an individual toolchain if you don't want to download them all + node_toolchain = select({ + "@bazel_tools//src/conditions:linux_x86_64": "@node16_linux_amd64//:node_toolchain", + "@bazel_tools//src/conditions:darwin": "@node16_darwin_amd64//:node_toolchain", + "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_toolchain", + }), +) + +js_test( + name = "main_toolchain_18", + args = ["18"], + entry_point = "binary_version.js", + # using the select statement will download toolchains for all three platforms + # you can also just provide an individual toolchain if you don't want to download them all + node_toolchain = select({ + "@bazel_tools//src/conditions:linux_x86_64": "@node18_linux_amd64//:node_toolchain", + "@bazel_tools//src/conditions:darwin": "@node18_darwin_amd64//:node_toolchain", + "@bazel_tools//src/conditions:windows": "@node18_windows_amd64//:node_toolchain", + }), +) diff --git a/js/private/test/node-patches/BUILD.bazel b/js/private/test/node-patches/BUILD.bazel index a505d6ea4..61c9a07b3 100644 --- a/js/private/test/node-patches/BUILD.bazel +++ b/js/private/test/node-patches/BUILD.bazel @@ -12,20 +12,20 @@ TESTS = [ # Multiple node toolchains for testing across versions TOOLCHAINS_NAMES = [ - "node14", "node16", + "node18", ] TOOLCHAINS_VERSIONS = [ select({ - "@bazel_tools//src/conditions:linux_x86_64": "@node14_linux_amd64//:node_toolchain", - "@bazel_tools//src/conditions:darwin": "@node14_darwin_amd64//:node_toolchain", - "@bazel_tools//src/conditions:windows": "@node14_windows_amd64//:node_toolchain", + "@bazel_tools//src/conditions:linux_x86_64": "@node16_linux_amd64//:node_toolchain", + "@bazel_tools//src/conditions:darwin": "@node16_darwin_amd64//:node_toolchain", + "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_toolchain", }), select({ - "@bazel_tools//src/conditions:linux_x86_64": "@node14_linux_amd64//:node_toolchain", - "@bazel_tools//src/conditions:darwin": "@node14_darwin_amd64//:node_toolchain", - "@bazel_tools//src/conditions:windows": "@node14_windows_amd64//:node_toolchain", + "@bazel_tools//src/conditions:linux_x86_64": "@node18_linux_amd64//:node_toolchain", + "@bazel_tools//src/conditions:darwin": "@node18_darwin_amd64//:node_toolchain", + "@bazel_tools//src/conditions:windows": "@node18_windows_amd64//:node_toolchain", }), ] @@ -74,6 +74,7 @@ TOOLCHAINS_VERSIONS = [ "//js/private/test/node-patches:spawn_patch_depth.sh", ], entry_point = "spawn.js", + node_toolchain = toolchain, patch_node_fs = True, ) for toolchain_name, toolchain in zip( diff --git a/js/private/test/node-patches_legacy/BUILD.bazel b/js/private/test/node-patches_legacy/BUILD.bazel index 90df181ce..1a8d57da6 100644 --- a/js/private/test/node-patches_legacy/BUILD.bazel +++ b/js/private/test/node-patches_legacy/BUILD.bazel @@ -12,20 +12,20 @@ TESTS = [ # Multiple node toolchains for testing across versions TOOLCHAINS_NAMES = [ - "node14", "node16", + "node18", ] TOOLCHAINS_VERSIONS = [ select({ - "@bazel_tools//src/conditions:linux_x86_64": "@node14_linux_amd64//:node_toolchain", - "@bazel_tools//src/conditions:darwin": "@node14_darwin_amd64//:node_toolchain", - "@bazel_tools//src/conditions:windows": "@node14_windows_amd64//:node_toolchain", + "@bazel_tools//src/conditions:linux_x86_64": "@node16_linux_amd64//:node_toolchain", + "@bazel_tools//src/conditions:darwin": "@node16_darwin_amd64//:node_toolchain", + "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_toolchain", }), select({ - "@bazel_tools//src/conditions:linux_x86_64": "@node14_linux_amd64//:node_toolchain", - "@bazel_tools//src/conditions:darwin": "@node14_darwin_amd64//:node_toolchain", - "@bazel_tools//src/conditions:windows": "@node14_windows_amd64//:node_toolchain", + "@bazel_tools//src/conditions:linux_x86_64": "@node18_linux_amd64//:node_toolchain", + "@bazel_tools//src/conditions:darwin": "@node18_darwin_amd64//:node_toolchain", + "@bazel_tools//src/conditions:windows": "@node18_windows_amd64//:node_toolchain", }), ] @@ -74,6 +74,7 @@ TOOLCHAINS_VERSIONS = [ "//js/private/test/node-patches_legacy:spawn_patch_depth.sh", ], entry_point = "spawn.js", + node_toolchain = toolchain, patch_node_fs = True, ) for toolchain_name, toolchain in zip(