Add docs and examples of complicated build scripts (#2635)

Fixes #2626

.bazelci/presubmit.yml

@@ -1,5 +1,5 @@
 ---
-minimum_bazel_version: &minimum_bazel_version "6.3.0"
+minimum_bazel_version: &minimum_bazel_version "6.4.0"
 aspects_flags: &aspects_flags
   - "--config=rustfmt"
   - "--config=clippy"
@@ -680,7 +680,6 @@ tasks:
     bazel: *minimum_bazel_version
     platform: ubuntu2004
     working_directory: examples/bzlmod/hello_world
-    test_flags: *bzlmod_flags
     shell_commands:
       - "rm MODULE.bazel.lock"
     run_targets:

.bazelrc

@@ -57,6 +57,9 @@ build --incompatible_disallow_empty_glob=true
 # https://github.com/bazelbuild/bazel/issues/12821
 build --nolegacy_external_runfiles
 
+# Required for cargo_build_script support before Bazel 7
+build --incompatible_merge_fixed_and_default_shell_env
+
 ###############################################################################
 ## Bzlmod
 ###############################################################################

cargo/private/cargo_build_script.bzl

@@ -315,6 +315,8 @@ def _cargo_build_script_impl(ctx):
         progress_message = "Running Cargo build script {}".format(pkg_name),
         env = env,
         toolchain = None,
+        # Set use_default_shell_env so that $PATH is set, as tools like cmake may want to probe $PATH for helper tools.
+        use_default_shell_env = True,
     )
 
     return [

crate_universe/docs.bzl

@@ -248,6 +248,36 @@ convenient accessors to larger sections of the dependency graph.
 - [all_crate_deps](#all_crate_deps)
 - [crate_repositories](#crate_repositories)
 
+## Building crates with complicated dependencies
+
+Some crates have build.rs scripts which are complicated to run. Typically these build C++ (or other languages), or attempt to find pre-installed libraries on the build machine.
+
+There are a few approaches to making sure these run:
+
+### Some things work without intervention
+
+Some build scripts will happily run without any support needed.
+
+rules_rust already supplies a configured C++ toolchain as input to build script execution, and sets variables like `CC`, `CXX`, `LD`, `LDFLAGS`, etc as needed. Many crates which invoke a compiler with the default environment, or forward these env vars, will Just Work (e.g. if using [`cc-rs`][cc-rs]).
+
+rules_rust is open to PRs which make build scripts more likely to work by default with intervention assuming they're broadly applicable (e.g. setting extra widely-known env vars is probably fine, wiring up additional toolchains like `cmake` that aren't present by default for most Bazel users probably isn't).
+
+### Supplying extra tools to build
+
+Some build scripts can be made to work by pulling in some extra files and making them available to the build script.
+
+Commonly this is done by passing the file to the `build_script_data` annotation for the crate, and using `build_script_env` to tell the build script where the file is. That env var may often use `$(execroot)` to get the path to the label, or `$${pwd}/` as a prefix if the path given is relative to the execroot (as will frequently happen when using a toolchain).A
+
+There is an example of this in the "complicated dependencies" section of https://github.com/bazelbuild/rules_rust/blob/main/examples/crate_universe/WORKSPACE.bazel which builds libz-ng-sys.
+
+### Building with Bazel and supplying via an override
+
+Some build scripts have hooks to allow replacing parts that are complicated to build with output prepared by Bazel.
+
+We can use those hooks by specifying paths (generally using the `build_script_data` and `build_script_env` annotations) and pointing them at labels which Bazel will then build. These env vars may often use `$(execroot)` to get the path to the label, or `$${pwd}/` as a prefix if the path given is relative to the execroot (as will frequently happen when using a toolchain).
+
+There is an example of this in the "complicated dependencies" section of https://github.com/bazelbuild/rules_rust/blob/main/examples/crate_universe/WORKSPACE.bazel which builds boring-sys.
+
 ---
 
 ---
@@ -256,6 +286,7 @@ convenient accessors to larger sections of the dependency graph.
 [cc]: https://docs.bazel.build/versions/main/be/c-cpp.html
 [proto]: https://rules-proto-grpc.com/en/latest/lang/rust.html
 [ra]: https://rust-analyzer.github.io/
+[cc-rs]: https://github.com/rust-lang/cc-rs
 """
 
 load(

crate_universe/private/crate.bzl

@@ -1,4 +1,4 @@
-"""Macros used for represeting crates or annotations for existing crates"""
+"""Macros used for representing crates or annotations for existing crates"""
 
 load(":common_utils.bzl", "parse_alias_rule")
 

docs/crate_universe.md

@@ -250,6 +250,36 @@ convenient accessors to larger sections of the dependency graph.
 - [all_crate_deps](#all_crate_deps)
 - [crate_repositories](#crate_repositories)
 
+## Building crates with complicated dependencies
+
+Some crates have build.rs scripts which are complicated to run. Typically these build C++ (or other languages), or attempt to find pre-installed libraries on the build machine.
+
+There are a few approaches to making sure these run:
+
+### Some things work without intervention
+
+Some build scripts will happily run without any support needed.
+
+rules_rust already supplies a configured C++ toolchain as input to build script execution, and sets variables like `CC`, `CXX`, `LD`, `LDFLAGS`, etc as needed. Many crates which invoke a compiler with the default environment, or forward these env vars, will Just Work (e.g. if using [`cc-rs`][cc-rs]).
+
+rules_rust is open to PRs which make build scripts more likely to work by default with intervention assuming they're broadly applicable (e.g. setting extra widely-known env vars is probably fine, wiring up additional toolchains like `cmake` that aren't present by default for most Bazel users probably isn't).
+
+### Supplying extra tools to build
+
+Some build scripts can be made to work by pulling in some extra files and making them available to the build script.
+
+Commonly this is done by passing the file to the `build_script_data` annotation for the crate, and using `build_script_env` to tell the build script where the file is. That env var may often use `$(execroot)` to get the path to the label, or `$${pwd}/` as a prefix if the path given is relative to the execroot (as will frequently happen when using a toolchain).A
+
+There is an example of this in the "complicated dependencies" section of https://github.com/bazelbuild/rules_rust/blob/main/examples/crate_universe/WORKSPACE.bazel which builds libz-ng-sys.
+
+### Building with Bazel and supplying via an override
+
+Some build scripts have hooks to allow replacing parts that are complicated to build with output prepared by Bazel.
+
+We can use those hooks by specifying paths (generally using the `build_script_data` and `build_script_env` annotations) and pointing them at labels which Bazel will then build. These env vars may often use `$(execroot)` to get the path to the label, or `$${pwd}/` as a prefix if the path given is relative to the execroot (as will frequently happen when using a toolchain).
+
+There is an example of this in the "complicated dependencies" section of https://github.com/bazelbuild/rules_rust/blob/main/examples/crate_universe/WORKSPACE.bazel which builds boring-sys.
+
 ---
 
 ---
@@ -258,6 +288,7 @@ convenient accessors to larger sections of the dependency graph.
 [cc]: https://docs.bazel.build/versions/main/be/c-cpp.html
 [proto]: https://rules-proto-grpc.com/en/latest/lang/rust.html
 [ra]: https://rust-analyzer.github.io/
+[cc-rs]: https://github.com/rust-lang/cc-rs
 
 
 <a id="crates_repository"></a>

examples/bzlmod/all_crate_deps/.bazelrc

@@ -7,3 +7,6 @@ build --experimental_enable_bzlmod
 
 # This isn't currently the defaut in Bazel, but we enable it to test we'll be ready if/when it flips.
 build --incompatible_disallow_empty_glob
+
+# Required for cargo_build_script support before Bazel 7
+build --incompatible_merge_fixed_and_default_shell_env

examples/bzlmod/cross_compile/.bazelrc

@@ -2,3 +2,6 @@ build --experimental_enable_bzlmod
 
 # This isn't currently the defaut in Bazel, but we enable it to test we'll be ready if/when it flips.
 build --incompatible_disallow_empty_glob
+
+# Required for cargo_build_script support before Bazel 7
+build --incompatible_merge_fixed_and_default_shell_env

examples/bzlmod/hello_world/.bazelrc

@@ -7,3 +7,6 @@ build --experimental_enable_bzlmod
 
 # This isn't currently the defaut in Bazel, but we enable it to test we'll be ready if/when it flips.
 build --incompatible_disallow_empty_glob
+
+# Required for cargo_build_script support before Bazel 7
+build --incompatible_merge_fixed_and_default_shell_env

examples/bzlmod/hello_world_no_cargo/.bazelrc

@@ -1,3 +1,6 @@
 common --experimental_enable_bzlmod
 common --noenable_workspace
 common --enable_runfiles
+
+# Required for cargo_build_script support before Bazel 7
+build --incompatible_merge_fixed_and_default_shell_env

examples/crate_universe/.bazelrc

@@ -25,6 +25,9 @@ common --noenable_bzlmod
 # This isn't currently the defaut in Bazel, but we enable it to test we'll be ready if/when it flips.
 build --incompatible_disallow_empty_glob
 
+# Required for cargo_build_script support before Bazel 7
+build --incompatible_merge_fixed_and_default_shell_env
+
 # This import should always be last to allow users to override
 # settings for local development.
 try-import %workspace%/user.bazelrc

examples/crate_universe/WORKSPACE.bazel

@@ -613,6 +613,111 @@ load(
 
 crates_vendor_packages_repositories()
 
+###############################################################################
+# C O M P L I C A T E D   D E P E N D E N C I E S
+###############################################################################
+
+http_archive(
+    name = "rules_foreign_cc",
+    sha256 = "1eee5d216a3cec7a4c731f71ed731ac353290b1db61ab68b79440655bcb9acaa",
+    strip_prefix = "rules_foreign_cc-7ce62009557d73da9aa0d2a1ca7eded49078b3cf",
+    # Pulls in https://github.com/bazelbuild/rules_foreign_cc/pull/1163 and https://github.com/bazelbuild/rules_foreign_cc/pull/1199 which aren't currently in a release.
+    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/7ce62009557d73da9aa0d2a1ca7eded49078b3cf.tar.gz",
+)
+
+load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
+
+rules_foreign_cc_dependencies()
+
+http_archive(
+    name = "aspect_bazel_lib",
+    sha256 = "f5ea76682b209cc0bd90d0f5a3b26d2f7a6a2885f0c5f615e72913f4805dbb0d",
+    strip_prefix = "bazel-lib-2.5.0",
+    url = "https://github.com/aspect-build/bazel-lib/releases/download/v2.5.0/bazel-lib-v2.5.0.tar.gz",
+)
+
+load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains")
+
+aspect_bazel_lib_dependencies()
+
+aspect_bazel_lib_register_toolchains()
+
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+git_repository(
+    name = "boringssl",
+    commit = "44b3df6f03d85c901767250329c571db405122d5",
+    patch_args = ["-p1"],
+    patches = [
+        "//complicated_dependencies:boringssl-filegroup.patch",
+        # On the macOS bazelci builders, there's a system-installed openssl, and that takes priority over -isystem flags, which is what cc_library.includes uses.
+        # This forces our local system-includes to be chosen with higher priority, which avoids conflicts.
+        "//complicated_dependencies:boringssl-system-includes.patch",
+    ],
+    remote = "https://github.com/google/boringssl.git",
+)
+
+crates_repository(
+    name = "complicated_dependencies",
+    annotations = {
+        # boringssl natively builds with Bazel, but boring-sys tries to build it with cmake.
+        # We could pass boring-sys a cmake binary it can invoke, but the boring-sys build script wouldn't get to exploit the caching and parallelism of full Bazel builds of boringssl.
+        # Instead, we use the build script env var hooks to point the build script at the output of Bazel building boringssl.
+        "boring-sys": [
+            crate.annotation(
+                build_script_data = [
+                    "@//complicated_dependencies:boringssl_gen_dir",
+                    "@boringssl//:headers",
+                ],
+                build_script_env = {
+                    # Ideally this would use an execpath macro, but we need to point at a directory and that's fiddly. We could probably copy the directory somewhere and point at that... For now, this works.
+                    "BORING_BSSL_INCLUDE_PATH": "$${pwd}/external/boringssl/src/include",
+                    "BORING_BSSL_PATH": "$(execpath @//complicated_dependencies:boringssl_gen_dir)",
+                },
+                compile_data = [
+                    "@//complicated_dependencies:boringssl_gen_dir",
+                    "@boringssl//:headers",
+                ],
+            ),
+        ],
+        # zlib-ng-sys's build script invokes cmake, so we need to make cmake available.
+        # Fortunately, rules_foreign_cc has a cmake toolchain we can use.
+        "libz-ng-sys": [crate.annotation(
+            # Setting build_script_data makes the files available on disk when the rule runs.
+            build_script_data = ["@rules_foreign_cc//toolchains:current_cmake_toolchain"],
+            build_script_env = {
+                # The toolchain supplies a value of $(CMAKE) which is an execroot-relative path, so we need to prefix it with $${pwd}/ because build scripts don't typically run in the execroot unlike most bazel rules, for improved compatibility with Cargo.
+                "CMAKE": "$${pwd}/$(CMAKE)",
+            },
+            # Setting build_script_toolchains makes makefile variable substitution work so that we can reference $(CNAME) in attributes.
+            build_script_toolchains = ["@rules_foreign_cc//toolchains:current_cmake_toolchain"],
+        )],
+    },
+    cargo_lockfile = "//complicated_dependencies:Cargo.Bazel.lock",
+    # `generator` is not necessary in official releases.
+    # See load satement for `cargo_bazel_bootstrap`.
+    generator = "@cargo_bazel_bootstrap//:cargo-bazel",
+    lockfile = "//complicated_dependencies:cargo-bazel-lock.json",
+    packages = {
+        "boring": crate.spec(
+            version = "3.0.4",
+        ),
+        "libz-ng-sys": crate.spec(
+            version = "=1.1.15",
+        ),
+    },
+    splicing_config = splicing_config(
+        resolver_version = "2",
+    ),
+)
+
+load(
+    "@complicated_dependencies//:defs.bzl",
+    complicated_dependencies_crate_repositories = "crate_repositories",
+)
+
+complicated_dependencies_crate_repositories()
+
 ###############################################################################
 
 # Used for Bazel CI

examples/crate_universe/complicated_dependencies/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+
+# This target lays out the output needed from boringssl in the directory structure needed by the boring-sys build script.
+copy_to_directory(
+    name = "boringssl_gen_dir",
+    srcs = [
+        "@boringssl//:crypto",
+        "@boringssl//:ssl",
+    ],
+    out = "boringssl_gen_dir_out",
+    include_external_repositories = ["*"],
+    replace_prefixes = {
+        "libcrypto.a": "build/libcrypto.a",
+        "libssl.a": "build/libssl.a",
+    },
+    visibility = ["//visibility:public"],
+)
+
+build_test(
+    name = "build_test",
+    targets = [
+        "@complicated_dependencies//:boring",
+        "@complicated_dependencies//:libz-ng-sys",
+    ],
+)