🍦 Bring CUDA setup in sync with recent nixpkgs (#242)

This is a subtle bug. The CUDA_TOOLKIT changed on the nix side and no
longer contained the actual cuda driver. This lead to only the stubs
being added to the runpath of CUDA executables. The stubs are only meant
to be used at link time but not functional during runtime.

This change prepends the runpath with the correct CUDA driver.

Also removes the now obsolete `LL_CUDA_RUNTIME` flag.

docs/reference/toolchain.md

@@ -6,14 +6,14 @@ This file declares the `ll_toolchain` rule.
 
 ## `ll_toolchain`
 
-<pre><code>ll_toolchain(<a href="#ll_toolchain-name">name</a>, <a href="#ll_toolchain-LL_AMD_INCLUDES">LL_AMD_INCLUDES</a>, <a href="#ll_toolchain-LL_AMD_LIBRARIES">LL_AMD_LIBRARIES</a>, <a href="#ll_toolchain-LL_CFLAGS">LL_CFLAGS</a>, <a href="#ll_toolchain-LL_CUDA_DRIVER">LL_CUDA_DRIVER</a>, <a href="#ll_toolchain-LL_CUDA_RUNTIME">LL_CUDA_RUNTIME</a>,
-             <a href="#ll_toolchain-LL_CUDA_TOOLKIT">LL_CUDA_TOOLKIT</a>, <a href="#ll_toolchain-LL_DYNAMIC_LINKER">LL_DYNAMIC_LINKER</a>, <a href="#ll_toolchain-LL_LDFLAGS">LL_LDFLAGS</a>, <a href="#ll_toolchain-address_sanitizer">address_sanitizer</a>, <a href="#ll_toolchain-archiver">archiver</a>,
-             <a href="#ll_toolchain-bitcode_linker">bitcode_linker</a>, <a href="#ll_toolchain-builtin_includes">builtin_includes</a>, <a href="#ll_toolchain-c_driver">c_driver</a>, <a href="#ll_toolchain-clang_tidy">clang_tidy</a>, <a href="#ll_toolchain-clang_tidy_runner">clang_tidy_runner</a>,
-             <a href="#ll_toolchain-compiler_runtime">compiler_runtime</a>, <a href="#ll_toolchain-cov">cov</a>, <a href="#ll_toolchain-cpp_abihdrs">cpp_abihdrs</a>, <a href="#ll_toolchain-cpp_abilib">cpp_abilib</a>, <a href="#ll_toolchain-cpp_driver">cpp_driver</a>, <a href="#ll_toolchain-cpp_stdhdrs">cpp_stdhdrs</a>, <a href="#ll_toolchain-cpp_stdlib">cpp_stdlib</a>,
-             <a href="#ll_toolchain-hip_libraries">hip_libraries</a>, <a href="#ll_toolchain-hip_runtime">hip_runtime</a>, <a href="#ll_toolchain-leak_sanitizer">leak_sanitizer</a>, <a href="#ll_toolchain-linker">linker</a>, <a href="#ll_toolchain-linker_wrapper">linker_wrapper</a>, <a href="#ll_toolchain-llvm_project_deps">llvm_project_deps</a>,
-             <a href="#ll_toolchain-machine_code_tool">machine_code_tool</a>, <a href="#ll_toolchain-memory_sanitizer">memory_sanitizer</a>, <a href="#ll_toolchain-objcopy">objcopy</a>, <a href="#ll_toolchain-offload_bundler">offload_bundler</a>, <a href="#ll_toolchain-offload_packager">offload_packager</a>, <a href="#ll_toolchain-opt">opt</a>,
-             <a href="#ll_toolchain-profdata">profdata</a>, <a href="#ll_toolchain-profile">profile</a>, <a href="#ll_toolchain-rocm_device_libs">rocm_device_libs</a>, <a href="#ll_toolchain-symbolizer">symbolizer</a>, <a href="#ll_toolchain-thread_sanitizer">thread_sanitizer</a>,
-             <a href="#ll_toolchain-undefined_behavior_sanitizer">undefined_behavior_sanitizer</a>, <a href="#ll_toolchain-unwind_library">unwind_library</a>)</code></pre>
+<pre><code>ll_toolchain(<a href="#ll_toolchain-name">name</a>, <a href="#ll_toolchain-LL_AMD_INCLUDES">LL_AMD_INCLUDES</a>, <a href="#ll_toolchain-LL_AMD_LIBRARIES">LL_AMD_LIBRARIES</a>, <a href="#ll_toolchain-LL_CFLAGS">LL_CFLAGS</a>, <a href="#ll_toolchain-LL_CUDA_DRIVER">LL_CUDA_DRIVER</a>, <a href="#ll_toolchain-LL_CUDA_TOOLKIT">LL_CUDA_TOOLKIT</a>,
+             <a href="#ll_toolchain-LL_DYNAMIC_LINKER">LL_DYNAMIC_LINKER</a>, <a href="#ll_toolchain-LL_LDFLAGS">LL_LDFLAGS</a>, <a href="#ll_toolchain-address_sanitizer">address_sanitizer</a>, <a href="#ll_toolchain-archiver">archiver</a>, <a href="#ll_toolchain-bitcode_linker">bitcode_linker</a>,
+             <a href="#ll_toolchain-builtin_includes">builtin_includes</a>, <a href="#ll_toolchain-c_driver">c_driver</a>, <a href="#ll_toolchain-clang_tidy">clang_tidy</a>, <a href="#ll_toolchain-clang_tidy_runner">clang_tidy_runner</a>, <a href="#ll_toolchain-compiler_runtime">compiler_runtime</a>, <a href="#ll_toolchain-cov">cov</a>,
+             <a href="#ll_toolchain-cpp_abihdrs">cpp_abihdrs</a>, <a href="#ll_toolchain-cpp_abilib">cpp_abilib</a>, <a href="#ll_toolchain-cpp_driver">cpp_driver</a>, <a href="#ll_toolchain-cpp_stdhdrs">cpp_stdhdrs</a>, <a href="#ll_toolchain-cpp_stdlib">cpp_stdlib</a>, <a href="#ll_toolchain-hip_libraries">hip_libraries</a>, <a href="#ll_toolchain-hip_runtime">hip_runtime</a>,
+             <a href="#ll_toolchain-leak_sanitizer">leak_sanitizer</a>, <a href="#ll_toolchain-linker">linker</a>, <a href="#ll_toolchain-linker_wrapper">linker_wrapper</a>, <a href="#ll_toolchain-llvm_project_deps">llvm_project_deps</a>, <a href="#ll_toolchain-machine_code_tool">machine_code_tool</a>,
+             <a href="#ll_toolchain-memory_sanitizer">memory_sanitizer</a>, <a href="#ll_toolchain-objcopy">objcopy</a>, <a href="#ll_toolchain-offload_bundler">offload_bundler</a>, <a href="#ll_toolchain-offload_packager">offload_packager</a>, <a href="#ll_toolchain-opt">opt</a>, <a href="#ll_toolchain-profdata">profdata</a>, <a href="#ll_toolchain-profile">profile</a>,
+             <a href="#ll_toolchain-rocm_device_libs">rocm_device_libs</a>, <a href="#ll_toolchain-symbolizer">symbolizer</a>, <a href="#ll_toolchain-thread_sanitizer">thread_sanitizer</a>, <a href="#ll_toolchain-undefined_behavior_sanitizer">undefined_behavior_sanitizer</a>,
+             <a href="#ll_toolchain-unwind_library">unwind_library</a>)</code></pre>
 
 `attributes`
 
@@ -24,7 +24,6 @@ This file declares the `ll_toolchain` rule.
 | <a id="ll_toolchain-LL_AMD_LIBRARIES"></a>`LL_AMD_LIBRARIES` | <code><a href="https://bazel.build/concepts/labels">Label</a></code>, optional, defaults to <code>None</code>.<br><br> Link search paths for dependencies making use of AMD toolchains.<br><br>Affects the `hip_amdgpu` toolchain.   |
 | <a id="ll_toolchain-LL_CFLAGS"></a>`LL_CFLAGS` | <code><a href="https://bazel.build/concepts/labels">Label</a></code>, optional, defaults to <code>None</code>.<br><br> Arbitrary flags added to all compile actions.   |
 | <a id="ll_toolchain-LL_CUDA_DRIVER"></a>`LL_CUDA_DRIVER` | <code><a href="https://bazel.build/concepts/labels">Label</a></code>, optional, defaults to <code>None</code>.<br><br> The path to the CUDA driver.<br><br>Affects the `cuda_nvptx` and `hip_nvptx` toolchains.   |
-| <a id="ll_toolchain-LL_CUDA_RUNTIME"></a>`LL_CUDA_RUNTIME` | <code><a href="https://bazel.build/concepts/labels">Label</a></code>, optional, defaults to <code>None</code>.<br><br> The path to the CUDA runtime.<br><br>Affects the `cuda_nvptx` and `hip_nvptx` toolchains.   |
 | <a id="ll_toolchain-LL_CUDA_TOOLKIT"></a>`LL_CUDA_TOOLKIT` | <code><a href="https://bazel.build/concepts/labels">Label</a></code>, optional, defaults to <code>None</code>.<br><br> The path to the CUDA toolkit.<br><br>Affects the `cuda_nvptx` and `hip_nvptx` toolchains.   |
 | <a id="ll_toolchain-LL_DYNAMIC_LINKER"></a>`LL_DYNAMIC_LINKER` | <code><a href="https://bazel.build/concepts/labels">Label</a></code>, optional, defaults to <code>None</code>.<br><br> The linker from the glibc we compile and link against.   |
 | <a id="ll_toolchain-LL_LDFLAGS"></a>`LL_LDFLAGS` | <code><a href="https://bazel.build/concepts/labels">Label</a></code>, optional, defaults to <code>None</code>.<br><br> Arbitrary flags added to all link actions.   |

ll/BUILD.bazel

@@ -94,7 +94,6 @@ string_flag(
         "LL_AMD_INCLUDES",
         "LL_AMD_LIBRARIES",
         "LL_CUDA_TOOLKIT",
-        "LL_CUDA_RUNTIME",
         "LL_CUDA_DRIVER",
 
         # Unset values default to an empty string.
@@ -125,11 +124,6 @@ ll_toolchain(
         ":cuda_nvptx": ":LL_CUDA_DRIVER",
         "//conditions:default": "LL_UNSET",
     }),
-    LL_CUDA_RUNTIME = select({
-        ":hip_nvptx": ":LL_CUDA_RUNTIME",
-        ":cuda_nvptx": ":LL_CUDA_RUNTIME",
-        "//conditions:default": "LL_UNSET",
-    }),
     LL_CUDA_TOOLKIT = select({
         ":hip_nvptx": ":LL_CUDA_TOOLKIT",
         ":cuda_nvptx": ":LL_CUDA_TOOLKIT",

ll/args.bzl

@@ -484,15 +484,16 @@ def link_executable_args(ctx, in_files, out_file, mode):
         "cuda_nvptx",
         "hip_nvptx",
     ]:
-        for location in [toolchain.LL_CUDA_TOOLKIT, toolchain.LL_CUDA_RUNTIME]:
-            if location != "":
-                args.add(location, format = "-rpath=%s/lib")
-                args.add(location, format = "-L%s/lib")
-
-                # TODO: Not pretty. With the right nix packages we can probably
-                #       do this more elegantly.
-                args.add(location, format = "-rpath=%s/lib/stubs")
-                args.add(location, format = "-L%s/lib/stubs")
+        # Both the CUDA driver and the CUDA toolkit contain `libcuda.so`.
+        # Link against `<cudatoolkit>/lib/libcuda.so` at build time, but make
+        # sure that `<cudadriver>/lib/libcuda.so` takes precedence at runtime.
+        if toolchain.LL_CUDA_DRIVER != "":
+            args.add(toolchain.LL_CUDA_DRIVER, format = "-rpath=%s/lib")
+
+        if toolchain.LL_CUDA_TOOLKIT != "":
+            args.add(toolchain.LL_CUDA_TOOLKIT, format = "-rpath=%s/lib")
+            args.add(toolchain.LL_CUDA_TOOLKIT, format = "-L%s/lib")
+            args.add(toolchain.LL_CUDA_TOOLKIT, format = "-L%s/lib/stubs")
 
         args.add("-lcuda")
         args.add("-lcudart_static")

ll/attributes.bzl

@@ -589,12 +589,6 @@ LL_TOOLCHAIN_ATTRS = {
         Affects the `cuda_nvptx` and `hip_nvptx` toolchains.
         """,
     ),
-    "LL_CUDA_RUNTIME": attr.label(
-        doc = """The path to the CUDA runtime.
-
-        Affects the `cuda_nvptx` and `hip_nvptx` toolchains.
-        """,
-    ),
     "LL_CUDA_DRIVER": attr.label(
         doc = """The path to the CUDA driver.
 

ll/toolchain.bzl

@@ -70,7 +70,6 @@ def _ll_toolchain_impl(ctx):
             LL_AMD_INCLUDES = ctx.attr.LL_AMD_INCLUDES[BuildSettingInfo].value,
             LL_AMD_LIBRARIES = ctx.attr.LL_AMD_LIBRARIES[BuildSettingInfo].value,
             LL_CUDA_TOOLKIT = ctx.attr.LL_CUDA_TOOLKIT[BuildSettingInfo].value,
-            LL_CUDA_RUNTIME = ctx.attr.LL_CUDA_RUNTIME[BuildSettingInfo].value,
             LL_CUDA_DRIVER = ctx.attr.LL_CUDA_DRIVER[BuildSettingInfo].value,
         ),
     ]

modules/defaultLlEnv.nix

@@ -74,6 +74,5 @@ in
 
   # Flags for CUDA dependencies.
   "LL_CUDA_TOOLKIT=${lib.strings.optionalString pkgs.config.cudaSupport "${cudatoolkit}"}"
-  "LL_CUDA_RUNTIME=${lib.strings.optionalString pkgs.config.cudaSupport "${cudatoolkit.lib}"}"
   "LL_CUDA_DRIVER=${lib.strings.optionalString pkgs.config.cudaSupport "${nvidia_x11}"}"
 ]

modules/rules_ll.nix

@@ -49,9 +49,7 @@ in
         - `LL_DYNAMIC_LINKER`
         - `LL_AMD_INCLUDES`
         - `LL_AMD_LIBRARIES`
-        - `LL_AMD_RPATHS`
         - `LL_CUDA_TOOLKIT`
-        - `LL_CUDA_RUNTIME`
         - `LL_CUDA_DRIVER`
 
         Attempting to set any other value will result in Bazel errors.