Skip to content

Commit

Permalink
Add support for Java platformization in Bazel@head (#926)
Browse files Browse the repository at this point in the history
* Add support for Java platformization in Bazel@head

* Buildifier

* Fixing Java version autodetection to use docker (or allowing user to specify the version)

* docstring

* Mitigate for docker images with entry point set.

* Remove failure if Java version cannot be detected (there are repos with java_home but no Java)

* Dump and read in Java version.

* Dump and read in Java version.

* Fix uninitialized java_version variable.
  • Loading branch information
comius authored Jan 12, 2021
1 parent 6121e2e commit b9bc541
Show file tree
Hide file tree
Showing 10 changed files with 223 additions and 14 deletions.
63 changes: 63 additions & 0 deletions bazelrc/bazel-4.1.0.bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# Copyright 2016 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This .bazelrc file contains all of the flags required for the provided
# toolchain with Remote Build Execution.
# Note your WORKSPACE must contain an rbe_autoconfig target with
# name="rbe_default" to use these flags as-is.

# Depending on how many machines are in the remote execution instance, setting
# this higher can make builds faster by allowing more jobs to run in parallel.
# Setting it too high can result in jobs that timeout, however, while waiting
# for a remote machine to execute them.
build:remote --jobs=50

# Set several flags related to specifying the platform, toolchain and java
# properties.
# These flags should only be used as is for the rbe-ubuntu16-04 container
# and need to be adapted to work with other toolchain containers.
build:remote --java_runtime_version=rbe_jdk
build:remote --tool_java_runtime_version=rbe_jdk
build:remote --extra_toolchains=@rbe_default//java:all

build:remote --crosstool_top=@rbe_default//cc:toolchain
build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
# Platform flags:
# The toolchain container used for execution is defined in the target indicated
# by "extra_execution_platforms", "host_platform" and "platforms".
# More about platforms: https://docs.bazel.build/versions/master/platforms.html
build:remote --extra_toolchains=@rbe_default//config:cc-toolchain
build:remote --extra_execution_platforms=@rbe_default//config:platform
build:remote --host_platform=@rbe_default//config:platform
build:remote --platforms=@rbe_default//config:platform

# Starting with Bazel 0.27.0 strategies do not need to be explicitly
# defined. See https://github.com/bazelbuild/bazel/issues/7480
build:remote --define=EXECUTOR=remote

# Enable remote execution so actions are performed on the remote systems.
build:remote --remote_executor=grpcs://remotebuildexecution.googleapis.com

# Enforce stricter environment rules, which eliminates some non-hermetic
# behavior and therefore improves both the remote cache hit rate and the
# correctness and repeatability of the build.
build:remote --incompatible_strict_action_env=true

# Set a higher timeout value, just in case.
build:remote --remote_timeout=3600

# Enable authentication. This will pick up application default credentials by
# default. You can use --google_credentials=some_file.json to use a service
# account credential instead.
build:remote --google_default_credentials=true
8 changes: 4 additions & 4 deletions configs/ubuntu16_04_clang/versions.bzl
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# Generated file, do not modify by hand
# Generated by 'rbe_autoconfig_autogen_ubuntu1604' rbe_autoconfig rule
"""Definitions to be used in rbe_repo attr of an rbe_autoconf rule """
toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"ABI_LIBC_VERSION": "glibc_2.19", "ABI_VERSION": "clang", "BAZEL_COMPILER": "clang", "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu", "BAZEL_TARGET_CPU": "k8", "BAZEL_TARGET_LIBC": "glibc_2.19", "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu", "CC": "clang", "CC_TOOLCHAIN_NAME": "linux_gnu_x86"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", name = "9.0.0")
toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"ABI_LIBC_VERSION": "glibc_2.19", "ABI_VERSION": "clang", "BAZEL_COMPILER": "clang", "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu", "BAZEL_TARGET_CPU": "k8", "BAZEL_TARGET_LIBC": "glibc_2.19", "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu", "CC": "clang", "CC_TOOLCHAIN_NAME": "linux_gnu_x86"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", name = "8.0.0")
toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"ABI_LIBC_VERSION": "glibc_2.19", "ABI_VERSION": "clang", "BAZEL_COMPILER": "clang", "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu", "BAZEL_TARGET_CPU": "k8", "BAZEL_TARGET_LIBC": "glibc_2.19", "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu", "CC": "clang", "CC_TOOLCHAIN_NAME": "linux_gnu_x86"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", name = "10.0.0")
toolchain_config_spec3 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"ABI_LIBC_VERSION": "glibc_2.19", "ABI_VERSION": "clang", "BAZEL_COMPILER": "clang", "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu", "BAZEL_TARGET_CPU": "k8", "BAZEL_TARGET_LIBC": "glibc_2.19", "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu", "CC": "clang", "CC_TOOLCHAIN_NAME": "linux_gnu_x86"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", name = "11.0.0")
toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"ABI_LIBC_VERSION": "glibc_2.19", "ABI_VERSION": "clang", "BAZEL_COMPILER": "clang", "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu", "BAZEL_TARGET_CPU": "k8", "BAZEL_TARGET_LIBC": "glibc_2.19", "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu", "CC": "clang", "CC_TOOLCHAIN_NAME": "linux_gnu_x86"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", java_version = "8", name = "9.0.0")
toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"ABI_LIBC_VERSION": "glibc_2.19", "ABI_VERSION": "clang", "BAZEL_COMPILER": "clang", "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu", "BAZEL_TARGET_CPU": "k8", "BAZEL_TARGET_LIBC": "glibc_2.19", "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu", "CC": "clang", "CC_TOOLCHAIN_NAME": "linux_gnu_x86"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", java_version = "8", name = "8.0.0")
toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"ABI_LIBC_VERSION": "glibc_2.19", "ABI_VERSION": "clang", "BAZEL_COMPILER": "clang", "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu", "BAZEL_TARGET_CPU": "k8", "BAZEL_TARGET_LIBC": "glibc_2.19", "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu", "CC": "clang", "CC_TOOLCHAIN_NAME": "linux_gnu_x86"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", java_version = "8", name = "10.0.0")
toolchain_config_spec3 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"ABI_LIBC_VERSION": "glibc_2.19", "ABI_VERSION": "clang", "BAZEL_COMPILER": "clang", "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu", "BAZEL_TARGET_CPU": "k8", "BAZEL_TARGET_LIBC": "glibc_2.19", "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu", "CC": "clang", "CC_TOOLCHAIN_NAME": "linux_gnu_x86"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", java_version = "8", name = "11.0.0")
_TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0, toolchain_config_spec1, toolchain_config_spec2, toolchain_config_spec3]
_BAZEL_TO_CONFIG_SPEC_NAMES = {"0.20.0": ["8.0.0"], "0.21.0": ["8.0.0"], "0.22.0": ["8.0.0", "9.0.0"], "0.23.0": ["8.0.0", "9.0.0"], "0.23.1": ["8.0.0", "9.0.0"], "0.23.2": ["9.0.0"], "0.24.0": ["9.0.0"], "0.24.1": ["9.0.0"], "0.25.0": ["9.0.0"], "0.25.1": ["9.0.0"], "0.25.2": ["9.0.0"], "0.26.0": ["9.0.0"], "0.26.1": ["9.0.0"], "0.27.0": ["9.0.0"], "0.27.1": ["9.0.0"], "0.28.0": ["9.0.0"], "0.28.1": ["9.0.0"], "0.29.0": ["9.0.0"], "0.29.1": ["9.0.0", "10.0.0"], "1.0.0": ["9.0.0", "10.0.0"], "1.0.1": ["10.0.0"], "1.1.0": ["10.0.0"], "1.2.0": ["10.0.0"], "1.2.1": ["10.0.0"], "2.0.0": ["10.0.0"], "2.1.0": ["10.0.0"], "2.1.1": ["10.0.0", "11.0.0"], "2.2.0": ["11.0.0"], "3.0.0": ["11.0.0"], "3.1.0": ["11.0.0"], "3.2.0": ["11.0.0"], "3.3.0": ["11.0.0"], "3.3.1": ["11.0.0"], "3.4.1": ["11.0.0"], "3.5.0": ["11.0.0"], "3.5.1": ["11.0.0"], "3.6.0": ["11.0.0"], "3.7.0": ["11.0.0"], "3.7.1": ["11.0.0"], "3.7.2": ["11.0.0"]}
LATEST = "sha256:f6568d8168b14aafd1b707019927a63c2d37113a03bcee188218f99bd0327ea1"
Expand Down
55 changes: 52 additions & 3 deletions rules/rbe_repo.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,7 @@ load(
load(
"//rules/rbe_repo:container.bzl",
"get_java_home",
"get_java_version",
"pull_container_needed",
"pull_image",
"run_and_extract",
Expand Down Expand Up @@ -543,10 +544,12 @@ def _rbe_autoconfig_impl(ctx):
# Get the value of JAVA_HOME to set in the produced
# java_runtime
java_home = None
java_version = None
if ctx.attr.create_java_configs:
java_home = get_java_home(ctx, docker_tool_path, image_name)
java_version = get_java_version(ctx, docker_tool_path, image_name, java_home)
if java_home:
create_java_runtime(ctx, java_home)
create_java_runtime(ctx, java_home, java_version)

toolchain_config_spec_name = ctx.attr.toolchain_config_spec_name
if ctx.attr.config_version:
Expand Down Expand Up @@ -603,6 +606,7 @@ def _rbe_autoconfig_impl(ctx):
digest = digest,
toolchain_config_spec_name = toolchain_config_spec_name,
java_home = java_home,
java_version = java_version,
project_root = export_project_root,
)

Expand Down Expand Up @@ -704,6 +708,10 @@ _rbe_autoconfig = repository_rule(
doc = ("Set by rbe_autoconfig macro. Set to list 'java_home' generated by config_to_string_lists def in " +
"//rules/rbe_repo/toolchain_config_suite_spec.bzl."),
),
"configs_obj_java_version": attr.string_list(
doc = ("Set by rbe_autoconfig macro. Set to list 'java_version' generated by config_to_string_lists def in " +
"//rules/rbe_repo/toolchain_config_suite_spec.bzl."),
),
"configs_obj_names": attr.string_list(
doc = ("Set by rbe_autoconfig macro. Set to list 'names' generated by config_to_string_lists def in " +
"//rules/rbe_repo/toolchain_config_suite_spec.bzl."),
Expand Down Expand Up @@ -811,6 +819,10 @@ _rbe_autoconfig = repository_rule(
"JAVA_HOME env var from the container. If that is not set, the rule " +
"will fail."),
),
"java_version": attr.string(
doc = ("Optional. The Java release version in the container. For " +
" example, 11. Should only be set if java_home is set."),
),
"registry": attr.string(
doc = ("Optional. The registry to pull the container from. For example, " +
"marketplace.gcr.io. The default is the value for the selected " +
Expand Down Expand Up @@ -899,6 +911,7 @@ def rbe_autoconfig(
exec_properties = None,
export_configs = False,
java_home = None,
java_version = None,
tag = None,
toolchain_config_suite_spec = default_toolchain_config_suite_spec(),
registry = None,
Expand Down Expand Up @@ -988,6 +1001,8 @@ def rbe_autoconfig(
the 'toolchain_config_suite_spec', fallback to turning on 'detect_java_home'
(unless use_checked_in_confs = Force was set), or otherwise fail with an
informative error.
java_version: Optional. The Java release version in the container. For
example, 11. Should only be set if java_home is set.
tag: Optional. The tag of the container to use.
Should not be set if 'digest' is used.
Must be set together with 'registry' and 'repository'.
Expand Down Expand Up @@ -1031,11 +1046,13 @@ def rbe_autoconfig(
if bazel_rc_version and not bazel_version:
fail("bazel_rc_version can only be used with bazel_version.")

if not create_java_configs and (java_home or detect_java_home):
fail("java_home / detect_java_home should not be set when " +
if not create_java_configs and (java_home or java_version or detect_java_home):
fail("java_home, java_version, or detect_java_home should not be set when " +
"create_java_configs is False.")
if java_home and detect_java_home:
fail("java_home should not be set when detect_java_home is True.")
if java_version and detect_java_home:
fail("java_version should not be set when detect_java_home is True.")

validate_toolchain_config_suite_spec(name, toolchain_config_suite_spec)

Expand Down Expand Up @@ -1115,6 +1132,35 @@ def rbe_autoconfig(
force = CHECKED_IN_CONFS_FORCE,
))

# If create_java_configs was requested but no java_version or detect_java_home was
# set, we try to resolve a java_home
if create_java_configs and not java_version and not detect_java_home:
# If a spec was found and that has a java_home, use it
if toolchain_config_spec and toolchain_config_spec.create_java_configs:
java_version = toolchain_config_spec.java_version

elif toolchain_config_suite_spec.get("default_java_version"):
# Fallback to try to using the default_java_version set in the
# toolchain_config_suite_spec
java_version = toolchain_config_suite_spec.get("default_java_version")

elif use_checked_in_confs != CHECKED_IN_CONFS_FORCE:
# Fallback to detecting the java_home if CHECKED_IN_CONFS_FORCE
# was not passed
detect_java_home = True

elif toolchain_config_spec and use_checked_in_confs == CHECKED_IN_CONFS_FORCE:
# If we get here, the toolchain_config_spec we found does not
# provide a java_home that we can use, and the toolchain_config_suite_spec
# does not have a default one either, so just fail early.
fail(("Target '{name}' failed: use_checked_in_confs was set to '{force}' " +
"but no checked-in configs were found which provide a value for java_version. " +
"This may be solved by defining a 'default_java_version' in the " +
"toolchain_config_spec or by explicitly setting 'java_version' in '{name}'").format(
name = name,
force = CHECKED_IN_CONFS_FORCE,
))

# If the user selected no digest explicitly, and one was returned
# by validateUseOfCheckedInConfigs, use that one.
if not digest and selected_digest:
Expand Down Expand Up @@ -1152,6 +1198,7 @@ def rbe_autoconfig(
config_objs = struct(
names = None,
java_home = None,
java_version = None,
create_java_configs = None,
create_cc_configs = None,
config_repos = None,
Expand All @@ -1176,6 +1223,7 @@ def rbe_autoconfig(
toolchain_config_spec_name = toolchain_config_spec_name,
configs_obj_names = config_objs.names,
configs_obj_java_home = config_objs.java_home if create_java_configs else None,
configs_obj_java_version = config_objs.java_version if create_java_configs else None,
configs_obj_create_java_configs = config_objs.create_java_configs,
configs_obj_create_cc_configs = config_objs.create_cc_configs,
configs_obj_config_repos = config_objs.config_repos,
Expand All @@ -1195,6 +1243,7 @@ def rbe_autoconfig(
internal_exec_properties = exec_properties,
export_configs = export_configs,
java_home = java_home,
java_version = java_version,
toolchain_config_suite_spec = toolchain_config_suite_spec_stripped,
registry = registry,
repository = repository,
Expand Down
32 changes: 32 additions & 0 deletions rules/rbe_repo/BUILD.local_java_runtime.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Copyright 2020 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is auto-generated by an rbe_autoconfig repository rule
# and should not be modified directly.
# See @bazel_toolchains//rules:rbe_repo.bzl

load("@bazel_tools//tools/jdk:local_java_repository.bzl", "local_java_runtime")

package(default_visibility = ["//visibility:public"])

alias(
name = "jdk",
actual = "rbe_jdk",
)

local_java_runtime(
name = "rbe_jdk",
java_home = "%{java_home}",
version = "%{java_version}",
)
12 changes: 9 additions & 3 deletions rules/rbe_repo/build_gen.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -81,19 +81,25 @@ def create_config_aliases(ctx, toolchain_config_spec_name):
False,
)

def create_java_runtime(ctx, java_home):
def create_java_runtime(ctx, java_home, java_version):
"""Creates a BUILD file with the java_runtime target.
Args:
ctx: the Bazel context object.
java_home: the seleceted/resolved location for java_home.
java_home: the selected/resolved location for java_home.
java_version: the Java runtime release version
"""
template = ctx.path(Label("@bazel_toolchains//rules/rbe_repo:BUILD.java.tpl"))
bazel_version = tuple([int(n) for n in ctx.attr.bazel_version.split(".")])
if bazel_version > (4, 0, 0) or not native.bazel_version:
template = ctx.path(Label("@bazel_toolchains//rules/rbe_repo:BUILD.local_java_runtime.tpl"))
else:
template = ctx.path(Label("@bazel_toolchains//rules/rbe_repo:BUILD.java.tpl"))
ctx.template(
JAVA_CONFIG_DIR + "/BUILD",
template,
{
"%{java_home}": java_home,
"%{java_version}": java_version,
},
False,
)
Expand Down
47 changes: 47 additions & 0 deletions rules/rbe_repo/container.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,53 @@ def get_java_home(ctx, docker_tool_path, image_name):
else:
return None

def get_java_version(ctx, docker_tool_path, image_name, java_home):
"""Gets the release version of Java runtime.
Gets the release version of Java runtime either from attr or
by running docker run image_name java -XshowSettings:properties.
Args:
ctx: the Bazel context object.
docker_tool_path: path to the docker binary.
image_name: name of the image to pull.
java_home: java_home.
Returns:
Returns the release version of Java runtime.
"""
if ctx.attr.java_version:
return ctx.attr.java_version
elif docker_tool_path:
properties_out = ctx.execute([
docker_tool_path,
"run",
"--entrypoint",
java_home + "/bin/java",
image_name,
"-XshowSettings:properties",
]).stderr
# This returns an indented list of properties separated with newlines:
# " java.vendor.url.bug = ... \n"
# " java.version = 11.0.8\n"
# " java.version.date = 2020-11-05\"

strip_properties = [property.strip() for property in properties_out.splitlines()]
version_property = [property for property in strip_properties if property.startswith("java.version = ")]
if len(version_property) != 1:
return "unknown"

version_value = version_property[0][len("java.version = "):]
(major, minor, rest) = version_value.split(".", 2)

if major == "1": # handles versions below 1.8
return minor
return major
elif ctx.attr.export_configs:
fail(("%s failed: export_configs was set but neither java_version nor " +
"detect_java_home was set.") % ctx.attr.name)
return "unknown"

def run_and_extract(
ctx,
bazel_version,
Expand Down
Loading

0 comments on commit b9bc541

Please sign in to comment.