diff --git a/cachi2/core/models/input.py b/cachi2/core/models/input.py index 90e8401d5..0196410af 100644 --- a/cachi2/core/models/input.py +++ b/cachi2/core/models/input.py @@ -73,6 +73,7 @@ class BundlerPackageInput(_PackageInputBase): """Accepted input for a bundler package.""" type: Literal["bundler"] + allow_binary: bool = False class GomodPackageInput(_PackageInputBase): diff --git a/cachi2/core/models/property_semantics.py b/cachi2/core/models/property_semantics.py index aa6f4afe6..6261b9a77 100644 --- a/cachi2/core/models/property_semantics.py +++ b/cachi2/core/models/property_semantics.py @@ -33,6 +33,7 @@ class PropertySet: npm_bundled: bool = False npm_development: bool = False pip_package_binary: bool = False + bundler_package_binary: bool = False @classmethod def from_properties(cls, props: Iterable[Property]) -> "Self": @@ -42,6 +43,7 @@ def from_properties(cls, props: Iterable[Property]) -> "Self": npm_bundled = False npm_development = False pip_package_binary = False + bundler_package_binary = False for prop in props: if prop.name == "cachi2:found_by": @@ -54,6 +56,8 @@ def from_properties(cls, props: Iterable[Property]) -> "Self": npm_development = True elif prop.name == "cachi2:pip:package:binary": pip_package_binary = True + elif prop.name == "cachi2:bundler:package:binary": + bundler_package_binary = True else: assert_never(prop.name) @@ -63,6 +67,7 @@ def from_properties(cls, props: Iterable[Property]) -> "Self": npm_bundled, npm_development, pip_package_binary, + bundler_package_binary, ) def to_properties(self) -> list[Property]: @@ -80,6 +85,8 @@ def to_properties(self) -> list[Property]: props.append(Property(name="cdx:npm:package:development", value="true")) if self.pip_package_binary: props.append(Property(name="cachi2:pip:package:binary", value="true")) + if self.bundler_package_binary: + props.append(Property(name="cachi2:bundler:package:binary", value="true")) return sorted(props, key=lambda p: (p.name, p.value)) @@ -92,4 +99,5 @@ def merge(self, other: "Self") -> "Self": npm_bundled=self.npm_bundled and other.npm_bundled, npm_development=self.npm_development and other.npm_development, pip_package_binary=self.pip_package_binary or other.pip_package_binary, + bundler_package_binary=self.bundler_package_binary or other.bundler_package_binary, ) diff --git a/cachi2/core/models/sbom.py b/cachi2/core/models/sbom.py index d9e00a45b..3df1ca624 100644 --- a/cachi2/core/models/sbom.py +++ b/cachi2/core/models/sbom.py @@ -5,6 +5,7 @@ from cachi2.core.models.validators import unique_sorted PropertyName = Literal[ + "cachi2:bundler:package:binary", "cachi2:found_by", "cachi2:missing_hash:in_file", "cachi2:pip:package:binary", diff --git a/cachi2/core/package_managers/bundler/main.py b/cachi2/core/package_managers/bundler/main.py index 084c8a1dd..06ae2ade7 100644 --- a/cachi2/core/package_managers/bundler/main.py +++ b/cachi2/core/package_managers/bundler/main.py @@ -2,15 +2,21 @@ import os from pathlib import Path from textwrap import dedent -from typing import Optional +from typing import Optional, cast from packageurl import PackageURL from cachi2.core.errors import PackageRejected, UnsupportedFeature -from cachi2.core.models.input import Request +from cachi2.core.models.input import BundlerPackageInput, Request from cachi2.core.models.output import EnvironmentVariable, ProjectFile, RequestOutput +from cachi2.core.models.property_semantics import PropertySet from cachi2.core.models.sbom import Component -from cachi2.core.package_managers.bundler.parser import ParseResult, PathDependency, parse_lockfile +from cachi2.core.package_managers.bundler.parser import ( + GemPlatformSpecificDependency, + ParseResult, + PathDependency, + parse_lockfile, +) from cachi2.core.rooted_path import RootedPath from cachi2.core.scm import get_repo_id @@ -30,7 +36,11 @@ def fetch_bundler_source(request: Request) -> RequestOutput: for package in request.packages: path_within_root = request.source_dir.join_within_root(package.path) components.extend( - _resolve_bundler_package(package_dir=path_within_root, output_dir=request.output_dir) + _resolve_bundler_package( + package_dir=path_within_root, + output_dir=request.output_dir, + allow_binary=cast(BundlerPackageInput, package).allow_binary, + ) ) project_files.append(_prepare_for_hermetic_build(request.source_dir, request.output_dir)) @@ -41,11 +51,15 @@ def fetch_bundler_source(request: Request) -> RequestOutput: ) -def _resolve_bundler_package(package_dir: RootedPath, output_dir: RootedPath) -> list[Component]: +def _resolve_bundler_package( + package_dir: RootedPath, + output_dir: RootedPath, + allow_binary: bool = False, +) -> list[Component]: """Process a request for a single bundler package.""" deps_dir = output_dir.join_within_root("deps", "bundler") deps_dir.path.mkdir(parents=True, exist_ok=True) - dependencies = parse_lockfile(package_dir) + dependencies = parse_lockfile(package_dir, allow_binary) name, version = _get_main_package_name_and_version(package_dir, dependencies) vcs_url = get_repo_id(package_dir.root).as_vcs_url_qualifier() @@ -60,7 +74,16 @@ def _resolve_bundler_package(package_dir: RootedPath, output_dir: RootedPath) -> components = [Component(name=name, version=version, purl=main_package_purl.to_string())] for dep in dependencies: dep.download_to(deps_dir) - components.append(Component(name=dep.name, version=dep.version, purl=dep.purl)) + if isinstance(dep, GemPlatformSpecificDependency): + c = Component( + name=dep.name, + version=dep.version, + purl=dep.purl, + properties=PropertySet(bundler_package_binary=True).to_properties(), + ) + else: + c = Component(name=dep.name, version=dep.version, purl=dep.purl) + components.append(c) return components diff --git a/cachi2/core/package_managers/bundler/parser.py b/cachi2/core/package_managers/bundler/parser.py index dd576ca33..546023b31 100644 --- a/cachi2/core/package_managers/bundler/parser.py +++ b/cachi2/core/package_managers/bundler/parser.py @@ -80,6 +80,32 @@ def download_to(self, deps_dir: RootedPath) -> None: download_binary_file(self.remote_location, fs_location) +class GemPlatformSpecificDependency(GemDependency): + """ + Represents a gem dependency built for a specific platform. + + Attributes: + platform: Platform for which the dependency was built. + """ + + platform: str + + @cached_property + def remote_location(self) -> str: + """Return remote location to download this gem from.""" + return f"{self.source}/downloads/{self.name}-{self.version}-{self.platform}.gem" + + def download_to(self, deps_dir: RootedPath) -> None: + """Download represented gem to specified file system location.""" + fs_location = deps_dir.join_within_root( + Path(f"{self.name}-{self.version}-{self.platform}.gem") + ) + log.info( + "Downloading platform-specific gem %s-%s-%s", self.name, self.version, self.platform + ) + download_binary_file(self.remote_location, fs_location) + + class GitDependency(_GemMetadata): """ Represents a git dependency. @@ -162,11 +188,13 @@ def purl(self) -> str: return purl.to_string() -BundlerDependency = Union[GemDependency, GitDependency, PathDependency] +BundlerDependency = Union[ + GemDependency, GemPlatformSpecificDependency, GitDependency, PathDependency +] ParseResult = list[BundlerDependency] -def parse_lockfile(package_dir: RootedPath) -> ParseResult: +def parse_lockfile(package_dir: RootedPath, allow_binary: bool = False) -> ParseResult: """Parse a Gemfile.lock file and return a list of dependencies.""" lockfile_path = package_dir.join_within_root(GEMFILE_LOCK) gemfile_path = package_dir.join_within_root(GEMFILE) @@ -194,7 +222,23 @@ def parse_lockfile(package_dir: RootedPath) -> ParseResult: result: ParseResult = [] for dep in dependencies: if dep["type"] == "rubygems": - result.append(GemDependency(**dep)) + if f'{dep["name"]}-{dep["version"]}' != dep["full_name"]: + log.warning("Found a binary dependency %s", dep["full_name"]) + if allow_binary: + log.warning( + "Downloading binary dependency %s because 'allow_binary' is set to True", + dep["full_name"], + ) + result.append(GemPlatformSpecificDependency(**dep)) + else: + # No need to force a platform if we skip the packages. + log.warning( + "Skipping binary dependency %s because 'allow_binary' is set to False." + " This will likely result in an unbuildable package.", + dep["full_name"], + ) + else: + result.append(GemDependency(**dep)) elif dep["type"] == "git": result.append(GitDependency(**dep)) elif dep["type"] == "path": diff --git a/cachi2/core/package_managers/bundler/scripts/lockfile_parser.rb b/cachi2/core/package_managers/bundler/scripts/lockfile_parser.rb index 96b25ca46..a56928515 100755 --- a/cachi2/core/package_managers/bundler/scripts/lockfile_parser.rb +++ b/cachi2/core/package_managers/bundler/scripts/lockfile_parser.rb @@ -11,7 +11,9 @@ lockfile_parser.specs.each do |spec| parsed_spec = { name: spec.name, - version: spec.version.to_s + version: spec.version.to_s, + full_name: spec.full_name, + platform: spec.platform.to_s } case spec.source diff --git a/tests/integration/test_bundler.py b/tests/integration/test_bundler.py index 806c86bd2..f9eafd217 100644 --- a/tests/integration/test_bundler.py +++ b/tests/integration/test_bundler.py @@ -100,3 +100,76 @@ def test_bundler_packages( utils.fetch_deps_and_check_output( tmp_path, test_case, test_params, source_folder, test_data_dir, cachi2_image ) + + +@pytest.mark.parametrize( + "test_params,check_cmd,expected_cmd_output", + [ + pytest.param( + utils.TestParameters( + repo="https://github.com/cachito-testing/cachi2-bundler.git", + ref="well_formed_ruby_all_features", + packages=({"path": ".", "type": "bundler", "allow_binary": "true"},), + flags=["--dev-package-managers"], + check_output=False, + check_deps_checksums=False, + check_vendor_checksums=False, + expected_exit_code=0, + expected_output="", + ), + [], # No additional commands are run to verify the build + [], + id="bundler_everything_present", + ), + pytest.param( + utils.TestParameters( + repo="https://github.com/cachito-testing/cachi2-bundler.git", + ref="well_formed_ruby_without_gemspec", + packages=({"path": ".", "type": "bundler", "allow_binary": "true"},), + flags=["--dev-package-managers"], + check_output=False, + check_deps_checksums=False, + check_vendor_checksums=False, + expected_exit_code=0, + expected_output="", + ), + [], # No additional commands are run to verify the build + [], + id="bundler_everything_present_except_gemspec", + ), + ], +) +def test_e2e_bundler( + test_params: utils.TestParameters, + check_cmd: list[str], + expected_cmd_output: str, + cachi2_image: utils.ContainerImage, + tmp_path: Path, + test_data_dir: Path, + request: pytest.FixtureRequest, +) -> None: + """ + End to end test for bundler. + + :param test_params: Test case arguments + :param tmp_path: Temp directory for pytest + """ + test_case = request.node.callspec.id + + source_folder = utils.clone_repository( + test_params.repo, test_params.ref, f"{test_case}-source", tmp_path + ) + + output_folder = utils.fetch_deps_and_check_output( + tmp_path, test_case, test_params, source_folder, test_data_dir, cachi2_image + ) + + utils.build_image_and_check_cmd( + tmp_path, + output_folder, + test_data_dir, + test_case, + check_cmd, + expected_cmd_output, + cachi2_image, + ) diff --git a/tests/integration/test_data/bundler_everything_present/container/Containerfile b/tests/integration/test_data/bundler_everything_present/container/Containerfile new file mode 100644 index 000000000..1251706aa --- /dev/null +++ b/tests/integration/test_data/bundler_everything_present/container/Containerfile @@ -0,0 +1,25 @@ +FROM docker.io/ruby:3.3 + +# Test disabled network access +RUN if curl -IsS www.google.com; then echo "Has network access!"; exit 1; fi + +# Print cachi2 env vars file +RUN cat /tmp/cachi2.env + +# Check bundler deps +RUN ls /tmp/bundler_everything_present-output/deps/bundler + +# Check content of source repository folder +RUN ls /tmp/bundler_everything_present-source/ + +# This should be a COPY, but the source code and Containerfile are in different directories +RUN cp -r /tmp/bundler_everything_present-source /src + +WORKDIR /src +# Bundler would try and install whichever version was used to generate +# This will cause bundler to attempt to download an earlier version even if +# just microversions diverged. This, int turn, would cause a build to fail. +# Running 'bundle __ install' is supposed to attempt to run installation +# with this specific version. The extra code below ensures that any present +# version is used: +RUN . /tmp/cachi2.env && bundle --version | cut -d ' ' -f3- | xargs -I {} bundle _{}_ install diff --git a/tests/integration/test_data/bundler_everything_present_except_gemspec/container/Containerfile b/tests/integration/test_data/bundler_everything_present_except_gemspec/container/Containerfile new file mode 100644 index 000000000..35426d85a --- /dev/null +++ b/tests/integration/test_data/bundler_everything_present_except_gemspec/container/Containerfile @@ -0,0 +1,25 @@ +FROM docker.io/ruby:3.3 + +# Test disabled network access +RUN if curl -IsS www.google.com; then echo "Has network access!"; exit 1; fi + +# Print cachi2 env vars file +RUN cat /tmp/cachi2.env + +# Check bundler deps +RUN ls /tmp/bundler_everything_present_except_gemspec-output/deps/bundler + +# Check content of source repository folder +RUN ls /tmp/bundler_everything_present_except_gemspec-source/ + +# This should be a COPY, but the source code and Containerfile are in different directories +RUN cp -r /tmp/bundler_everything_present_except_gemspec-source /src + +WORKDIR /src +# Bundler would try and install whichever version was used to generate +# This will cause bundler to attempt to download an earlier version even if +# just microversions diverged. This, int turn, would cause a build to fail. +# Running 'bundle __ install' is supposed to attempt to run installation +# with this specific version. The extra code below ensures that any present +# version is used: +RUN . /tmp/cachi2.env && bundle --version | cut -d ' ' -f3- | xargs -I {} bundle _{}_ install diff --git a/tests/unit/package_managers/bundler/test_main.py b/tests/unit/package_managers/bundler/test_main.py index dd4d195da..2fabe2283 100644 --- a/tests/unit/package_managers/bundler/test_main.py +++ b/tests/unit/package_managers/bundler/test_main.py @@ -64,7 +64,7 @@ def test_resolve_bundler_package( components = _resolve_bundler_package(package_dir=package_dir, output_dir=output_dir) - mock_parse_lockfile.assert_called_once_with(package_dir) + mock_parse_lockfile.assert_called_once_with(package_dir, False) mock_get_main_package_name_and_version.assert_called_once_with(package_dir, deps) mock_gem_dep_download_to.assert_called_with(deps_dir) mock_git_dep_download_to.assert_called_with(deps_dir) diff --git a/tests/unit/package_managers/bundler/test_parser.py b/tests/unit/package_managers/bundler/test_parser.py index 687a4ca05..e079f71b4 100644 --- a/tests/unit/package_managers/bundler/test_parser.py +++ b/tests/unit/package_managers/bundler/test_parser.py @@ -2,7 +2,7 @@ import subprocess from copy import deepcopy from pathlib import Path -from typing import Any +from typing import Any, Iterable from unittest import mock import pydantic @@ -15,6 +15,7 @@ GEMFILE_LOCK, BundlerDependency, GemDependency, + GemPlatformSpecificDependency, GitDependency, PathDependency, parse_lockfile, @@ -23,6 +24,15 @@ from tests.common_utils import GIT_REF +def relaxed_in(substring: str, messages: Iterable[str]) -> bool: + """Check if substring could be found in any message. + + This produces a bit less coupling between tests and code than + checking for a full message. + """ + return any(substring in m for m in messages) + + @pytest.fixture def empty_bundler_files(rooted_tmp_path: RootedPath) -> tuple[RootedPath, RootedPath]: gemfile_path = rooted_tmp_path.join_within_root(GEMFILE) @@ -150,6 +160,8 @@ def test_parse_gemlock( { "type": "rubygems", "source": "https://rubygems.org/", + "full_name": f"{base_dep['name']}-{base_dep['version']}", + "platform": "", **base_dep, }, ] @@ -199,7 +211,7 @@ def test_parse_gemlock_empty( ], ) @mock.patch("cachi2.core.package_managers.bundler.parser.download_binary_file") -def test_dependencies_could_be_downloaded( +def test_source_gem_dependencies_could_be_downloaded( mock_downloader: mock.MagicMock, caplog: pytest.LogCaptureFixture, source: str, @@ -215,6 +227,29 @@ def test_dependencies_could_be_downloaded( mock_downloader.assert_called_once_with(expected_source_url, expected_destination) +@mock.patch("cachi2.core.package_managers.bundler.parser.download_binary_file") +def test_binary_gem_dependencies_could_be_downloaded( + mock_downloader: mock.MagicMock, + caplog: pytest.LogCaptureFixture, +) -> None: + base_destination = RootedPath("/tmp/foo") + source = "https://rubygems.org" + platform = "m6502_wm" + dependency = GemPlatformSpecificDependency( + name="foo", + version="0.0.2", + source=source, + platform=platform, + ) + expected_source_url = f"{source}/downloads/foo-0.0.2-{platform}.gem" + expected_destination = base_destination.join_within_root(Path(f"foo-0.0.2-{platform}.gem")) + + dependency.download_to(base_destination) + + assert relaxed_in("Downloading platform-specific gem", caplog.messages) + mock_downloader.assert_called_once_with(expected_source_url, expected_destination) + + @mock.patch("cachi2.core.package_managers.bundler.parser.Repo.clone_from") def test_download_git_dependency_works( mock_git_clone: mock.Mock, @@ -302,3 +337,69 @@ def test_purls(rooted_tmp_path_repo: RootedPath) -> None: for dep, expected_purl in deps: assert dep.purl == expected_purl + + +@mock.patch("cachi2.core.package_managers.bundler.parser.run_cmd") +def test_parse_gemlock_detects_binaries_and_adds_to_parse_result_when_allowed_to( + mock_run_cmd: mock.MagicMock, + empty_bundler_files: tuple[RootedPath, RootedPath], + sample_parser_output: dict[str, Any], + rooted_tmp_path: RootedPath, + caplog: pytest.LogCaptureFixture, +) -> None: + base_dep: dict[str, str] = sample_parser_output["dependencies"][0] + sample_parser_output["dependencies"] = [ + { + "type": "rubygems", + "source": "https://rubygems.org/", + "full_name": f"{base_dep['name']}-{base_dep['version']}-i8080_cpm", + "platform": "i8080_cpm", + **base_dep, + }, + ] + + mock_run_cmd.return_value = json.dumps(sample_parser_output) + result = parse_lockfile(rooted_tmp_path, allow_binary=True) + + expected_deps = [ + GemPlatformSpecificDependency( + name="example", + version="0.1.0", + source="https://rubygems.org/", + platform="i8080_cpm", + ), + ] + + assert relaxed_in("Found a binary dependency", caplog.messages) + assert relaxed_in("Downloading binary dependency", caplog.messages) + assert result == expected_deps + + +@mock.patch("cachi2.core.package_managers.bundler.parser.run_cmd") +def test_parse_gemlock_detects_binaries_and_skips_then_when_instructed_to_skip( + mock_run_cmd: mock.MagicMock, + empty_bundler_files: tuple[RootedPath, RootedPath], + sample_parser_output: dict[str, Any], + rooted_tmp_path: RootedPath, + caplog: pytest.LogCaptureFixture, +) -> None: + base_dep: dict[str, str] = sample_parser_output["dependencies"][0] + sample_parser_output["dependencies"] = [ + { + "type": "rubygems", + "source": "https://rubygems.org/", + "full_name": f"{base_dep['name']}-{base_dep['version']}-i8080_cpm", + "platform": "i8080_cpm", + **base_dep, + }, + ] + + mock_run_cmd.return_value = json.dumps(sample_parser_output) + result = parse_lockfile(rooted_tmp_path, allow_binary=False) + + expected_deps: list = [] # mypy demanded this annotation and is content with it. + + assert relaxed_in("Found a binary dependency", caplog.messages) + assert relaxed_in("Skipping binary dependency", caplog.messages) + + assert result == expected_deps