diff --git a/conda_build/build.py b/conda_build/build.py index 28ffc04a70..526d665c8a 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -38,6 +38,7 @@ env_path_backup_var_exists, get_conda_channel, get_rc_urls, + pkgs_dirs, prefix_placeholder, reset_context, root_dir, @@ -3394,6 +3395,25 @@ def test( # folder destination _extract_test_files_from_package(metadata) + # Remove any previously cached build from the package cache to ensure we + # really test the requested build and not some clashing or corrupted build. + # (Corruption of the extracted package can happen, e.g., in multi-output + # builds if one of the subpackages overwrites files from the other.) + # Special case: + # If test is requested for .tar.bz2/.conda file from the pkgs dir itself, + # clean_pkg_cache() will remove it; don't call that function in this case. + in_pkg_cache = ( + not hasattr(recipedir_or_package_or_metadata, "config") + and os.path.isfile(recipedir_or_package_or_metadata) + and recipedir_or_package_or_metadata.endswith(CONDA_PACKAGE_EXTENSIONS) + and any( + os.path.dirname(recipedir_or_package_or_metadata) in pkgs_dir + for pkgs_dir in pkgs_dirs + ) + ) + if not in_pkg_cache: + environ.clean_pkg_cache(metadata.dist(), metadata.config) + copy_test_source_files(metadata, metadata.config.test_dir) # this is also copying tests/source_files from work_dir to testing workdir diff --git a/conda_build/environ.py b/conda_build/environ.py index 3026f1bf60..762b9c7479 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -16,10 +16,15 @@ from logging import getLogger from os.path import join, normpath -from conda.base.constants import DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL +from conda.base.constants import ( + CONDA_PACKAGE_EXTENSIONS, + DEFAULTS_CHANNEL_NAME, + UNKNOWN_CHANNEL, +) from conda.common.io import env_vars from conda.core.index import LAST_CHANNEL_URLS from conda.core.link import PrefixSetup, UnlinkLinkTransaction +from conda.core.package_cache_data import PackageCacheData from conda.core.prefix_data import PrefixData from conda.models.channel import prioritize_channels @@ -43,6 +48,7 @@ reset_context, root_dir, ) +from .config import Config from .deprecations import deprecated from .exceptions import BuildLockError, DependencyNeedsBuildingError from .features import feature_list @@ -1264,6 +1270,29 @@ def get_pkg_dirs_locks(dirs, config): return [utils.get_lock(folder, timeout=config.timeout) for folder in dirs] +def clean_pkg_cache(dist: str, config: Config) -> None: + with utils.LoggingContext(logging.DEBUG if config.debug else logging.WARN): + locks = get_pkg_dirs_locks([config.bldpkgs_dir] + pkgs_dirs, config) + with utils.try_acquire_locks(locks, timeout=config.timeout): + for pkgs_dir in pkgs_dirs: + if any( + os.path.exists(os.path.join(pkgs_dir, f"{dist}{ext}")) + for ext in ("", *CONDA_PACKAGE_EXTENSIONS) + ): + log.debug( + "Conda caching error: %s package remains in cache after removal", + dist, + ) + log.debug("manually removing to compensate") + package_cache = PackageCacheData.first_writable([pkgs_dir]) + for cache_pkg_id in package_cache.query(dist): + package_cache.remove(cache_pkg_id) + + # Note that this call acquires the relevant locks, so this must be called + # outside the lock context above. + remove_existing_packages(pkgs_dirs, [dist], config) + + def remove_existing_packages(dirs, fns, config): locks = get_pkg_dirs_locks(dirs, config) if config.locking else [] diff --git a/news/5184-fix-multi-output-package-corruption b/news/5184-fix-multi-output-package-corruption new file mode 100644 index 0000000000..584a2a1f35 --- /dev/null +++ b/news/5184-fix-multi-output-package-corruption @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Fix corrupted package cache for outputs in subpackage tests. (#5184) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test-recipes/metadata/outputs_overwrite_base_file/install.bat b/tests/test-recipes/metadata/outputs_overwrite_base_file/install.bat new file mode 100644 index 0000000000..b6584f3971 --- /dev/null +++ b/tests/test-recipes/metadata/outputs_overwrite_base_file/install.bat @@ -0,0 +1,2 @@ +:: Always output 4 characters to properly test even if "SafetyError: ... incorrect size." is not triggered. +< nul set /p="%PKG_NAME:~0,4%" > "%PREFIX%\file" & call; diff --git a/tests/test-recipes/metadata/outputs_overwrite_base_file/install.sh b/tests/test-recipes/metadata/outputs_overwrite_base_file/install.sh new file mode 100644 index 0000000000..cb0be8cb2b --- /dev/null +++ b/tests/test-recipes/metadata/outputs_overwrite_base_file/install.sh @@ -0,0 +1,2 @@ +## Always output 4 characters to properly test even if "SafetyError: ... incorrect size." is not triggered. +printf '%.4s' "${PKG_NAME}" > "${PREFIX}/file" diff --git a/tests/test-recipes/metadata/outputs_overwrite_base_file/meta.yaml b/tests/test-recipes/metadata/outputs_overwrite_base_file/meta.yaml new file mode 100644 index 0000000000..1c27afc126 --- /dev/null +++ b/tests/test-recipes/metadata/outputs_overwrite_base_file/meta.yaml @@ -0,0 +1,40 @@ +{% set name = "outputs_overwrite_base_file" %} + +package: + name: {{ name }} + version: 1.0 + +outputs: + - name: base-{{ name }} + script: install.sh # [unix] + script: install.bat # [win] + + - name: first-{{ name }} + script: install.sh # [unix] + script: install.bat # [win] + requirements: + host: + - {{ pin_subpackage("base-" + name) }} + run: + - {{ pin_subpackage("base-" + name) }} + test: + commands: + - content="$(cat "${PREFIX}/file")" # [unix] + - test "${content}" = base # [unix] + - < "%PREFIX%\file%" set /p content= # [win] + - if not "%content%" == "base" exit 1 # [win] + + - name: second-{{ name }} + script: install.sh # [unix] + script: install.bat # [win] + requirements: + host: + - {{ pin_subpackage("base-" + name) }} + run: + - {{ pin_subpackage("base-" + name) }} + test: + commands: + - content="$(cat "${PREFIX}/file")" # [unix] + - test "${content}" = "base" # [unix] + - < "%PREFIX%\file%" set /p content= # [win] + - if not "%content%" == "base" exit 1 # [win]