From a379a911cadcffde213fcbb8160d8b56ddc1c1a4 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 14 Feb 2024 10:54:17 +0100 Subject: [PATCH 1/3] Add test recipe outputs_overwrite_base_file Signed-off-by: Marcel Bargull --- .../outputs_overwrite_base_file/install.bat | 2 + .../outputs_overwrite_base_file/install.sh | 2 + .../outputs_overwrite_base_file/meta.yaml | 40 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 tests/test-recipes/metadata/outputs_overwrite_base_file/install.bat create mode 100644 tests/test-recipes/metadata/outputs_overwrite_base_file/install.sh create mode 100644 tests/test-recipes/metadata/outputs_overwrite_base_file/meta.yaml diff --git a/tests/test-recipes/metadata/outputs_overwrite_base_file/install.bat b/tests/test-recipes/metadata/outputs_overwrite_base_file/install.bat new file mode 100644 index 0000000000..b6584f3971 --- /dev/null +++ b/tests/test-recipes/metadata/outputs_overwrite_base_file/install.bat @@ -0,0 +1,2 @@ +:: Always output 4 characters to properly test even if "SafetyError: ... incorrect size." is not triggered. +< nul set /p="%PKG_NAME:~0,4%" > "%PREFIX%\file" & call; diff --git a/tests/test-recipes/metadata/outputs_overwrite_base_file/install.sh b/tests/test-recipes/metadata/outputs_overwrite_base_file/install.sh new file mode 100644 index 0000000000..cb0be8cb2b --- /dev/null +++ b/tests/test-recipes/metadata/outputs_overwrite_base_file/install.sh @@ -0,0 +1,2 @@ +## Always output 4 characters to properly test even if "SafetyError: ... incorrect size." is not triggered. +printf '%.4s' "${PKG_NAME}" > "${PREFIX}/file" diff --git a/tests/test-recipes/metadata/outputs_overwrite_base_file/meta.yaml b/tests/test-recipes/metadata/outputs_overwrite_base_file/meta.yaml new file mode 100644 index 0000000000..1c27afc126 --- /dev/null +++ b/tests/test-recipes/metadata/outputs_overwrite_base_file/meta.yaml @@ -0,0 +1,40 @@ +{% set name = "outputs_overwrite_base_file" %} + +package: + name: {{ name }} + version: 1.0 + +outputs: + - name: base-{{ name }} + script: install.sh # [unix] + script: install.bat # [win] + + - name: first-{{ name }} + script: install.sh # [unix] + script: install.bat # [win] + requirements: + host: + - {{ pin_subpackage("base-" + name) }} + run: + - {{ pin_subpackage("base-" + name) }} + test: + commands: + - content="$(cat "${PREFIX}/file")" # [unix] + - test "${content}" = base # [unix] + - < "%PREFIX%\file%" set /p content= # [win] + - if not "%content%" == "base" exit 1 # [win] + + - name: second-{{ name }} + script: install.sh # [unix] + script: install.bat # [win] + requirements: + host: + - {{ pin_subpackage("base-" + name) }} + run: + - {{ pin_subpackage("base-" + name) }} + test: + commands: + - content="$(cat "${PREFIX}/file")" # [unix] + - test "${content}" = "base" # [unix] + - < "%PREFIX%\file%" set /p content= # [win] + - if not "%content%" == "base" exit 1 # [win] From 34552d5ba85a7b385f823541a5c62bc3e4843c26 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 14 Feb 2024 14:03:46 +0100 Subject: [PATCH 2/3] Fix corrupted package cache for outputs in subpackage tests This re-introduces conda_build.environ.clean_pkg_cache with slight changes to not use conda.models.dist.Dist and handle multiple pkgs_dirs better. Signed-off-by: Marcel Bargull --- conda_build/build.py | 20 +++++++++++++ conda_build/environ.py | 30 ++++++++++++++++++- news/5184-fix-multi-output-package-corruption | 19 ++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 news/5184-fix-multi-output-package-corruption diff --git a/conda_build/build.py b/conda_build/build.py index 28ffc04a70..526d665c8a 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -38,6 +38,7 @@ env_path_backup_var_exists, get_conda_channel, get_rc_urls, + pkgs_dirs, prefix_placeholder, reset_context, root_dir, @@ -3394,6 +3395,25 @@ def test( # folder destination _extract_test_files_from_package(metadata) + # Remove any previously cached build from the package cache to ensure we + # really test the requested build and not some clashing or corrupted build. + # (Corruption of the extracted package can happen, e.g., in multi-output + # builds if one of the subpackages overwrites files from the other.) + # Special case: + # If test is requested for .tar.bz2/.conda file from the pkgs dir itself, + # clean_pkg_cache() will remove it; don't call that function in this case. + in_pkg_cache = ( + not hasattr(recipedir_or_package_or_metadata, "config") + and os.path.isfile(recipedir_or_package_or_metadata) + and recipedir_or_package_or_metadata.endswith(CONDA_PACKAGE_EXTENSIONS) + and any( + os.path.dirname(recipedir_or_package_or_metadata) in pkgs_dir + for pkgs_dir in pkgs_dirs + ) + ) + if not in_pkg_cache: + environ.clean_pkg_cache(metadata.dist(), metadata.config) + copy_test_source_files(metadata, metadata.config.test_dir) # this is also copying tests/source_files from work_dir to testing workdir diff --git a/conda_build/environ.py b/conda_build/environ.py index 3026f1bf60..285143428a 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -16,10 +16,15 @@ from logging import getLogger from os.path import join, normpath -from conda.base.constants import DEFAULTS_CHANNEL_NAME, UNKNOWN_CHANNEL +from conda.base.constants import ( + CONDA_PACKAGE_EXTENSIONS, + DEFAULTS_CHANNEL_NAME, + UNKNOWN_CHANNEL, +) from conda.common.io import env_vars from conda.core.index import LAST_CHANNEL_URLS from conda.core.link import PrefixSetup, UnlinkLinkTransaction +from conda.core.package_cache_data import PackageCacheData from conda.core.prefix_data import PrefixData from conda.models.channel import prioritize_channels @@ -1264,6 +1269,29 @@ def get_pkg_dirs_locks(dirs, config): return [utils.get_lock(folder, timeout=config.timeout) for folder in dirs] +def clean_pkg_cache(dist, config): + with utils.LoggingContext(logging.DEBUG if config.debug else logging.WARN): + locks = get_pkg_dirs_locks([config.bldpkgs_dir] + pkgs_dirs, config) + with utils.try_acquire_locks(locks, timeout=config.timeout): + for pkgs_dir in pkgs_dirs: + if any( + os.path.exists(os.path.join(pkgs_dir, f"{dist}{ext}")) + for ext in ("", *CONDA_PACKAGE_EXTENSIONS) + ): + log.debug( + "Conda caching error: %s package remains in cache after removal", + dist, + ) + log.debug("manually removing to compensate") + package_cache = PackageCacheData.first_writable([pkgs_dir]) + for cache_pkg_id in package_cache.query(dist): + package_cache.remove(cache_pkg_id) + + # Note that this call acquires the relevant locks, so this must be called + # outside the lock context above. + remove_existing_packages(pkgs_dirs, [dist], config) + + def remove_existing_packages(dirs, fns, config): locks = get_pkg_dirs_locks(dirs, config) if config.locking else [] diff --git a/news/5184-fix-multi-output-package-corruption b/news/5184-fix-multi-output-package-corruption new file mode 100644 index 0000000000..584a2a1f35 --- /dev/null +++ b/news/5184-fix-multi-output-package-corruption @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Fix corrupted package cache for outputs in subpackage tests. (#5184) + +### Deprecations + +* + +### Docs + +* + +### Other + +* From 6db894e734a9b2a4c76149d52459eb4b3f846e71 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Wed, 14 Feb 2024 16:40:33 +0100 Subject: [PATCH 3/3] Add typing annotations to .environ.clean_pkg_cache Signed-off-by: Marcel Bargull Co-authored-by: Ken Odegard --- conda_build/environ.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 285143428a..762b9c7479 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -48,6 +48,7 @@ reset_context, root_dir, ) +from .config import Config from .deprecations import deprecated from .exceptions import BuildLockError, DependencyNeedsBuildingError from .features import feature_list @@ -1269,7 +1270,7 @@ def get_pkg_dirs_locks(dirs, config): return [utils.get_lock(folder, timeout=config.timeout) for folder in dirs] -def clean_pkg_cache(dist, config): +def clean_pkg_cache(dist: str, config: Config) -> None: with utils.LoggingContext(logging.DEBUG if config.debug else logging.WARN): locks = get_pkg_dirs_locks([config.bldpkgs_dir] + pkgs_dirs, config) with utils.try_acquire_locks(locks, timeout=config.timeout):