From 7c949b8eb267318a7755dfb1cde97736e5b1e2af Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 16 Jan 2025 09:26:35 -0500 Subject: [PATCH] feat: implement CEP-20 for Python ABI3 packages (#1320) --- src/build.rs | 2 +- src/env_vars.rs | 2 +- src/package_test/content_test.rs | 22 +++++---- src/packaging.rs | 2 +- src/packaging/file_mapper.rs | 3 +- src/packaging/metadata.rs | 14 ++++-- src/post_process/python.rs | 2 +- src/recipe/parser/build.rs | 16 ++++++- src/recipe/parser/requirements.rs | 6 ++- ...recipe__parser__tests__recipe_windows.snap | 1 + ...d__recipe__parser__tests__unix_recipe.snap | 1 + src/variant_config.rs | 10 ++++- src/variant_render.rs | 2 +- test-data/recipes/abi3/recipe.yaml | 45 +++++++++++++++++++ test-data/recipes/abi3/variants.yaml | 2 + test/end-to-end/test_simple.py | 27 +++++++++++ 16 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 test-data/recipes/abi3/recipe.yaml create mode 100644 test-data/recipes/abi3/variants.yaml diff --git a/src/build.rs b/src/build.rs index d28a3c8be..2ee85ae2d 100644 --- a/src/build.rs +++ b/src/build.rs @@ -146,7 +146,7 @@ pub async fn run_build( for test in output.recipe.tests() { if let TestType::PackageContents { package_contents } = test { package_contents - .run_test(&paths_json, &output.build_configuration.target_platform) + .run_test(&paths_json, &output) .into_diagnostic()?; } } diff --git a/src/env_vars.rs b/src/env_vars.rs index 0a70e761e..e31fdcf83 100644 --- a/src/env_vars.rs +++ b/src/env_vars.rs @@ -255,7 +255,7 @@ pub fn vars(output: &Output, build_state: &str) -> HashMap Result, globset::Error> { let mut result = Vec::new(); - let site_packages_base = if target_platform.is_windows() { - "Lib/site-packages" - } else if matches!(target_platform, Platform::NoArch) { + let site_packages_base = if version_independent { "site-packages" + } else if target_platform.is_windows() { + "Lib/site-packages" } else { "lib/python*/site-packages" }; @@ -214,10 +215,10 @@ impl PackageContentsTest { } /// Run the package content test - pub fn run_test(&self, paths: &PathsJson, target_platform: &Platform) -> Result<(), TestError> { + pub fn run_test(&self, paths: &PathsJson, output: &Output) -> Result<(), TestError> { let span = tracing::info_span!("Package content test"); let _enter = span.enter(); - + let target_platform = output.target_platform(); let paths = paths .paths .iter() @@ -227,7 +228,10 @@ impl PackageContentsTest { let include_globs = self.include_as_globs(target_platform)?; let bin_globs = self.bin_as_globs(target_platform)?; let lib_globs = self.lib_as_globs(target_platform)?; - let site_package_globs = self.site_packages_as_globs(target_platform)?; + let site_package_globs = self.site_packages_as_globs( + target_platform, + output.recipe.build().is_python_version_independent(), + )?; let file_globs = self.files_as_globs()?; fn match_glob<'a>(glob: &GlobSet, paths: &'a Vec<&PathBuf>) -> Vec<&'a PathBuf> { @@ -431,7 +435,9 @@ mod tests { if !tests.site_packages.is_empty() { println!("site_package globs: {:?}", tests.site_packages); - let globs = tests.site_packages_as_globs(&test_case.platform).unwrap(); + let globs = tests + .site_packages_as_globs(&test_case.platform, false) + .unwrap(); test_glob_matches(&globs, &test_case.paths)?; if !test_case.fail_paths.is_empty() { test_glob_matches(&globs, &test_case.fail_paths).unwrap_err(); diff --git a/src/packaging.rs b/src/packaging.rs index 7b50f6fba..d81413246 100644 --- a/src/packaging.rs +++ b/src/packaging.rs @@ -277,7 +277,7 @@ pub fn package_conda( tracing::info!("Creating entry points"); // create any entry points or link.json for noarch packages - if output.recipe.build().noarch().is_python() { + if output.recipe.build().is_python_version_independent() { let link_json = File::create(info_folder.join("link.json"))?; serde_json::to_writer_pretty(link_json, &output.link_json()?)?; tmp.add_files(vec![info_folder.join("link.json")]); diff --git a/src/packaging/file_mapper.rs b/src/packaging/file_mapper.rs index 49de2c260..d2792b081 100644 --- a/src/packaging/file_mapper.rs +++ b/src/packaging/file_mapper.rs @@ -99,7 +99,6 @@ impl Output { dest_folder: &Path, ) -> Result, PackagingError> { let target_platform = &self.build_configuration.target_platform; - let noarch_type = self.recipe.build().noarch(); let entry_points = &self.recipe.build().python().entry_points; let path_rel = path.strip_prefix(prefix)?; @@ -120,7 +119,7 @@ impl Output { } } - if noarch_type.is_python() { + if self.recipe.build().is_python_version_independent() { // we need to remove files in bin/ that are registered as entry points if path_rel.starts_with("bin") { if let Some(name) = path_rel.file_name() { diff --git a/src/packaging/metadata.rs b/src/packaging/metadata.rs index fe2ffc99f..591ed3d82 100644 --- a/src/packaging/metadata.rs +++ b/src/packaging/metadata.rs @@ -18,7 +18,7 @@ use rattler_conda_types::{ AboutJson, FileMode, IndexJson, LinkJson, NoArchLinks, PackageFile, PathType, PathsEntry, PathsJson, PrefixPlaceholder, PythonEntryPoints, RunExportsJson, }, - Platform, + NoArchType, Platform, }; use rattler_digest::{compute_bytes_digest, compute_file_digest}; @@ -250,7 +250,7 @@ impl Output { /// Create the contents of the index.json file for the given output. pub fn index_json(&self) -> Result { let recipe = &self.recipe; - let target_platform = self.build_configuration.target_platform; + let target_platform = self.target_platform(); let arch = target_platform.arch().map(|a| a.to_string()); let platform = target_platform.only_platform().map(|p| p.to_string()); @@ -283,6 +283,14 @@ impl Output { return Err(PackagingError::InvalidMetadata("Cannot set python_site_packages_path for a package that is not called `python`".to_string())); } } + + // Support CEP-20 / ABI3 packages + let noarch = if self.recipe.build().is_python_version_independent() { + NoArchType::python() + } else { + *self.recipe.build().noarch() + }; + Ok(IndexJson { name: self.name().clone(), version: self.version().clone().into(), @@ -308,7 +316,7 @@ impl Output { .map(|dep| dep.spec().to_string()) .dedup() .collect(), - noarch: *recipe.build().noarch(), + noarch, track_features, features: None, python_site_packages_path: recipe.build().python().site_packages_path.clone(), diff --git a/src/post_process/python.rs b/src/post_process/python.rs index b70ba2ec7..2961c0c41 100644 --- a/src/post_process/python.rs +++ b/src/post_process/python.rs @@ -158,7 +158,7 @@ pub fn python(temp_files: &TempFiles, output: &Output) -> Result &Vec { &self.post_process } + + /// The output is python version independent if the package is + /// `noarch: python` or the python version independent flag is set + /// which can also be true for `abi3` packages. + pub(crate) fn is_python_version_independent(&self) -> bool { + self.python().version_independent || self.noarch().is_python() + } } impl TryConvertNode for RenderedNode { @@ -505,6 +512,12 @@ pub struct Python { #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub use_python_app_entrypoint: bool, + /// Whether the package is Python version independent. + /// This is used for abi3 packages that are not tied to a specific Python version, but + /// still contain compiled code (and thus need to end up in the right subdir). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub version_independent: bool, + /// The relative site-packages path that a Python build _exports_ for other /// packages to use. This setting only makes sense for the `python` package /// itself. For example, a python 3.13 version could advertise a @@ -538,7 +551,8 @@ impl TryConvertNode for RenderedMappingNode { entry_points, skip_pyc_compilation, use_python_app_entrypoint, - site_packages_path + site_packages_path, + version_independent ); Ok(python) } diff --git a/src/recipe/parser/requirements.rs b/src/recipe/parser/requirements.rs index d7a857052..6f9f49b6c 100644 --- a/src/recipe/parser/requirements.rs +++ b/src/recipe/parser/requirements.rs @@ -522,10 +522,12 @@ impl TryConvertNode for RenderedMappingNode { /// Run exports to ignore #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct IgnoreRunExports { + /// Run exports to ignore by name of the package that is exported #[serde(default, skip_serializing_if = "IndexSet::is_empty")] - pub(super) by_name: IndexSet, + pub by_name: IndexSet, + /// Run exports to ignore by the package that applies them #[serde(default, skip_serializing_if = "IndexSet::is_empty")] - pub(super) from_package: IndexSet, + pub from_package: IndexSet, } impl IgnoreRunExports { diff --git a/src/recipe/snapshots/rattler_build__recipe__parser__tests__recipe_windows.snap b/src/recipe/snapshots/rattler_build__recipe__parser__tests__recipe_windows.snap index 3120f6481..824f8d758 100644 --- a/src/recipe/snapshots/rattler_build__recipe__parser__tests__recipe_windows.snap +++ b/src/recipe/snapshots/rattler_build__recipe__parser__tests__recipe_windows.snap @@ -126,6 +126,7 @@ Recipe { entry_points: [], skip_pyc_compilation: [], use_python_app_entrypoint: false, + version_independent: false, site_packages_path: None, }, dynamic_linking: DynamicLinking { diff --git a/src/recipe/snapshots/rattler_build__recipe__parser__tests__unix_recipe.snap b/src/recipe/snapshots/rattler_build__recipe__parser__tests__unix_recipe.snap index 788d560ef..ad20c9894 100644 --- a/src/recipe/snapshots/rattler_build__recipe__parser__tests__unix_recipe.snap +++ b/src/recipe/snapshots/rattler_build__recipe__parser__tests__unix_recipe.snap @@ -126,6 +126,7 @@ Recipe { entry_points: [], skip_pyc_compilation: [], use_python_app_entrypoint: false, + version_independent: false, site_packages_path: None, }, dynamic_linking: DynamicLinking { diff --git a/src/variant_config.rs b/src/variant_config.rs index 010f8972e..fce53ff49 100644 --- a/src/variant_config.rs +++ b/src/variant_config.rs @@ -440,7 +440,7 @@ impl VariantConfig { // Now we need to convert the stage 1 renders to DiscoveredOutputs let mut recipes = IndexSet::new(); for sx in stage_1 { - for ((node, recipe), variant) in sx.into_sorted_outputs()? { + for ((node, mut recipe), variant) in sx.into_sorted_outputs()? { let target_platform = if recipe.build().noarch().is_none() { selector_config.target_platform } else { @@ -454,6 +454,14 @@ impl VariantConfig { .expect("Build string has to be resolved") .to_string(); + if recipe.build().python().version_independent { + recipe + .requirements + .ignore_run_exports + .from_package + .insert("python".parse().unwrap()); + } + recipes.insert(DiscoveredOutput { name: recipe.package().name.as_normalized().to_string(), version: recipe.package().version.to_string(), diff --git a/src/variant_render.rs b/src/variant_render.rs index 4cb646d7b..6f968fe3c 100644 --- a/src/variant_render.rs +++ b/src/variant_render.rs @@ -352,7 +352,7 @@ pub(crate) fn stage_1_render( additional_variables.extend(extra_use_keys); // If the recipe is `noarch: python` we can remove an empty python key that comes from the dependencies - if output.build().noarch().is_python() { + if output.build().is_python_version_independent() { additional_variables.remove(&"python".into()); } diff --git a/test-data/recipes/abi3/recipe.yaml b/test-data/recipes/abi3/recipe.yaml new file mode 100644 index 000000000..39ee4288c --- /dev/null +++ b/test-data/recipes/abi3/recipe.yaml @@ -0,0 +1,45 @@ +package: + name: python-abi3-package-sample + version: 0.0.1 + +source: + url: https://github.com/joerick/python-abi3-package-sample/archive/6f74ae7b31e58ef5f8f09b647364854122e61155.tar.gz + sha256: e81fd4d4c4f5b7bc9786d9ee990afc659e14a25ce11182b7b69f826407cc1718 + +build: + number: 0 + python: + version_independent: true + script: ${{ PYTHON }} -m pip install . -vv + +requirements: + build: + - ${{ compiler('c') }} + host: + - python-abi3 + - python + - pip + - setuptools + run: + - python + +tests: + - python: + imports: + - spam + - script: + - export SP_DIR=$(python -c "import site; print(site.getsitepackages()[0])") + - abi3audit $SP_DIR/spam.abi3.so -s -v --assume-minimum-abi3 ${{ python_min }} + requirements: + run: + - abi3audit + +about: + homepage: https://github.com/joerick/python-abi3-package-sample + summary: 'ABI3 example' + license: Apache-2.0 + license_file: LICENSE + +extra: + recipe-maintainers: + - isuruf diff --git a/test-data/recipes/abi3/variants.yaml b/test-data/recipes/abi3/variants.yaml new file mode 100644 index 000000000..349fb8663 --- /dev/null +++ b/test-data/recipes/abi3/variants.yaml @@ -0,0 +1,2 @@ +python_min: + - "3.8" diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index a6c524cbf..a2537aa07 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -1200,3 +1200,30 @@ def test_cache_select_files(rattler_build: RattlerBuild, recipes: Path, tmp_path assert paths["paths"][0]["path_type"] == "softlink" assert paths["paths"][1]["_path"] == "lib/libdav1d.so.7.0.0" assert paths["paths"][1]["path_type"] == "hardlink" + + +@pytest.mark.skipif( + os.name == "nt", reason="recipe does not support execution on windows" +) +def test_abi3(rattler_build: RattlerBuild, recipes: Path, tmp_path: Path): + rattler_build.build(recipes / "abi3", tmp_path) + pkg = get_extracted_package(tmp_path, "python-abi3-package-sample") + + assert (pkg / "info/paths.json").exists() + paths = json.loads((pkg / "info/paths.json").read_text()) + # ensure that all paths start with `site-packages` + for p in paths["paths"]: + assert p["_path"].startswith("site-packages") + + actual_paths = [p["_path"] for p in paths["paths"]] + if os.name == "nt": + assert "site-packages\\spam.dll" in actual_paths + else: + assert "site-packages/spam.abi3.so" in actual_paths + + # load index.json + index = json.loads((pkg / "info/index.json").read_text()) + assert index["name"] == "python-abi3-package-sample" + assert index["noarch"] == "python" + assert index["subdir"] == host_subdir() + assert index["platform"] == host_subdir().split("-")[0]