From c025c51ca9f623f1f5287aa79edf0ff695ea485f Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 1 Nov 2023 19:56:30 -0500 Subject: [PATCH 1/8] Use MetaData's get_section & get_value --- conda_build/api.py | 4 +- conda_build/build.py | 32 +++++----- conda_build/create_test.py | 4 +- conda_build/metadata.py | 4 +- conda_build/post.py | 44 +++++++------- conda_build/render.py | 117 ++++++++++++++++++++----------------- 6 files changed, 105 insertions(+), 100 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index 2d5fa7ee7d..18f3639c5c 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -76,8 +76,8 @@ def render( raise # remove outputs section from output objects for simplicity - if not om.path and om.meta.get("outputs"): - om.parent_outputs = om.meta["outputs"] + if not om.path and om.get_section("outputs"): + om.parent_outputs = om.get_section("outputs") del om.meta["outputs"] output_metas[ diff --git a/conda_build/build.py b/conda_build/build.py index 1d66cf114f..ab826adb6e 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -151,7 +151,7 @@ def log_stats(stats_dict, descriptor): ) -def create_post_scripts(m): +def create_post_scripts(m: MetaData): """ Create scripts to run after build step """ @@ -162,12 +162,9 @@ def create_post_scripts(m): is_output = "package:" not in m.get_recipe_text() scriptname = tp if is_output: - if m.meta.get("build", {}).get(tp, ""): - scriptname = m.meta["build"][tp] - else: - scriptname = m.name() + "-" + tp + scriptname = m.get_value(f"build/{tp}", f"{m.name()}-{tp}") scriptname += ext - dst_name = "." + m.name() + "-" + tp + ext + dst_name = f".{m.name()}-{tp}{ext}" src = join(m.path, scriptname) if isfile(src): dst_dir = join( @@ -1456,12 +1453,12 @@ def write_about_json(m): json.dump(d, fo, indent=2, sort_keys=True) -def write_info_json(m): +def write_info_json(m: MetaData): info_index = m.info_index() if m.pin_depends: # Wtih 'strict' depends, we will have pinned run deps during rendering if m.pin_depends == "strict": - runtime_deps = m.meta.get("requirements", {}).get("run", []) + runtime_deps = m.get_value("requirements/run", []) info_index["depends"] = runtime_deps else: runtime_deps = environ.get_pinned_deps(m, "run") @@ -1508,8 +1505,8 @@ def get_entry_point_script_names(entry_point_scripts): return scripts -def write_run_exports(m): - run_exports = m.meta.get("build", {}).get("run_exports", {}) +def write_run_exports(m: MetaData): + run_exports = m.get_value("build/run_exports", {}) if run_exports: with open(os.path.join(m.config.info_dir, "run_exports.json"), "w") as f: if not hasattr(run_exports, "keys"): @@ -2317,7 +2314,7 @@ def _write_activation_text(script_path, m): fh.write(data) -def create_build_envs(m, notest): +def create_build_envs(m: MetaData, notest): build_ms_deps = m.ms_depends("build") build_ms_deps = [utils.ensure_valid_spec(spec) for spec in build_ms_deps] host_ms_deps = m.ms_depends("host") @@ -2371,11 +2368,12 @@ def create_build_envs(m, notest): try: if not notest: utils.insert_variant_versions( - m.meta.get("requirements", {}), m.config.variant, "run" + m.get_section("requirements"), m.config.variant, "run" ) - test_run_ms_deps = utils.ensure_list( - m.get_value("test/requires", []) - ) + utils.ensure_list(m.get_value("requirements/run", [])) + test_run_ms_deps = [ + *utils.ensure_list(m.get_value("test/requires", [])), + *utils.ensure_list(m.get_value("requirements/run", [])), + ] # make sure test deps are available before taking time to create build env environ.get_install_actions( m.config.test_prefix, @@ -2424,7 +2422,7 @@ def create_build_envs(m, notest): def build( - m, + m: MetaData, stats, post=None, need_source_download=True, @@ -2516,7 +2514,7 @@ def build( ) specs = [ms.spec for ms in m.ms_depends("build")] - if any(out.get("type") == "wheel" for out in m.meta.get("outputs", [])): + if any(out.get("type") == "wheel" for out in m.get_section("outputs")): specs.extend(["pip", "wheel"]) # TODO :: This is broken. It does not respect build/script for example and also if you need git diff --git a/conda_build/create_test.py b/conda_build/create_test.py index 45cb20ebfe..35511ef503 100644 --- a/conda_build/create_test.py +++ b/conda_build/create_test.py @@ -47,7 +47,7 @@ def _get_output_script_name( src_name = dst_name if m.is_output: src_name = "no-file" - for out in m.meta.get("outputs", []): + for out in m.get_section("outputs"): if m.name() == out.get("name"): out_test_script = out.get("test", {}).get("script", "no-file") if os.path.splitext(out_test_script)[1].lower() == ext: @@ -103,7 +103,7 @@ def _create_test_files( name = "" # the way this works is that each output needs to explicitly define a test script to run # They do not automatically pick up run_test.*, but can be pointed at that explicitly. - for out in m.meta.get("outputs", []): + for out in m.get_section("outputs"): if m.name() == out.get("name"): out_test_script = out.get("test", {}).get("script", "no-file") if out_test_script.endswith(ext): diff --git a/conda_build/metadata.py b/conda_build/metadata.py index d2d87912bf..29d3c40275 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -409,7 +409,7 @@ def ensure_matching_hashes(output_metadata): for _, m in output_metadata.values(): for _, om in output_metadata.values(): if m != om: - run_exports = om.meta.get("build", {}).get("run_exports", []) + run_exports = om.get_value("build/run_exports", []) if hasattr(run_exports, "keys"): run_exports_list = [] for export_type in utils.RUN_EXPORTS_TYPES: @@ -2574,7 +2574,7 @@ def get_output_metadata_set( ) output_d["requirements"] = output_d.get("requirements", {}) output_d["requirements"]["build"] = build_reqs - m.meta["requirements"] = m.meta.get("requirements", {}) + m.meta["requirements"] = m.get_section("requirements") m.meta["requirements"]["build"] = build_reqs non_conda_packages.append((output_d, m)) else: diff --git a/conda_build/post.py b/conda_build/post.py index bef71e31af..7be43cbe21 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -61,6 +61,8 @@ machofile, ) +from .metadata import MetaData + filetypes_for_platform = { "win": (DLLfile, EXEfile), "osx": (machofile,), @@ -1583,33 +1585,27 @@ def check_overlinking_impl( return dict() -def check_overlinking(m, files, host_prefix=None): - if not host_prefix: - host_prefix = m.config.host_prefix - - overlinking_ignore_patterns = m.meta.get("build", {}).get( - "overlinking_ignore_patterns" - ) - if overlinking_ignore_patterns: - files = [ - f - for f in files - if not any([fnmatch(f, p) for p in overlinking_ignore_patterns]) - ] +def check_overlinking(m: MetaData, files, host_prefix=None): + patterns = m.get_value("build/overlinking_ignore_patterns", []) + files = [ + file + for file in files + if not any([fnmatch(file, pattern) for pattern in patterns]) + ] return check_overlinking_impl( - m.get_value("package/name"), - m.get_value("package/version"), - m.get_value("build/string"), - m.get_value("build/number"), + m.name(), + m.version(), + m.build_id(), + m.build_number(), m.config.target_subdir, m.get_value("build/ignore_run_exports"), - [req.split(" ")[0] for req in m.meta.get("requirements", {}).get("run", [])], - [req.split(" ")[0] for req in m.meta.get("requirements", {}).get("build", [])], - [req.split(" ")[0] for req in m.meta.get("requirements", {}).get("host", [])], - host_prefix, + [req.split(" ")[0] for req in m.get_value("requirements/run", [])], + [req.split(" ")[0] for req in m.get_value("requirements/build", [])], + [req.split(" ")[0] for req in m.get_value("requirements/host", [])], + host_prefix or m.config.host_prefix, m.config.build_prefix, - m.meta.get("build", {}).get("missing_dso_whitelist", []), - m.meta.get("build", {}).get("runpath_whitelist", []), + m.get_value("build/missing_dso_whitelist", []), + m.get_value("build/runpath_whitelist", []), m.config.error_overlinking, m.config.error_overdepending, m.config.verbose, @@ -1617,7 +1613,7 @@ def check_overlinking(m, files, host_prefix=None): files, m.config.bldpkgs_dir, m.config.output_folder, - list(m.config.channel_urls) + ["local"], + [*m.config.channel_urls, "local"], m.config.enable_static, m.config.variant, ) diff --git a/conda_build/render.py b/conda_build/render.py index fa428e07f6..3a70ef47ac 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -13,7 +13,15 @@ import tempfile from collections import OrderedDict, defaultdict from functools import lru_cache -from os.path import abspath, isdir, isfile +from os.path import ( + abspath, + dirname, + isabs, + isdir, + isfile, + join, + normpath, +) from pathlib import Path import yaml @@ -67,15 +75,17 @@ def bldpkg_path(m): # the default case will switch over to conda_v2 at some point if pkg_type == "conda": - path = os.path.join( + path = join( m.config.output_folder, subdir, f"{m.dist()}{CONDA_PACKAGE_EXTENSION_V1}" ) elif pkg_type == "conda_v2": - path = os.path.join( + path = join( m.config.output_folder, subdir, f"{m.dist()}{CONDA_PACKAGE_EXTENSION_V2}" ) else: - path = f"{m.type} file for {m.name()} in: {os.path.join(m.config.output_folder, subdir)}" + path = ( + f"{m.type} file for {m.name()} in: {join(m.config.output_folder, subdir)}" + ) return path @@ -118,7 +128,7 @@ def _categorize_deps(m, specs, exclude_pattern, variant): def get_env_dependencies( - m, + m: MetaData, env, variant, exclude_pattern=None, @@ -178,7 +188,7 @@ def get_env_dependencies( return ( utils.ensure_list( (specs + subpackages + pass_through_deps) - or m.meta.get("requirements", {}).get(env, []) + or m.get_value(f"requirements/{env}", []) ), actions, unsat, @@ -278,19 +288,19 @@ def find_pkg_dir_or_file_in_pkgs_dirs( @lru_cache(maxsize=None) def _read_specs_from_package(pkg_loc, pkg_dist): specs = {} - if pkg_loc and os.path.isdir(pkg_loc): - downstream_file = os.path.join(pkg_loc, "info/run_exports") - if os.path.isfile(downstream_file): + if pkg_loc and isdir(pkg_loc): + downstream_file = join(pkg_loc, "info/run_exports") + if isfile(downstream_file): with open(downstream_file) as f: specs = {"weak": [spec.rstrip() for spec in f.readlines()]} # a later attempt: record more info in the yaml file, to support "strong" run exports - elif os.path.isfile(downstream_file + ".yaml"): + elif isfile(downstream_file + ".yaml"): with open(downstream_file + ".yaml") as f: specs = yaml.safe_load(f) - elif os.path.isfile(downstream_file + ".json"): + elif isfile(downstream_file + ".json"): with open(downstream_file + ".json") as f: specs = json.load(f) - if not specs and pkg_loc and os.path.isfile(pkg_loc): + if not specs and pkg_loc and isfile(pkg_loc): # switching to json for consistency in conda-build 4 specs_yaml = utils.package_has_file(pkg_loc, "info/run_exports.yaml") specs_json = utils.package_has_file(pkg_loc, "info/run_exports.json") @@ -384,8 +394,8 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files with utils.LoggingContext(): pfe.execute() for pkg_dir in pkgs_dirs: - _loc = os.path.join(pkg_dir, index.get(pkg, pkg).fn) - if os.path.isfile(_loc): + _loc = join(pkg_dir, index.get(pkg, pkg).fn) + if isfile(_loc): pkg_loc = _loc break pkg_files[pkg] = pkg_loc, pkg_dist @@ -393,11 +403,11 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files return pkg_files -def get_upstream_pins(m, actions, env): +def get_upstream_pins(m: MetaData, actions, env): """Download packages from specs, then inspect each downloaded package for additional downstream dependency specs. Return these additional specs.""" - env_specs = m.meta.get("requirements", {}).get(env, []) + env_specs = m.get_value(f"requirements/{env}", []) explicit_specs = [req.split(" ")[0] for req in env_specs] if env_specs else [] linked_packages = actions.get("LINK", []) linked_packages = [pkg for pkg in linked_packages if pkg.name in explicit_specs] @@ -427,7 +437,12 @@ def get_upstream_pins(m, actions, env): return additional_specs -def _read_upstream_pin_files(m, env, permit_unsatisfiable_variants, exclude_pattern): +def _read_upstream_pin_files( + m: MetaData, + env, + permit_unsatisfiable_variants, + exclude_pattern, +): deps, actions, unsat = get_env_dependencies( m, env, @@ -439,16 +454,16 @@ def _read_upstream_pin_files(m, env, permit_unsatisfiable_variants, exclude_patt # vc feature activation to work correctly in the host env. extra_run_specs = get_upstream_pins(m, actions, env) return ( - list(set(deps)) or m.meta.get("requirements", {}).get(env, []), + list(set(deps)) or m.get_value(f"requirements/{env}", []), unsat, extra_run_specs, ) -def add_upstream_pins(m, permit_unsatisfiable_variants, exclude_pattern): +def add_upstream_pins(m: MetaData, permit_unsatisfiable_variants, exclude_pattern): """Applies run_exports from any build deps to host and run sections""" # if we have host deps, they're more important than the build deps. - requirements = m.meta.get("requirements", {}) + requirements = m.get_section("requirements") build_deps, build_unsat, extra_run_specs_from_build = _read_upstream_pin_files( m, "build", permit_unsatisfiable_variants, exclude_pattern ) @@ -464,7 +479,7 @@ def add_upstream_pins(m, permit_unsatisfiable_variants, exclude_pattern): if not host_reqs: matching_output = [ - out for out in m.meta.get("outputs", []) if out.get("name") == m.name() + out for out in m.get_section("outputs") if out.get("name") == m.name() ] if matching_output: requirements = utils.expand_reqs( @@ -580,7 +595,11 @@ def _simplify_to_exact_constraints(metadata): metadata.meta["requirements"] = requirements -def finalize_metadata(m, parent_metadata=None, permit_unsatisfiable_variants=False): +def finalize_metadata( + m: MetaData, + parent_metadata=None, + permit_unsatisfiable_variants=False, +): """Fully render a recipe. Fill in versions for build/host dependencies.""" if not parent_metadata: parent_metadata = m @@ -605,7 +624,7 @@ def finalize_metadata(m, parent_metadata=None, permit_unsatisfiable_variants=Fal ) ) - parent_recipe = m.meta.get("extra", {}).get("parent_recipe", {}) + parent_recipe = m.get_value("extra/parent_recipe", {}) # extract the topmost section where variables are defined, and put it on top of the # requirements for a particular output @@ -625,12 +644,12 @@ def finalize_metadata(m, parent_metadata=None, permit_unsatisfiable_variants=Fal requirements = utils.expand_reqs(output.get("requirements", {})) m.meta["requirements"] = requirements - if m.meta.get("requirements"): + if m.get_section("requirements"): utils.insert_variant_versions( - m.meta["requirements"], m.config.variant, "build" + m.get_section("requirements"), m.config.variant, "build" ) utils.insert_variant_versions( - m.meta["requirements"], m.config.variant, "host" + m.get_section("requirements"), m.config.variant, "host" ) m = parent_metadata.get_output_metadata(m.get_rendered_output(m.name())) @@ -639,7 +658,7 @@ def finalize_metadata(m, parent_metadata=None, permit_unsatisfiable_variants=Fal ) # getting this AFTER add_upstream_pins is important, because that function adds deps # to the metadata. - requirements = m.meta.get("requirements", {}) + requirements = m.get_section("requirements") # here's where we pin run dependencies to their build time versions. This happens based # on the keys in the 'pin_run_as_build' key in the variant, which is a list of package @@ -700,34 +719,26 @@ def finalize_metadata(m, parent_metadata=None, permit_unsatisfiable_variants=Fal utils.ensure_valid_spec(spec, warn=True) for spec in versioned_test_deps ] m.meta["test"]["requires"] = versioned_test_deps - extra = m.meta.get("extra", {}) + extra = m.get_section("extra") extra["copy_test_source_files"] = m.config.copy_test_source_files m.meta["extra"] = extra # if source/path is relative, then the output package makes no sense at all. The next # best thing is to hard-code the absolute path. This probably won't exist on any # system other than the original build machine, but at least it will work there. - if m.meta.get("source"): - if "path" in m.meta["source"]: - source_path = m.meta["source"]["path"] - os.path.expanduser(source_path) - if not os.path.isabs(source_path): - m.meta["source"]["path"] = os.path.normpath( - os.path.join(m.path, source_path) - ) - elif "git_url" in m.meta["source"] and not ( - # absolute paths are not relative paths - os.path.isabs(m.meta["source"]["git_url"]) - or - # real urls are not relative paths - ":" in m.meta["source"]["git_url"] - ): - m.meta["source"]["git_url"] = os.path.normpath( - os.path.join(m.path, m.meta["source"]["git_url"]) - ) - - if not m.meta.get("build"): - m.meta["build"] = {} + if source_path := m.get_value("source/path"): + if not isabs(source_path): + m.meta["source"]["path"] = normpath(join(m.path, source_path)) + elif ( + (git_url := m.get_value("source/git_url")) + # absolute paths are not relative paths + and not isabs(git_url) + # real urls are not relative paths + and ":" not in git_url + ): + m.meta["source"]["git_url"] = normpath(join(m.path, git_url)) + + m.meta.setdefault("build", {}) _simplify_to_exact_constraints(m) @@ -953,7 +964,7 @@ def render_recipe( t.close() need_cleanup = True elif arg.endswith(".yaml"): - recipe_dir = os.path.dirname(arg) + recipe_dir = dirname(arg) need_cleanup = False else: print("Ignoring non-recipe: %s" % arg) @@ -987,9 +998,9 @@ def render_recipe( if m.final: if not hasattr(m.config, "variants") or not m.config.variant: m.config.ignore_system_variants = True - if os.path.isfile(os.path.join(m.path, "conda_build_config.yaml")): + if isfile(join(m.path, "conda_build_config.yaml")): m.config.variant_config_files = [ - os.path.join(m.path, "conda_build_config.yaml") + join(m.path, "conda_build_config.yaml") ] m.config.variants = get_package_variants(m, variants=variants) m.config.variant = m.config.variants[0] @@ -1076,7 +1087,7 @@ def output_yaml(metadata, filename=None, suppress_outputs=False): if filename: if any(sep in filename for sep in ("\\", "/")): try: - os.makedirs(os.path.dirname(filename)) + os.makedirs(dirname(filename)) except OSError: pass with open(filename, "w") as f: From 331c1e836955ea085abdff4d40aa92d644ae15c7 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 1 Nov 2023 22:15:14 -0500 Subject: [PATCH 2/8] Additional cleanup --- conda_build/api.py | 14 ++++++-------- conda_build/build.py | 14 +++++++------- conda_build/environ.py | 5 +++-- conda_build/skeletons/cran.py | 29 ++++++++++++++++++----------- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index 18f3639c5c..10cef41e86 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -8,6 +8,7 @@ Design philosophy: put variability into config. Make each function here accept kwargs, but only use those kwargs in config. Config must change to support new features elsewhere. """ +from __future__ import annotations import sys as _sys @@ -571,7 +572,7 @@ def debug( test=False, output_id=None, config=None, - verbose=True, + verbose: bool = True, link_source_method="auto", **kwargs, ): @@ -587,6 +588,8 @@ def debug( from conda_build.build import test as run_test from conda_build.utils import CONDA_PACKAGE_EXTENSIONS, LoggingContext, on_win + from .metadata import MetaData + is_package = False default_config = get_or_merge_config(config, **kwargs) args = {"set_build_id": False} @@ -622,15 +625,13 @@ def debug( config.channel_urls = get_channel_urls(kwargs) - metadata_tuples = [] + metadata_tuples: list[tuple[MetaData, bool, bool]] = [] best_link_source_method = "skip" if isinstance(recipe_or_package_path_or_metadata_tuples, str): if path_is_build_dir: for metadata_conda_debug in metadatas_conda_debug: best_link_source_method = "symlink" - from conda_build.metadata import MetaData - metadata = MetaData(metadata_conda_debug, config, {}) metadata_tuples.append((metadata, False, True)) else: @@ -681,10 +682,7 @@ def debug( "local", "src", "conda", - "{}-{}".format( - metadata.get_value("package/name"), - metadata.get_value("package/version"), - ), + f"{metadata.name()}-{metadata.version()}", ) link_target = os.path.dirname(metadata.meta_path) try: diff --git a/conda_build/build.py b/conda_build/build.py index ab826adb6e..31170c565c 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -1744,8 +1744,8 @@ def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix): return checksums -def post_process_files(m, initial_prefix_files): - package_name = m.get_value("package/name") +def post_process_files(m: MetaData, initial_prefix_files): + package_name = m.name() host_prefix = m.config.host_prefix missing = [] for f in initial_prefix_files: @@ -1775,7 +1775,7 @@ def post_process_files(m, initial_prefix_files): ) post_process( package_name, - m.get_value("package/version"), + m.version(), sorted(current_prefix_files - initial_prefix_files), prefix=host_prefix, config=m.config, @@ -1836,7 +1836,7 @@ def post_process_files(m, initial_prefix_files): return new_files -def bundle_conda(output, metadata, env, stats, **kw): +def bundle_conda(output, metadata: MetaData, env, stats, **kw): log = utils.get_logger(__name__) log.info("Packaging %s", metadata.dist()) get_all_replacements(metadata.config) @@ -1908,7 +1908,7 @@ def bundle_conda(output, metadata, env, stats, **kw): env_output["TOP_PKG_NAME"] = env["PKG_NAME"] env_output["TOP_PKG_VERSION"] = env["PKG_VERSION"] env_output["PKG_VERSION"] = metadata.version() - env_output["PKG_NAME"] = metadata.get_value("package/name") + env_output["PKG_NAME"] = metadata.name() env_output["RECIPE_DIR"] = metadata.path env_output["MSYS2_PATH_TYPE"] = "inherit" env_output["CHERE_INVOKING"] = "1" @@ -2126,7 +2126,7 @@ def bundle_conda(output, metadata, env, stats, **kw): return final_outputs -def bundle_wheel(output, metadata, env, stats): +def bundle_wheel(output, metadata: MetaData, env, stats): ext = ".bat" if utils.on_win else ".sh" with TemporaryDirectory() as tmpdir, utils.tmp_chdir(metadata.config.work_dir): dest_file = os.path.join(metadata.config.work_dir, "wheel_output" + ext) @@ -2142,7 +2142,7 @@ def bundle_wheel(output, metadata, env, stats): env["TOP_PKG_NAME"] = env["PKG_NAME"] env["TOP_PKG_VERSION"] = env["PKG_VERSION"] env["PKG_VERSION"] = metadata.version() - env["PKG_NAME"] = metadata.get_value("package/name") + env["PKG_NAME"] = metadata.name() interpreter_and_args = guess_interpreter(dest_file) bundle_stats = {} diff --git a/conda_build/environ.py b/conda_build/environ.py index 5afcf93c4d..7a7acb7252 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -42,6 +42,7 @@ root_dir, ) from .deprecations import deprecated +from .metadata import MetaData # these are things that we provide env vars for more explicitly. This list disables the # pass-through of variant values to env vars for these keys. @@ -484,7 +485,7 @@ def r_vars(metadata, prefix, escape_backslash): return vars_ -def meta_vars(meta, skip_build_id=False): +def meta_vars(meta: MetaData, skip_build_id=False): d = {} for var_name in ensure_list(meta.get_value("build/script_env", [])): if "=" in var_name: @@ -546,7 +547,7 @@ def meta_vars(meta, skip_build_id=False): d.update(get_hg_build_info(hg_dir)) # use `get_value` to prevent early exit while name is still unresolved during rendering - d["PKG_NAME"] = meta.get_value("package/name") + d["PKG_NAME"] = meta.name() d["PKG_VERSION"] = meta.version() d["PKG_BUILDNUM"] = str(meta.build_number()) if meta.final and not skip_build_id: diff --git a/conda_build/skeletons/cran.py b/conda_build/skeletons/cran.py index e3b22ef7d2..cd093e6d9e 100755 --- a/conda_build/skeletons/cran.py +++ b/conda_build/skeletons/cran.py @@ -3,7 +3,7 @@ """ Tools for converting Cran packages to conda recipes. """ - +from __future__ import annotations import argparse import copy @@ -28,6 +28,7 @@ realpath, relpath, ) +from typing import Literal import requests import yaml @@ -40,13 +41,15 @@ from conda.common.io import dashlist -from conda_build import metadata, source +from conda_build import source from conda_build.conda_interface import TemporaryDirectory, cc_conda_build from conda_build.config import get_or_merge_config from conda_build.license_family import allowed_license_families, guess_license_family from conda_build.utils import ensure_list, rm_rf from conda_build.variants import DEFAULT_VARIANTS, get_package_variants +from ..metadata import MetaData + SOURCE_META = """\ {archive_keys} {git_url_key} {git_url} @@ -736,7 +739,9 @@ def strip_end(string, end): return string -def package_to_inputs_dict(output_dir, output_suffix, git_tag, package, version=None): +def package_to_inputs_dict( + output_dir, output_suffix, git_tag, package: str, version=None +): """ Converts `package` (*) into a tuple of: @@ -802,9 +807,10 @@ def package_to_inputs_dict(output_dir, output_suffix, git_tag, package, version= location = existing_location = existing_recipe_dir( output_dir, output_suffix, package, version ) + m: MetaData | None if existing_location: try: - m = metadata.MetaData(existing_location) + m = MetaData(existing_location) except: # Happens when the folder exists but contains no recipe. m = None @@ -868,7 +874,7 @@ def skeletonize( r_interp="r-base", use_binaries_ver=None, use_noarch_generic=False, - use_when_no_binary="src", + use_when_no_binary: Literal["error" | "src" | "old" | "old-src"] = "src", use_rtools_win=False, config=None, variant_config_files=None, @@ -884,6 +890,9 @@ def skeletonize( ): print(f"ERROR: --use_when_no_binary={use_when_no_binary} not yet implemented") sys.exit(1) + + m: MetaData + output_dir = realpath(output_dir) config = get_or_merge_config(config, variant_config_files=variant_config_files) @@ -970,9 +979,7 @@ def skeletonize( elif is_github_url or is_tarfile: rm_rf(config.work_dir) - m = metadata.MetaData.fromdict( - {"source": {"git_url": location}}, config=config - ) + m = MetaData.fromdict({"source": {"git_url": location}}, config=config) source.git_source( m.get_section("source"), m.config.git_cache, m.config.work_dir ) @@ -1088,7 +1095,7 @@ def skeletonize( m, "extra/recipe-maintainers", add_maintainer ) if m.version() == d["conda_version"]: - build_number = int(m.get_value("build/number", 0)) + build_number = m.build_number() build_number += 1 if update_policy == "merge-incr-build-num" else 0 if add_maintainer: new_maintainer = "{indent}{add_maintainer}".format( @@ -1695,8 +1702,8 @@ def skeletonize( ) -def version_compare(recipe_dir, newest_conda_version): - m = metadata.MetaData(recipe_dir) +def version_compare(recipe_dir: str, newest_conda_version): + m = MetaData(recipe_dir) local_version = m.version() package = basename(recipe_dir) From a015f99201ce115b34a43a8dfbd42c5fa03b4901 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 1 Nov 2023 22:23:43 -0500 Subject: [PATCH 3/8] Resolve circular import --- conda_build/metadata.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 29d3c40275..e5a76d2dd4 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -16,7 +16,7 @@ from bs4 import UnicodeDammit -from conda_build import environ, exceptions, utils, variants +from conda_build import exceptions, utils, variants from conda_build.config import Config, get_or_merge_config from conda_build.features import feature_list from conda_build.license_family import ensure_valid_license_family @@ -1891,8 +1891,10 @@ def _get_contents( loader = FilteredLoader(jinja2.ChoiceLoader(loaders), config=self.config) env = jinja2.Environment(loader=loader, undefined=undefined_type) + from .environ import get_dict + env.globals.update(get_selectors(self.config)) - env.globals.update(environ.get_dict(m=self, skip_build_id=skip_build_id)) + env.globals.update(get_dict(m=self, skip_build_id=skip_build_id)) env.globals.update({"CONDA_BUILD_STATE": "RENDER"}) env.globals.update( context_processor( From d8f68d841899209c9f012ce8818749fe8669d5c8 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 1 Nov 2023 23:09:52 -0500 Subject: [PATCH 4/8] Additional cleanup --- conda_build/environ.py | 3 +- conda_build/metadata.py | 218 +++++++++++++++++++--------------------- 2 files changed, 107 insertions(+), 114 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 7a7acb7252..5b02dbeaaa 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -546,8 +546,7 @@ def meta_vars(meta: MetaData, skip_build_id=False): ): d.update(get_hg_build_info(hg_dir)) - # use `get_value` to prevent early exit while name is still unresolved during rendering - d["PKG_NAME"] = meta.name() + d["PKG_NAME"] = meta.name(fail_ok=True) d["PKG_VERSION"] = meta.version() d["PKG_BUILDNUM"] = str(meta.build_number()) if meta.final and not skip_build_id: diff --git a/conda_build/metadata.py b/conda_build/metadata.py index e5a76d2dd4..75ec38e061 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -13,6 +13,7 @@ from collections import OrderedDict from functools import lru_cache from os.path import isfile, join +from typing import Any, Literal from bs4 import UnicodeDammit @@ -45,6 +46,8 @@ except AttributeError: Loader = yaml.Loader +log = utils.get_logger(__name__) + class StringifyNumbersLoader(Loader): @classmethod @@ -182,7 +185,7 @@ def get_selectors(config: Config) -> dict[str, bool]: if not np: np = defaults["numpy"] if config.verbose: - utils.get_logger(__name__).warn( + log.warn( "No numpy version specified in conda_build_config.yaml. " "Falling back to default numpy value of {}".format(defaults["numpy"]) ) @@ -256,7 +259,6 @@ def eval_selector(selector_string, namespace, variants_in_place): except NameError as e: missing_var = parseNameNotFound(e) if variants_in_place: - log = utils.get_logger(__name__) log.debug( "Treating unknown selector '" + missing_var + "' as if it was False." ) @@ -324,7 +326,6 @@ def ensure_valid_fields(meta): def _trim_None_strings(meta_dict): - log = utils.get_logger(__name__) for key, value in meta_dict.items(): if hasattr(value, "keys"): meta_dict[key] = _trim_None_strings(value) @@ -910,7 +911,6 @@ def finalize_outputs_pass( if metadata.skip(): continue try: - log = utils.get_logger(__name__) # We should reparse the top-level recipe to get all of our dependencies fixed up. # we base things on base_metadata because it has the record of the full origin recipe if base_metadata.config.verbose: @@ -967,7 +967,6 @@ def finalize_outputs_pass( if not permit_unsatisfiable_variants: raise else: - log = utils.get_logger(__name__) log.warn( "Could not finalize metadata due to missing dependencies: " "{}".format(e.packages) @@ -1123,33 +1122,31 @@ def __init__(self, path, config=None, variant=None): # establish whether this recipe should squish build and host together @property - def is_cross(self): - return bool(self.get_depends_top_and_out("host")) or "host" in self.meta.get( - "requirements", {} + def is_cross(self) -> bool: + return bool( + self.get_depends_top_and_out("host") + or "host" in self.get_section("requirements") ) @property - def final(self): + def final(self) -> bool: return self.get_value("extra/final") @final.setter - def final(self, boolean): - extra = self.meta.get("extra", {}) - extra["final"] = boolean - self.meta["extra"] = extra + def final(self, value: bool) -> None: + self.meta.setdefault("extra", {})["final"] = bool(value) @property - def disable_pip(self): + def disable_pip(self) -> bool: return self.config.disable_pip or ( "build" in self.meta and "disable_pip" in self.meta["build"] ) @disable_pip.setter - def disable_pip(self, value): - self.config.disable_pip = value - build = self.meta.get("build", {}) - build["disable_pip"] = value - self.meta["build"] = build + def disable_pip(self, value: bool) -> None: + self.config.disable_pip = self.meta.setdefault("build", {})[ + "disable_pip" + ] = bool(value) def append_metadata_sections( self, sections_file_or_dict, merge, raise_on_clobber=False @@ -1175,10 +1172,9 @@ def append_metadata_sections( ) @property - def is_output(self): - self_name = self.name(fail_ok=True) - parent_name = self.meta.get("extra", {}).get("parent_recipe", {}).get("name") - return bool(parent_name) and parent_name != self_name + def is_output(self) -> str: + parent_name = self.get_value("extra/parent_recipe", {}).get("name") + return parent_name and parent_name != self.name(fail_ok=True) def parse_again( self, @@ -1198,7 +1194,6 @@ def parse_again( """ assert not self.final, "modifying metadata after finalization" - log = utils.get_logger(__name__) if kw: log.warn( "using unsupported internal conda-build function `parse_again`. Please use " @@ -1245,17 +1240,18 @@ def parse_again( dependencies = _get_dependencies_from_environment(self.config.bootstrap) self.append_metadata_sections(dependencies, merge=True) - if "error_overlinking" in self.meta.get("build", {}): - self.config.error_overlinking = self.meta["build"]["error_overlinking"] - if "error_overdepending" in self.meta.get("build", {}): - self.config.error_overdepending = self.meta["build"]["error_overdepending"] + if "error_overlinking" in self.get_section("build"): + self.config.error_overlinking = self.get_value("build/error_overlinking") + if "error_overdepending" in self.get_section("build"): + self.config.error_overdepending = self.get_value( + "build/error_overdepending" + ) self.validate_features() self.ensure_no_pip_requirements() def ensure_no_pip_requirements(self): - keys = "requirements/build", "requirements/run", "test/requires" - for key in keys: + for key in ("requirements/build", "requirements/run", "test/requires"): if any(hasattr(item, "keys") for item in (self.get_value(key) or [])): raise ValueError( "Dictionaries are not supported as values in requirements sections" @@ -1265,15 +1261,13 @@ def ensure_no_pip_requirements(self): def append_requirements(self): """For dynamic determination of build or run reqs, based on configuration""" - reqs = self.meta.get("requirements", {}) - run_reqs = reqs.get("run", []) + run_reqs = self.meta.setdefault("requirements", {}).setdefault("run", []) if ( - bool(self.get_value("build/osx_is_app", False)) + self.get_value("build/osx_is_app", False) and self.config.platform == "osx" + and "python.app" not in run_reqs ): - if "python.app" not in run_reqs: - run_reqs.append("python.app") - self.meta["requirements"] = reqs + run_reqs.append("python.app") def parse_until_resolved( self, allow_no_other_outputs=False, bypass_env_check=False @@ -1350,10 +1344,10 @@ def fromdict(cls, metadata, config=None, variant=None): return m - def get_section(self, section): + def get_section(self, section: str): return self.meta.get(section, {}) - def get_value(self, name, default=None, autotype=True): + def get_value(self, name: str, default=None, autotype=True): """ Get a value from a meta.yaml. :param field: Field to return, e.g. 'package/name'. @@ -1387,7 +1381,6 @@ def get_value(self, name, default=None, autotype=True): # The 'source' section can be written a list, in which case the name # is passed in with an index, e.g. get_value('source/0/git_url') if index is None: - log = utils.get_logger(__name__) log.warn( f"No index specified in get_value('{name}'). Assuming index 0." ) @@ -1436,28 +1429,29 @@ def check_field(key, section): check_field(key_or_dict, section) return True - def name(self, fail_ok=False): - res = self.meta.get("package", {}).get("name", "") - if not res and not fail_ok: + def name(self, fail_ok: bool = False) -> str: + name = self.get_value("package/name", "") + if not name and not fail_ok: sys.exit("Error: package/name missing in: %r" % self.meta_path) - res = str(res) - if res != res.lower(): - sys.exit("Error: package/name must be lowercase, got: %r" % res) - check_bad_chrs(res, "package/name") - return res - - def version(self): - res = str(self.get_value("package/version")) - if res is None: + name = str(name) + if name != name.lower(): + sys.exit("Error: package/name must be lowercase, got: %r" % name) + check_bad_chrs(name, "package/name") + return name + + def version(self) -> str: + version = self.get_value("package/version") + if not version: sys.exit("Error: package/version missing in: %r" % self.meta_path) - check_bad_chrs(res, "package/version") - if self.final and res.startswith("."): + version = str(version) + check_bad_chrs(version, "package/version") + if self.final and version.startswith("."): raise ValueError( - "Fully-rendered version can't start with period - got %s", res + "Fully-rendered version can't start with period - got %s", version ) - return res + return version - def build_number(self): + def build_number(self) -> int: number = self.get_value("build/number") # build number can come back as None if no setting (or jinja intermediate) @@ -1633,7 +1627,7 @@ def hash_dependencies(self): return _hash_dependencies(hashing_dependencies, self.config.hash_length) return hash_ - def build_id(self): + def build_id(self) -> str: manual_build_string = self.get_value("build/string") # we need the raw recipe for this metadata (possibly an output), so that we can say whether # PKG_HASH is used for anything. @@ -1669,16 +1663,16 @@ def build_id(self): out = re.sub("h[0-9a-f]{%s}" % self.config.hash_length, hash_, out) return out - def dist(self): + def dist(self) -> str: return f"{self.name()}-{self.version()}-{self.build_id()}" - def pkg_fn(self): + def pkg_fn(self) -> str: return "%s.tar.bz2" % self.dist() - def is_app(self): + def is_app(self) -> bool: return bool(self.get_value("app/entry")) - def app_meta(self): + def app_meta(self) -> dict[str, Any]: d = {"type": "app"} if self.get_value("app/icon"): d["icon"] = "%s.png" % md5_file(join(self.path, self.get_value("app/icon"))) @@ -1809,8 +1803,8 @@ def binary_relocation(self): expand_globs(ret, self.config.host_prefix) if isinstance(ret, list) else ret ) - def include_recipe(self): - return self.get_value("build/include_recipe", True) + def include_recipe(self) -> bool: + return bool(self.get_value("build/include_recipe", True)) def binary_has_prefix_files(self): ret = ensure_list(self.get_value("build/binary_has_prefix_files", [])) @@ -1826,8 +1820,8 @@ def binary_has_prefix_files(self): ) return expand_globs(ret, self.config.host_prefix) - def skip(self): - return self.get_value("build/skip", False) + def skip(self) -> bool: + return bool(self.get_value("build/skip", False)) def _get_contents( self, @@ -1947,34 +1941,39 @@ def _get_contents( finally: if "CONDA_BUILD_STATE" in os.environ: del os.environ["CONDA_BUILD_STATE"] + + log.debug(f"{path=}") + log.debug(f"{filename=}") + log.debug(f"{rendered=}") + return rendered - def __unicode__(self): + def __unicode__(self) -> str: """ String representation of the MetaData. """ return str(self.__dict__) - def __str__(self): + def __str__(self) -> str: return self.__unicode__() - def __repr__(self): + def __repr__(self) -> str: """ String representation of the MetaData. """ return self.__str__() @property - def meta_path(self): - meta_path = self._meta_path or self.meta.get("extra", {}).get( - "parent_recipe", {} - ).get("path", "") + def meta_path(self) -> str: + meta_path = self._meta_path or self.get_value("extra/parent_recipe", {}).get( + "path", "" + ) if meta_path and os.path.basename(meta_path) != self._meta_name: meta_path = os.path.join(meta_path, self._meta_name) return meta_path @property - def uses_setup_py_in_meta(self): + def uses_setup_py_in_meta(self) -> bool: meta_text = "" if self.meta_path: with open(self.meta_path, "rb") as f: @@ -1982,7 +1981,7 @@ def uses_setup_py_in_meta(self): return "load_setup_py_data" in meta_text or "load_setuptools" in meta_text @property - def uses_regex_in_meta(self): + def uses_regex_in_meta(self) -> bool: meta_text = "" if self.meta_path: with open(self.meta_path, "rb") as f: @@ -1990,7 +1989,7 @@ def uses_regex_in_meta(self): return "load_file_regex" in meta_text @property - def uses_load_file_data_in_meta(self): + def uses_load_file_data_in_meta(self) -> bool: meta_text = "" if self.meta_path: with open(self.meta_path, "rb") as f: @@ -1998,7 +1997,7 @@ def uses_load_file_data_in_meta(self): return "load_file_data" in meta_text @property - def needs_source_for_render(self): + def needs_source_for_render(self) -> bool: return ( self.uses_vcs_in_meta or self.uses_setup_py_in_meta @@ -2007,7 +2006,7 @@ def needs_source_for_render(self): ) @property - def uses_jinja(self): + def uses_jinja(self) -> bool: if not self.meta_path: return False with open(self.meta_path, "rb") as f: @@ -2016,7 +2015,7 @@ def uses_jinja(self): return len(matches) > 0 @property - def uses_vcs_in_meta(self): + def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None: """returns name of vcs used if recipe contains metadata associated with version control systems. If this metadata is present, a download/copy will be forced in parse_or_try_download. """ @@ -2028,7 +2027,7 @@ def uses_vcs_in_meta(self): meta_text = UnicodeDammit(f.read()).unicode_markup for _vcs in vcs_types: matches = re.findall(rf"{_vcs.upper()}_[^\.\s\'\"]+", meta_text) - if len(matches) > 0 and _vcs != self.meta["package"]["name"]: + if len(matches) > 0 and _vcs != self.get_value("package/name"): if _vcs == "hg": _vcs = "mercurial" vcs = _vcs @@ -2036,7 +2035,7 @@ def uses_vcs_in_meta(self): return vcs @property - def uses_vcs_in_build(self): + def uses_vcs_in_build(self) -> Literal["git" | "svn" | "mercurial"] | None: # TODO :: Re-work this. Is it even useful? We can declare any vcs in our build deps. build_script = "bld.bat" if on_win else "build.sh" build_script = os.path.join(self.path, build_script) @@ -2063,7 +2062,7 @@ def uses_vcs_in_build(self): def get_recipe_text( self, extract_pattern=None, force_top_level=False, apply_selectors=True - ): + ) -> str: meta_path = self.meta_path if meta_path: recipe_text = read_meta_file(meta_path) @@ -2148,7 +2147,7 @@ def extract_single_output_text( output = output_matches[output_index] if output_matches else "" except ValueError: if not self.path and self.meta.get("extra", {}).get("parent_recipe"): - utils.get_logger(__name__).warn( + log.warn( f"Didn't match any output in raw metadata. Target value was: {output_name}" ) output = "" @@ -2157,15 +2156,14 @@ def extract_single_output_text( return output @property - def numpy_xx(self): + def numpy_xx(self) -> bool: """This is legacy syntax that we need to support for a while. numpy x.x means "pin run as build" for numpy. It was special-cased to only numpy.""" text = self.extract_requirements_text() - uses_xx = bool(numpy_xx_re.search(text)) - return uses_xx + return bool(numpy_xx_re.search(text)) @property - def uses_numpy_pin_compatible_without_xx(self): + def uses_numpy_pin_compatible_without_xx(self) -> tuple[bool, bool]: text = self.extract_requirements_text() compatible_search = numpy_compatible_re.search(text) max_pin_search = None @@ -2199,11 +2197,11 @@ def uses_subpackage(self): return in_reqs or bool(subpackage_pin) @property - def uses_new_style_compiler_activation(self): + def uses_new_style_compiler_activation(self) -> bool: text = self.extract_requirements_text() return bool(re.search(r"\{\{\s*compiler\(.*\)\s*\}\}", text)) - def validate_features(self): + def validate_features(self) -> None: if any( "-" in feature for feature in ensure_list(self.get_value("build/features")) ): @@ -2223,28 +2221,24 @@ def copy(self): return new @property - def noarch(self): + def noarch(self) -> str | None: return self.get_value("build/noarch") @noarch.setter - def noarch(self, value): - build = self.meta.get("build", {}) - build["noarch"] = value - self.meta["build"] = build + def noarch(self, value: str | None) -> None: + self.meta.setdefault("build", {})["noarch"] = value if not self.noarch_python and not value: self.config.reset_platform() elif value: self.config.host_platform = "noarch" @property - def noarch_python(self): - return self.get_value("build/noarch_python") + def noarch_python(self) -> bool: + return bool(self.get_value("build/noarch_python")) @noarch_python.setter - def noarch_python(self, value): - build = self.meta.get("build", {}) - build["noarch_python"] = value - self.meta["build"] = build + def noarch_python(self, value: bool) -> None: + self.meta.setdefault("build", {})["noarch_python"] = value if not self.noarch and not value: self.config.reset_platform() elif value: @@ -2273,11 +2267,11 @@ def variant_in_source(self): return False @property - def pin_depends(self): + def pin_depends(self) -> str: return self.get_value("build/pin_depends", "").lower() @property - def source_provided(self): + def source_provided(self) -> bool: return not bool(self.meta.get("source")) or ( os.path.isdir(self.config.work_dir) and len(os.listdir(self.config.work_dir)) > 0 @@ -2690,11 +2684,11 @@ def get_rendered_output(self, name, permit_undefined_jinja=False, variant=None): return output @property - def force_ignore_keys(self): + def force_ignore_keys(self) -> list[str]: return ensure_list(self.get_value("build/force_ignore_keys")) @property - def force_use_keys(self): + def force_use_keys(self) -> list[str]: return ensure_list(self.get_value("build/force_use_keys")) def get_used_vars(self, force_top_level=False, force_global=False): @@ -2876,7 +2870,6 @@ def _get_used_vars_output_script(self): ) ) else: - log = utils.get_logger(__name__) log.warn( "Not detecting used variables in output script {}; conda-build only knows " "how to search .sh and .bat files right now.".format(script) @@ -2886,23 +2879,24 @@ def _get_used_vars_output_script(self): def get_variants_as_dict_of_lists(self): return variants.list_of_dicts_to_dict_of_lists(self.config.variants) - def clean(self): + def clean(self) -> None: """This ensures that clean is called with the correct build id""" self.config.clean() @property - def activate_build_script(self): - b = self.meta.get("build", {}) or {} - should_activate = b.get("activate_in_script") is not False - return bool(self.config.activate and should_activate) + def activate_build_script(self) -> bool: + return bool( + self.config.activate + and self.get_value("build/activate_in_script") is not False + ) @property - def build_is_host(self): + def build_is_host(self) -> bool: manual_overrides = ( - self.meta.get("build", {}).get("merge_build_host") is True + self.get_value("build/merge_build_host") is True or self.config.build_is_host ) - manually_disabled = self.meta.get("build", {}).get("merge_build_host") is False + manually_disabled = self.get_value("build/merge_build_host") is False return manual_overrides or ( self.config.subdirs_same and not manually_disabled From 340b3261a4bed545a54fb6dc448bdb6a245b20e6 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 1 Nov 2023 23:15:29 -0500 Subject: [PATCH 5/8] Add MetaData.version(fail_ok) --- conda_build/environ.py | 5 +++-- conda_build/metadata.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 5b02dbeaaa..a9fd84fa7d 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -546,11 +546,12 @@ def meta_vars(meta: MetaData, skip_build_id=False): ): d.update(get_hg_build_info(hg_dir)) + # fail_ok=True to allow failures during initial MetaData parsing d["PKG_NAME"] = meta.name(fail_ok=True) - d["PKG_VERSION"] = meta.version() + d["PKG_VERSION"] = meta.version(fail_ok=True) d["PKG_BUILDNUM"] = str(meta.build_number()) if meta.final and not skip_build_id: - d["PKG_BUILD_STRING"] = str(meta.build_id()) + d["PKG_BUILD_STRING"] = meta.build_id() d["PKG_HASH"] = meta.hash_dependencies() else: d["PKG_BUILD_STRING"] = "placeholder" diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 75ec38e061..b2e7dc2d3c 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -1439,9 +1439,9 @@ def name(self, fail_ok: bool = False) -> str: check_bad_chrs(name, "package/name") return name - def version(self) -> str: - version = self.get_value("package/version") - if not version: + def version(self, fail_ok: bool = False) -> str: + version = self.get_value("package/version", "") + if not version and not fail_ok: sys.exit("Error: package/version missing in: %r" % self.meta_path) version = str(version) check_bad_chrs(version, "package/version") From 20ba6e0f0f26360b139ff8191c1a76add6644606 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 1 Nov 2023 23:58:58 -0500 Subject: [PATCH 6/8] Make fail_ok conditional on final --- conda_build/environ.py | 14 +++++--------- conda_build/metadata.py | 10 +++++----- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index a9fd84fa7d..9e128ad511 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -389,7 +389,7 @@ def python_vars(metadata, prefix, escape_backslash): } build_or_host = "host" if metadata.is_cross else "build" deps = [str(ms.name) for ms in metadata.ms_depends(build_or_host)] - if "python" in deps or metadata.name(fail_ok=True) == "python": + if "python" in deps or metadata.name() == "python": python_bin = metadata.config.python_bin(prefix, metadata.config.host_subdir) if utils.on_win and escape_backslash: @@ -418,7 +418,7 @@ def perl_vars(metadata, prefix, escape_backslash): } build_or_host = "host" if metadata.is_cross else "build" deps = [str(ms.name) for ms in metadata.ms_depends(build_or_host)] - if "perl" in deps or metadata.name(fail_ok=True) == "perl": + if "perl" in deps or metadata.name() == "perl": perl_bin = metadata.config.perl_bin(prefix, metadata.config.host_subdir) if utils.on_win and escape_backslash: @@ -465,10 +465,7 @@ def r_vars(metadata, prefix, escape_backslash): build_or_host = "host" if metadata.is_cross else "build" deps = [str(ms.name) for ms in metadata.ms_depends(build_or_host)] - if ( - any(r_pkg in deps for r_pkg in R_PACKAGES) - or metadata.name(fail_ok=True) in R_PACKAGES - ): + if any(r_pkg in deps for r_pkg in R_PACKAGES) or metadata.name() in R_PACKAGES: r_bin = metadata.config.r_bin(prefix, metadata.config.host_subdir) # set R_USER explicitly to prevent crosstalk with existing R_LIBS_USER packages r_user = join(prefix, "Libs", "R") @@ -546,9 +543,8 @@ def meta_vars(meta: MetaData, skip_build_id=False): ): d.update(get_hg_build_info(hg_dir)) - # fail_ok=True to allow failures during initial MetaData parsing - d["PKG_NAME"] = meta.name(fail_ok=True) - d["PKG_VERSION"] = meta.version(fail_ok=True) + d["PKG_NAME"] = meta.name() + d["PKG_VERSION"] = meta.version() d["PKG_BUILDNUM"] = str(meta.build_number()) if meta.final and not skip_build_id: d["PKG_BUILD_STRING"] = meta.build_id() diff --git a/conda_build/metadata.py b/conda_build/metadata.py index b2e7dc2d3c..e58d9f3bf7 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -1174,7 +1174,7 @@ def append_metadata_sections( @property def is_output(self) -> str: parent_name = self.get_value("extra/parent_recipe", {}).get("name") - return parent_name and parent_name != self.name(fail_ok=True) + return parent_name and parent_name != self.name() def parse_again( self, @@ -1429,9 +1429,9 @@ def check_field(key, section): check_field(key_or_dict, section) return True - def name(self, fail_ok: bool = False) -> str: + def name(self) -> str: name = self.get_value("package/name", "") - if not name and not fail_ok: + if not name and self.final: sys.exit("Error: package/name missing in: %r" % self.meta_path) name = str(name) if name != name.lower(): @@ -1439,9 +1439,9 @@ def name(self, fail_ok: bool = False) -> str: check_bad_chrs(name, "package/name") return name - def version(self, fail_ok: bool = False) -> str: + def version(self) -> str: version = self.get_value("package/version", "") - if not version and not fail_ok: + if not version and self.final: sys.exit("Error: package/version missing in: %r" % self.meta_path) version = str(version) check_bad_chrs(version, "package/version") From ace93941a8b80e59d4b13f7e42cc4a1d75a7b02d Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 2 Nov 2023 00:10:10 -0500 Subject: [PATCH 7/8] Fixes --- conda_build/api.py | 4 +- conda_build/metadata.py | 101 ++++++++++++++++++++-------------------- conda_build/render.py | 11 ++--- 3 files changed, 55 insertions(+), 61 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index 10cef41e86..727240aece 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -77,8 +77,8 @@ def render( raise # remove outputs section from output objects for simplicity - if not om.path and om.get_section("outputs"): - om.parent_outputs = om.get_section("outputs") + if not om.path and (outputs := om.get_section("outputs")): + om.parent_outputs = outputs del om.meta["outputs"] output_metas[ diff --git a/conda_build/metadata.py b/conda_build/metadata.py index e58d9f3bf7..7ad51c7880 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -13,7 +13,7 @@ from collections import OrderedDict from functools import lru_cache from os.path import isfile, join -from typing import Any, Literal +from typing import Literal from bs4 import UnicodeDammit @@ -32,6 +32,7 @@ ) from .conda_interface import MatchSpec, envs_dirs, md5_file +from .deprecations import deprecated try: import yaml @@ -46,8 +47,6 @@ except AttributeError: Loader = yaml.Loader -log = utils.get_logger(__name__) - class StringifyNumbersLoader(Loader): @classmethod @@ -185,7 +184,7 @@ def get_selectors(config: Config) -> dict[str, bool]: if not np: np = defaults["numpy"] if config.verbose: - log.warn( + utils.get_logger(__name__).warn( "No numpy version specified in conda_build_config.yaml. " "Falling back to default numpy value of {}".format(defaults["numpy"]) ) @@ -259,6 +258,7 @@ def eval_selector(selector_string, namespace, variants_in_place): except NameError as e: missing_var = parseNameNotFound(e) if variants_in_place: + log = utils.get_logger(__name__) log.debug( "Treating unknown selector '" + missing_var + "' as if it was False." ) @@ -326,6 +326,7 @@ def ensure_valid_fields(meta): def _trim_None_strings(meta_dict): + log = utils.get_logger(__name__) for key, value in meta_dict.items(): if hasattr(value, "keys"): meta_dict[key] = _trim_None_strings(value) @@ -551,7 +552,7 @@ def parse(data, config, path=None): "provides_features": dict, "force_use_keys": list, "force_ignore_keys": list, - "merge_build_host": bool, + "merge_build_host": None, "pre-link": str, "post-link": str, "pre-unlink": str, @@ -911,6 +912,7 @@ def finalize_outputs_pass( if metadata.skip(): continue try: + log = utils.get_logger(__name__) # We should reparse the top-level recipe to get all of our dependencies fixed up. # we base things on base_metadata because it has the record of the full origin recipe if base_metadata.config.verbose: @@ -967,6 +969,7 @@ def finalize_outputs_pass( if not permit_unsatisfiable_variants: raise else: + log = utils.get_logger(__name__) log.warn( "Could not finalize metadata due to missing dependencies: " "{}".format(e.packages) @@ -1130,7 +1133,7 @@ def is_cross(self) -> bool: @property def final(self) -> bool: - return self.get_value("extra/final") + return bool(self.get_value("extra/final")) @final.setter def final(self, value: bool) -> None: @@ -1138,15 +1141,12 @@ def final(self, value: bool) -> None: @property def disable_pip(self) -> bool: - return self.config.disable_pip or ( - "build" in self.meta and "disable_pip" in self.meta["build"] - ) + return bool(self.config.disable_pip or self.get_value("build/disable_pip")) @disable_pip.setter def disable_pip(self, value: bool) -> None: - self.config.disable_pip = self.meta.setdefault("build", {})[ - "disable_pip" - ] = bool(value) + self.config.disable_pip = bool(value) + self.meta.setdefault("build", {})["disable_pip"] = bool(value) def append_metadata_sections( self, sections_file_or_dict, merge, raise_on_clobber=False @@ -1194,6 +1194,7 @@ def parse_again( """ assert not self.final, "modifying metadata after finalization" + log = utils.get_logger(__name__) if kw: log.warn( "using unsupported internal conda-build function `parse_again`. Please use " @@ -1241,11 +1242,9 @@ def parse_again( self.append_metadata_sections(dependencies, merge=True) if "error_overlinking" in self.get_section("build"): - self.config.error_overlinking = self.get_value("build/error_overlinking") + self.config.error_overlinking = self.meta["build"]["error_overlinking"] if "error_overdepending" in self.get_section("build"): - self.config.error_overdepending = self.get_value( - "build/error_overdepending" - ) + self.config.error_overdepending = self.meta["build"]["error_overdepending"] self.validate_features() self.ensure_no_pip_requirements() @@ -1344,10 +1343,10 @@ def fromdict(cls, metadata, config=None, variant=None): return m - def get_section(self, section: str): + def get_section(self, section): return self.meta.get(section, {}) - def get_value(self, name: str, default=None, autotype=True): + def get_value(self, name, default=None, autotype=True): """ Get a value from a meta.yaml. :param field: Field to return, e.g. 'package/name'. @@ -1381,6 +1380,7 @@ def get_value(self, name: str, default=None, autotype=True): # The 'source' section can be written a list, in which case the name # is passed in with an index, e.g. get_value('source/0/git_url') if index is None: + log = utils.get_logger(__name__) log.warn( f"No index specified in get_value('{name}'). Assuming index 0." ) @@ -1429,6 +1429,7 @@ def check_field(key, section): check_field(key_or_dict, section) return True + @deprecated.argument("3.28.0", "4.0.0", "fail_ok") def name(self) -> str: name = self.get_value("package/name", "") if not name and self.final: @@ -1451,7 +1452,7 @@ def version(self) -> str: ) return version - def build_number(self) -> int: + def build_number(self): number = self.get_value("build/number") # build number can come back as None if no setting (or jinja intermediate) @@ -1627,7 +1628,7 @@ def hash_dependencies(self): return _hash_dependencies(hashing_dependencies, self.config.hash_length) return hash_ - def build_id(self) -> str: + def build_id(self): manual_build_string = self.get_value("build/string") # we need the raw recipe for this metadata (possibly an output), so that we can say whether # PKG_HASH is used for anything. @@ -1663,16 +1664,16 @@ def build_id(self) -> str: out = re.sub("h[0-9a-f]{%s}" % self.config.hash_length, hash_, out) return out - def dist(self) -> str: + def dist(self): return f"{self.name()}-{self.version()}-{self.build_id()}" - def pkg_fn(self) -> str: + def pkg_fn(self): return "%s.tar.bz2" % self.dist() - def is_app(self) -> bool: + def is_app(self): return bool(self.get_value("app/entry")) - def app_meta(self) -> dict[str, Any]: + def app_meta(self): d = {"type": "app"} if self.get_value("app/icon"): d["icon"] = "%s.png" % md5_file(join(self.path, self.get_value("app/icon"))) @@ -1941,39 +1942,36 @@ def _get_contents( finally: if "CONDA_BUILD_STATE" in os.environ: del os.environ["CONDA_BUILD_STATE"] - - log.debug(f"{path=}") - log.debug(f"{filename=}") - log.debug(f"{rendered=}") - return rendered - def __unicode__(self) -> str: + def __unicode__(self): """ String representation of the MetaData. """ return str(self.__dict__) - def __str__(self) -> str: + def __str__(self): return self.__unicode__() - def __repr__(self) -> str: + def __repr__(self): """ String representation of the MetaData. """ return self.__str__() @property - def meta_path(self) -> str: - meta_path = self._meta_path or self.get_value("extra/parent_recipe", {}).get( - "path", "" + def meta_path(self): + meta_path = ( + self._meta_path + # get the parent recipe path if this is a subpackage + or self.get_value("extra/parent_recipe", {}).get("path", "") ) if meta_path and os.path.basename(meta_path) != self._meta_name: meta_path = os.path.join(meta_path, self._meta_name) return meta_path @property - def uses_setup_py_in_meta(self) -> bool: + def uses_setup_py_in_meta(self): meta_text = "" if self.meta_path: with open(self.meta_path, "rb") as f: @@ -1981,7 +1979,7 @@ def uses_setup_py_in_meta(self) -> bool: return "load_setup_py_data" in meta_text or "load_setuptools" in meta_text @property - def uses_regex_in_meta(self) -> bool: + def uses_regex_in_meta(self): meta_text = "" if self.meta_path: with open(self.meta_path, "rb") as f: @@ -1989,7 +1987,7 @@ def uses_regex_in_meta(self) -> bool: return "load_file_regex" in meta_text @property - def uses_load_file_data_in_meta(self) -> bool: + def uses_load_file_data_in_meta(self): meta_text = "" if self.meta_path: with open(self.meta_path, "rb") as f: @@ -1997,7 +1995,7 @@ def uses_load_file_data_in_meta(self) -> bool: return "load_file_data" in meta_text @property - def needs_source_for_render(self) -> bool: + def needs_source_for_render(self): return ( self.uses_vcs_in_meta or self.uses_setup_py_in_meta @@ -2006,7 +2004,7 @@ def needs_source_for_render(self) -> bool: ) @property - def uses_jinja(self) -> bool: + def uses_jinja(self): if not self.meta_path: return False with open(self.meta_path, "rb") as f: @@ -2054,7 +2052,7 @@ def uses_vcs_in_build(self) -> Literal["git" | "svn" | "mercurial"] | None: build_script, flags=re.IGNORECASE, ) - if len(matches) > 0 and vcs != self.meta["package"]["name"]: + if len(matches) > 0 and vcs != self.get_value("package/name"): if vcs == "hg": vcs = "mercurial" return vcs @@ -2062,7 +2060,7 @@ def uses_vcs_in_build(self) -> Literal["git" | "svn" | "mercurial"] | None: def get_recipe_text( self, extract_pattern=None, force_top_level=False, apply_selectors=True - ) -> str: + ): meta_path = self.meta_path if meta_path: recipe_text = read_meta_file(meta_path) @@ -2147,7 +2145,7 @@ def extract_single_output_text( output = output_matches[output_index] if output_matches else "" except ValueError: if not self.path and self.meta.get("extra", {}).get("parent_recipe"): - log.warn( + utils.get_logger(__name__).warn( f"Didn't match any output in raw metadata. Target value was: {output_name}" ) output = "" @@ -2197,11 +2195,11 @@ def uses_subpackage(self): return in_reqs or bool(subpackage_pin) @property - def uses_new_style_compiler_activation(self) -> bool: + def uses_new_style_compiler_activation(self): text = self.extract_requirements_text() return bool(re.search(r"\{\{\s*compiler\(.*\)\s*\}\}", text)) - def validate_features(self) -> None: + def validate_features(self): if any( "-" in feature for feature in ensure_list(self.get_value("build/features")) ): @@ -2221,7 +2219,7 @@ def copy(self): return new @property - def noarch(self) -> str | None: + def noarch(self): return self.get_value("build/noarch") @noarch.setter @@ -2267,11 +2265,11 @@ def variant_in_source(self): return False @property - def pin_depends(self) -> str: + def pin_depends(self): return self.get_value("build/pin_depends", "").lower() @property - def source_provided(self) -> bool: + def source_provided(self): return not bool(self.meta.get("source")) or ( os.path.isdir(self.config.work_dir) and len(os.listdir(self.config.work_dir)) > 0 @@ -2684,11 +2682,11 @@ def get_rendered_output(self, name, permit_undefined_jinja=False, variant=None): return output @property - def force_ignore_keys(self) -> list[str]: + def force_ignore_keys(self): return ensure_list(self.get_value("build/force_ignore_keys")) @property - def force_use_keys(self) -> list[str]: + def force_use_keys(self): return ensure_list(self.get_value("build/force_use_keys")) def get_used_vars(self, force_top_level=False, force_global=False): @@ -2870,6 +2868,7 @@ def _get_used_vars_output_script(self): ) ) else: + log = utils.get_logger(__name__) log.warn( "Not detecting used variables in output script {}; conda-build only knows " "how to search .sh and .bat files right now.".format(script) @@ -2879,7 +2878,7 @@ def _get_used_vars_output_script(self): def get_variants_as_dict_of_lists(self): return variants.list_of_dicts_to_dict_of_lists(self.config.variants) - def clean(self) -> None: + def clean(self): """This ensures that clean is called with the correct build id""" self.config.clean() diff --git a/conda_build/render.py b/conda_build/render.py index 3a70ef47ac..c0f1d8be73 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -406,7 +406,6 @@ def execute_download_actions(m, actions, env, package_subset=None, require_files def get_upstream_pins(m: MetaData, actions, env): """Download packages from specs, then inspect each downloaded package for additional downstream dependency specs. Return these additional specs.""" - env_specs = m.get_value(f"requirements/{env}", []) explicit_specs = [req.split(" ")[0] for req in env_specs] if env_specs else [] linked_packages = actions.get("LINK", []) @@ -644,13 +643,9 @@ def finalize_metadata( requirements = utils.expand_reqs(output.get("requirements", {})) m.meta["requirements"] = requirements - if m.get_section("requirements"): - utils.insert_variant_versions( - m.get_section("requirements"), m.config.variant, "build" - ) - utils.insert_variant_versions( - m.get_section("requirements"), m.config.variant, "host" - ) + if requirements := m.get_section("requirements"): + utils.insert_variant_versions(requirements, m.config.variant, "build") + utils.insert_variant_versions(requirements, m.config.variant, "host") m = parent_metadata.get_output_metadata(m.get_rendered_output(m.name())) build_unsat, host_unsat = add_upstream_pins( From 51d9159a471c3fb018f069e2adf1df2830457c12 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 2 Nov 2023 11:00:19 -0500 Subject: [PATCH 8/8] Add missing version --- tests/test-recipes/metadata/_empty_host_avoids_merge/meta.yaml | 1 + tests/test-recipes/metadata/_no_merge_build_host/meta.yaml | 1 + tests/test-recipes/split-packages/_order/meta.yaml | 2 ++ tests/test-recipes/variants/27_requirements_host/meta.yaml | 1 + 4 files changed, 5 insertions(+) diff --git a/tests/test-recipes/metadata/_empty_host_avoids_merge/meta.yaml b/tests/test-recipes/metadata/_empty_host_avoids_merge/meta.yaml index cbe6ac859b..4bc665ad7d 100644 --- a/tests/test-recipes/metadata/_empty_host_avoids_merge/meta.yaml +++ b/tests/test-recipes/metadata/_empty_host_avoids_merge/meta.yaml @@ -1,5 +1,6 @@ package: name: pkg + version: 0.0.1 # build: # merge_build_host: False diff --git a/tests/test-recipes/metadata/_no_merge_build_host/meta.yaml b/tests/test-recipes/metadata/_no_merge_build_host/meta.yaml index d4f463886f..8aae740991 100644 --- a/tests/test-recipes/metadata/_no_merge_build_host/meta.yaml +++ b/tests/test-recipes/metadata/_no_merge_build_host/meta.yaml @@ -1,5 +1,6 @@ package: name: pkg + version: 0.0.1 build: merge_build_host: False diff --git a/tests/test-recipes/split-packages/_order/meta.yaml b/tests/test-recipes/split-packages/_order/meta.yaml index df0c0db7b2..0db9f6bbce 100644 --- a/tests/test-recipes/split-packages/_order/meta.yaml +++ b/tests/test-recipes/split-packages/_order/meta.yaml @@ -1,5 +1,7 @@ package: name: toplevel-ab + version: 0.0.1 + outputs: - name: a version: 1 diff --git a/tests/test-recipes/variants/27_requirements_host/meta.yaml b/tests/test-recipes/variants/27_requirements_host/meta.yaml index 0c4a833fa8..0ab071e56b 100644 --- a/tests/test-recipes/variants/27_requirements_host/meta.yaml +++ b/tests/test-recipes/variants/27_requirements_host/meta.yaml @@ -1,5 +1,6 @@ package: name: cfastpm + version: 0.0.1 requirements: host: