diff --git a/conda_build/build.py b/conda_build/build.py index 47600ffff4..c007286474 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -1564,15 +1564,11 @@ def create_info_files(m, replacements, files, prefix): write_no_link(m, files) - sources = m.get_section("source") - if hasattr(sources, "keys"): - sources = [sources] - with open(join(m.config.info_dir, "git"), "w", encoding="utf-8") as fo: - for src in sources: - if src.get("git_url"): + for source_dict in m.get_section("source"): + if source_dict.get("git_url"): source.git_info( - os.path.join(m.config.work_dir, src.get("folder", "")), + os.path.join(m.config.work_dir, source_dict.get("folder", "")), m.config.build_prefix, git=None, verbose=m.config.verbose, diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 9539dc5d14..f25e57f280 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -13,7 +13,7 @@ from collections import OrderedDict from functools import lru_cache from os.path import isfile, join -from typing import Literal +from typing import Literal, overload from bs4 import UnicodeDammit @@ -622,6 +622,7 @@ def parse(data, config, path=None): "prelink_message": None, "readme": None, }, + "extra": {}, } # Fields that may either be a dictionary or a list of dictionaries. @@ -1316,9 +1317,11 @@ def parse_until_resolved( @classmethod def fromstring(cls, metadata, config=None, variant=None): m = super().__new__(cls) - if not config: - config = Config() - m.meta = parse(metadata, config=config, path="", variant=variant) + m.path = "" + m._meta_path = "" + m.requirements_path = "" + config = config or Config(variant=variant) + m.meta = parse(metadata, config=config, path="") m.config = config m.parse_again(permit_undefined_jinja=True) return m @@ -1333,18 +1336,45 @@ def fromdict(cls, metadata, config=None, variant=None): m._meta_path = "" m.requirements_path = "" m.meta = sanitize(metadata) - - if not config: - config = Config(variant=variant) - - m.config = config + m.config = config or Config(variant=variant) m.undefined_jinja_vars = [] m.final = False - return m - def get_section(self, section): - return self.meta.get(section, {}) + @overload + def get_section(self, section: Literal["source", "outputs"]) -> list[dict]: + ... + + @overload + def get_section( + self, + section: Literal[ + "package", + "build", + "requirements", + "app", + "test", + "about", + "extra", + ], + ) -> dict: + ... + + def get_section(self, name): + section = self.meta.get(name) + if name in OPTIONALLY_ITERABLE_FIELDS: + if not section: + return [] + elif isinstance(section, dict): + return [section] + elif not isinstance(section, list): + raise ValueError(f"Expected {name} to be a list") + else: + if not section: + return {} + elif not isinstance(section, dict): + raise ValueError(f"Expected {name} to be a dict") + return section def get_value(self, name, default=None, autotype=True): """ @@ -1364,7 +1394,9 @@ def get_value(self, name, default=None, autotype=True): index = None elif len(names) == 3: section, index, key = names - assert section == "source", "Section is not a list: " + section + assert section in OPTIONALLY_ITERABLE_FIELDS, ( + "Section is not a list: " + section + ) index = int(index) # get correct default @@ -1386,7 +1418,7 @@ def get_value(self, name, default=None, autotype=True): ) index = 0 - if len(section_data) == 0: + if not section_data: section_data = {} else: section_data = section_data[index] @@ -1475,7 +1507,7 @@ def get_depends_top_and_out(self, typ): if not self.is_output: matching_output = [ out - for out in self.meta.get("outputs", []) + for out in self.get_section("outputs") if out.get("name") == self.name() ] if matching_output: @@ -2014,7 +2046,7 @@ def uses_jinja(self): return len(matches) > 0 @property - def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None: + def uses_vcs_in_meta(self) -> Literal["git", "svn", "mercurial"] | None: """returns name of vcs used if recipe contains metadata associated with version control systems. If this metadata is present, a download/copy will be forced in parse_or_try_download. """ @@ -2034,7 +2066,7 @@ def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None: return vcs @property - def uses_vcs_in_build(self) -> Literal["git" | "svn" | "mercurial"] | None: + def uses_vcs_in_build(self) -> Literal["git", "svn", "mercurial"] | None: # TODO :: Re-work this. Is it even useful? We can declare any vcs in our build deps. build_script = "bld.bat" if on_win else "build.sh" build_script = os.path.join(self.path, build_script) @@ -2271,9 +2303,8 @@ def pin_depends(self): @property def source_provided(self): - return not bool(self.meta.get("source")) or ( - os.path.isdir(self.config.work_dir) - and len(os.listdir(self.config.work_dir)) > 0 + return not self.get_section("source") or ( + os.path.isdir(self.config.work_dir) and os.listdir(self.config.work_dir) ) def reconcile_metadata_with_output_dict(self, output_metadata, output_dict): diff --git a/conda_build/render.py b/conda_build/render.py index c0f1d8be73..1e8ddae08a 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -721,17 +721,17 @@ def finalize_metadata( # if source/path is relative, then the output package makes no sense at all. The next # best thing is to hard-code the absolute path. This probably won't exist on any # system other than the original build machine, but at least it will work there. - if source_path := m.get_value("source/path"): - if not isabs(source_path): - m.meta["source"]["path"] = normpath(join(m.path, source_path)) + for source_dict in m.get_section("source"): + if (source_path := source_dict.get("path")) and not isabs(source_path): + source_dict["path"] = normpath(join(m.path, source_path)) elif ( - (git_url := m.get_value("source/git_url")) + (git_url := source_dict.get("git_url")) # absolute paths are not relative paths and not isabs(git_url) # real urls are not relative paths and ":" not in git_url ): - m.meta["source"]["git_url"] = normpath(join(m.path, git_url)) + source_dict["git_url"] = normpath(join(m.path, git_url)) m.meta.setdefault("build", {}) diff --git a/conda_build/source.py b/conda_build/source.py index c8d21a4c2e..85e64c8292 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -1027,18 +1027,11 @@ def provide(metadata): - unpack - apply patches (if any) """ - meta = metadata.get_section("source") - if not os.path.isdir(metadata.config.build_folder): - os.makedirs(metadata.config.build_folder) + os.makedirs(metadata.config.build_folder, exist_ok=True) git = None - if hasattr(meta, "keys"): - dicts = [meta] - else: - dicts = meta - try: - for source_dict in dicts: + for source_dict in metadata.get_section("source"): folder = source_dict.get("folder") src_dir = os.path.join(metadata.config.work_dir, folder if folder else "") if any(k in source_dict for k in ("fn", "url")): diff --git a/news/5112-fix-multiple-sources b/news/5112-fix-multiple-sources new file mode 100644 index 0000000000..f988080a8b --- /dev/null +++ b/news/5112-fix-multiple-sources @@ -0,0 +1,19 @@ +### Enhancements + +* Update `conda_build.metadata.MetaData.get_section` to always return lists for "source" and "outputs". (#5112) + +### Bug fixes + +* Fix finalizing recipes with multiple sources. (#5111 via #5112) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 0fd89a22c3..7ac5bbdf01 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -15,6 +15,8 @@ from conda_build import api from conda_build.config import Config from conda_build.metadata import ( + FIELDS, + OPTIONALLY_ITERABLE_FIELDS, MetaData, _hash_dependencies, get_selectors, @@ -23,7 +25,7 @@ ) from conda_build.utils import DEFAULT_SUBDIRS -from .utils import metadata_dir, thisdir +from .utils import metadata_dir, metadata_path, thisdir def test_uses_vcs_in_metadata(testing_workdir, testing_metadata): @@ -459,3 +461,22 @@ def test_get_selectors( # override with True values **{key: True for key in expected}, } + + +def test_fromstring(): + MetaData.fromstring((metadata_path / "multiple_sources" / "meta.yaml").read_text()) + + +def test_fromdict(): + MetaData.fromdict( + yamlize((metadata_path / "multiple_sources" / "meta.yaml").read_text()) + ) + + +def test_get_section(testing_metadata: MetaData): + for name in FIELDS: + section = testing_metadata.get_section(name) + if name in OPTIONALLY_ITERABLE_FIELDS: + assert isinstance(section, list) + else: + assert isinstance(section, dict)