Fix MetaData.get_section handling for "source" & "outputs"

conda_build/build.py

@@ -1564,15 +1564,11 @@ def create_info_files(m, replacements, files, prefix):
 
     write_no_link(m, files)
 
-    sources = m.get_section("source")
-    if hasattr(sources, "keys"):
-        sources = [sources]
-
     with open(join(m.config.info_dir, "git"), "w", encoding="utf-8") as fo:
-        for src in sources:
-            if src.get("git_url"):
+        for source_dict in m.get_section("source"):
+            if source_dict.get("git_url"):
                 source.git_info(
-                    os.path.join(m.config.work_dir, src.get("folder", "")),
+                    os.path.join(m.config.work_dir, source_dict.get("folder", "")),
                     m.config.build_prefix,
                     git=None,
                     verbose=m.config.verbose,

conda_build/metadata.py

@@ -13,7 +13,7 @@
 from collections import OrderedDict
 from functools import lru_cache
 from os.path import isfile, join
-from typing import Literal
+from typing import Literal, overload
 
 from bs4 import UnicodeDammit
 
@@ -622,6 +622,7 @@ def parse(data, config, path=None):
         "prelink_message": None,
         "readme": None,
     },
+    "extra": {},
 }
 
 # Fields that may either be a dictionary or a list of dictionaries.
@@ -1316,9 +1317,11 @@ def parse_until_resolved(
     @classmethod
     def fromstring(cls, metadata, config=None, variant=None):
         m = super().__new__(cls)
-        if not config:
-            config = Config()
-        m.meta = parse(metadata, config=config, path="", variant=variant)
+        m.path = ""
+        m._meta_path = ""
+        m.requirements_path = ""
+        config = config or Config(variant=variant)
+        m.meta = parse(metadata, config=config, path="")
         m.config = config
         m.parse_again(permit_undefined_jinja=True)
         return m
@@ -1333,18 +1336,45 @@ def fromdict(cls, metadata, config=None, variant=None):
         m._meta_path = ""
         m.requirements_path = ""
         m.meta = sanitize(metadata)
-
-        if not config:
-            config = Config(variant=variant)
-
-        m.config = config
+        m.config = config or Config(variant=variant)
         m.undefined_jinja_vars = []
         m.final = False
-
         return m
 
-    def get_section(self, section):
-        return self.meta.get(section, {})
+    @overload
+    def get_section(self, section: Literal["source", "outputs"]) -> list[dict]:
+        ...
+
+    @overload
+    def get_section(
+        self,
+        section: Literal[
+            "package",
+            "build",
+            "requirements",
+            "app",
+            "test",
+            "about",
+            "extra",
+        ],
+    ) -> dict:
+        ...
+
+    def get_section(self, name):
+        section = self.meta.get(name)
+        if name in OPTIONALLY_ITERABLE_FIELDS:
+            if not section:
+                return []
+            elif isinstance(section, dict):
+                return [section]
+            elif not isinstance(section, list):
+                raise ValueError(f"Expected {name} to be a list")
+        else:
+            if not section:
+                return {}
+            elif not isinstance(section, dict):
+                raise ValueError(f"Expected {name} to be a dict")
+        return section
 
     def get_value(self, name, default=None, autotype=True):
         """
@@ -1364,7 +1394,9 @@ def get_value(self, name, default=None, autotype=True):
             index = None
         elif len(names) == 3:
             section, index, key = names
-            assert section == "source", "Section is not a list: " + section
+            assert section in OPTIONALLY_ITERABLE_FIELDS, (
+                "Section is not a list: " + section
+            )
             index = int(index)
 
         # get correct default
@@ -1386,7 +1418,7 @@ def get_value(self, name, default=None, autotype=True):
                 )
                 index = 0
 
-            if len(section_data) == 0:
+            if not section_data:
                 section_data = {}
             else:
                 section_data = section_data[index]
@@ -1475,7 +1507,7 @@ def get_depends_top_and_out(self, typ):
         if not self.is_output:
             matching_output = [
                 out
-                for out in self.meta.get("outputs", [])
+                for out in self.get_section("outputs")
                 if out.get("name") == self.name()
             ]
             if matching_output:
@@ -2014,7 +2046,7 @@ def uses_jinja(self):
         return len(matches) > 0
 
     @property
-    def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None:
+    def uses_vcs_in_meta(self) -> Literal["git", "svn", "mercurial"] | None:
         """returns name of vcs used if recipe contains metadata associated with version control systems.
         If this metadata is present, a download/copy will be forced in parse_or_try_download.
         """
@@ -2034,7 +2066,7 @@ def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None:
         return vcs
 
     @property
-    def uses_vcs_in_build(self) -> Literal["git" | "svn" | "mercurial"] | None:
+    def uses_vcs_in_build(self) -> Literal["git", "svn", "mercurial"] | None:
         # TODO :: Re-work this. Is it even useful? We can declare any vcs in our build deps.
         build_script = "bld.bat" if on_win else "build.sh"
         build_script = os.path.join(self.path, build_script)
@@ -2271,9 +2303,8 @@ def pin_depends(self):
 
     @property
     def source_provided(self):
-        return not bool(self.meta.get("source")) or (
-            os.path.isdir(self.config.work_dir)
-            and len(os.listdir(self.config.work_dir)) > 0
+        return not self.get_section("source") or (
+            os.path.isdir(self.config.work_dir) and os.listdir(self.config.work_dir)
         )
 
     def reconcile_metadata_with_output_dict(self, output_metadata, output_dict):

conda_build/render.py

@@ -721,17 +721,17 @@ def finalize_metadata(
         # if source/path is relative, then the output package makes no sense at all.  The next
         #   best thing is to hard-code the absolute path.  This probably won't exist on any
         #   system other than the original build machine, but at least it will work there.
-        if source_path := m.get_value("source/path"):
-            if not isabs(source_path):
-                m.meta["source"]["path"] = normpath(join(m.path, source_path))
+        for source_dict in m.get_section("source"):
+            if (source_path := source_dict.get("path")) and not isabs(source_path):
+                source_dict["path"] = normpath(join(m.path, source_path))
             elif (
-                (git_url := m.get_value("source/git_url"))
+                (git_url := source_dict.get("git_url"))
                 # absolute paths are not relative paths
                 and not isabs(git_url)
                 # real urls are not relative paths
                 and ":" not in git_url
             ):
-                m.meta["source"]["git_url"] = normpath(join(m.path, git_url))
+                source_dict["git_url"] = normpath(join(m.path, git_url))
 
         m.meta.setdefault("build", {})
 

conda_build/source.py

@@ -1027,18 +1027,11 @@ def provide(metadata):
       - unpack
       - apply patches (if any)
     """
-    meta = metadata.get_section("source")
-    if not os.path.isdir(metadata.config.build_folder):
-        os.makedirs(metadata.config.build_folder)
+    os.makedirs(metadata.config.build_folder, exist_ok=True)
     git = None
 
-    if hasattr(meta, "keys"):
-        dicts = [meta]
-    else:
-        dicts = meta
-
     try:
-        for source_dict in dicts:
+        for source_dict in metadata.get_section("source"):
             folder = source_dict.get("folder")
             src_dir = os.path.join(metadata.config.work_dir, folder if folder else "")
             if any(k in source_dict for k in ("fn", "url")):

news/5112-fix-multiple-sources

@@ -0,0 +1,19 @@
+### Enhancements
+
+* Update `conda_build.metadata.MetaData.get_section` to always return lists for "source" and "outputs". (#5112)
+
+### Bug fixes
+
+* Fix finalizing recipes with multiple sources. (#5111 via #5112)
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>

tests/test_metadata.py

@@ -15,6 +15,8 @@
 from conda_build import api
 from conda_build.config import Config
 from conda_build.metadata import (
+    FIELDS,
+    OPTIONALLY_ITERABLE_FIELDS,
     MetaData,
     _hash_dependencies,
     get_selectors,
@@ -23,7 +25,7 @@
 )
 from conda_build.utils import DEFAULT_SUBDIRS
 
-from .utils import metadata_dir, thisdir
+from .utils import metadata_dir, metadata_path, thisdir
 
 
 def test_uses_vcs_in_metadata(testing_workdir, testing_metadata):
@@ -459,3 +461,22 @@ def test_get_selectors(
         # override with True values
         **{key: True for key in expected},
     }
+
+
+def test_fromstring():
+    MetaData.fromstring((metadata_path / "multiple_sources" / "meta.yaml").read_text())
+
+
+def test_fromdict():
+    MetaData.fromdict(
+        yamlize((metadata_path / "multiple_sources" / "meta.yaml").read_text())
+    )
+
+
+def test_get_section(testing_metadata: MetaData):
+    for name in FIELDS:
+        section = testing_metadata.get_section(name)
+        if name in OPTIONALLY_ITERABLE_FIELDS:
+            assert isinstance(section, list)
+        else:
+            assert isinstance(section, dict)