From 39ee6d89aa4c3ff77e496ea04011089a43861b1b Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 12 Dec 2023 11:10:37 -0600 Subject: [PATCH 1/9] Correct variant handling in MetaData.fromstring --- conda_build/metadata.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 9539dc5d14..452f95c434 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -1316,9 +1316,8 @@ def parse_until_resolved( @classmethod def fromstring(cls, metadata, config=None, variant=None): m = super().__new__(cls) - if not config: - config = Config() - m.meta = parse(metadata, config=config, path="", variant=variant) + config = config or Config(variant=variant) + m.meta = parse(metadata, config=config, path="") m.config = config m.parse_again(permit_undefined_jinja=True) return m @@ -1333,14 +1332,10 @@ def fromdict(cls, metadata, config=None, variant=None): m._meta_path = "" m.requirements_path = "" m.meta = sanitize(metadata) - - if not config: - config = Config(variant=variant) - + config = config or Config(variant=variant) m.config = config m.undefined_jinja_vars = [] m.final = False - return m def get_section(self, section): From ef0cc058d19b94a3cde0765e4fb7b0e76270bf64 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 12 Dec 2023 13:57:59 -0600 Subject: [PATCH 2/9] Correct typing --- conda_build/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 452f95c434..184987e4ad 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -2009,7 +2009,7 @@ def uses_jinja(self): return len(matches) > 0 @property - def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None: + def uses_vcs_in_meta(self) -> Literal["git", "svn", "mercurial"] | None: """returns name of vcs used if recipe contains metadata associated with version control systems. If this metadata is present, a download/copy will be forced in parse_or_try_download. """ @@ -2029,7 +2029,7 @@ def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None: return vcs @property - def uses_vcs_in_build(self) -> Literal["git" | "svn" | "mercurial"] | None: + def uses_vcs_in_build(self) -> Literal["git", "svn", "mercurial"] | None: # TODO :: Re-work this. Is it even useful? We can declare any vcs in our build deps. build_script = "bld.bat" if on_win else "build.sh" build_script = os.path.join(self.path, build_script) From 93e98dd793e9b32c33ee453802da36d09d64ee88 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 12 Dec 2023 15:09:42 -0600 Subject: [PATCH 3/9] Ensure get_section returns list for source and outputs --- conda_build/build.py | 2 -- conda_build/metadata.py | 69 +++++++++++++++++++++++++++++++---------- conda_build/render.py | 10 +++--- conda_build/source.py | 7 +---- 4 files changed, 58 insertions(+), 30 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 47600ffff4..cfcd5f1d76 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -1565,8 +1565,6 @@ def create_info_files(m, replacements, files, prefix): write_no_link(m, files) sources = m.get_section("source") - if hasattr(sources, "keys"): - sources = [sources] with open(join(m.config.info_dir, "git"), "w", encoding="utf-8") as fo: for src in sources: diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 184987e4ad..262b689f8d 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -13,7 +13,6 @@ from collections import OrderedDict from functools import lru_cache from os.path import isfile, join -from typing import Literal from bs4 import UnicodeDammit @@ -47,6 +46,9 @@ except AttributeError: Loader = yaml.Loader +if TYPE_CHECKING := False: + from typing import Literal, overload + class StringifyNumbersLoader(Loader): @classmethod @@ -622,6 +624,7 @@ def parse(data, config, path=None): "prelink_message": None, "readme": None, }, + "extra": {}, } # Fields that may either be a dictionary or a list of dictionaries. @@ -1338,8 +1341,40 @@ def fromdict(cls, metadata, config=None, variant=None): m.final = False return m - def get_section(self, section): - return self.meta.get(section, {}) + @overload + def get_section(self, section: Literal["source", "outputs"]) -> list[dict]: + ... + + @overload + def get_section( + self, + section: Literal[ + "package", + "build", + "requirements", + "app", + "test", + "about", + "extra", + ], + ) -> dict: + ... + + def get_section(self, name): + section = self.meta.get(name) + if name in OPTIONALLY_ITERABLE_FIELDS: + if not section: + return [] + elif isinstance(section, dict): + return [section] + elif not isinstance(section, list): + raise ValueError(f"Expected {name} to be a list") + else: + if not section: + return {} + elif not isinstance(section, dict): + raise ValueError(f"Expected {name} to be a dict") + return section def get_value(self, name, default=None, autotype=True): """ @@ -1353,24 +1388,26 @@ def get_value(self, name, default=None, autotype=True): :return: The named value from meta.yaml """ names = name.split("/") - assert len(names) in (2, 3), "Bad field name: " + name if len(names) == 2: section, key = names index = None elif len(names) == 3: section, index, key = names - assert section == "source", "Section is not a list: " + section + if section not in OPTIONALLY_ITERABLE_FIELDS: + raise ValueError(f"Section is not indexable: {section}") index = int(index) + else: + raise ValueError(f"Bad field name: {name}") # get correct default if autotype and default is None and FIELDS.get(section, {}).get(key): default = FIELDS[section][key]() section_data = self.get_section(section) - if isinstance(section_data, dict): - assert ( - not index - ), f"Got non-zero index ({index}), but section {section} is not a list." + if isinstance(section_data, dict) and not index: + raise ValueError( + f"Got non-zero index ({index}), but section {section} is not a list." + ) elif isinstance(section_data, list): # The 'source' section can be written a list, in which case the name # is passed in with an index, e.g. get_value('source/0/git_url') @@ -1381,13 +1418,12 @@ def get_value(self, name, default=None, autotype=True): ) index = 0 - if len(section_data) == 0: + if not section_data: section_data = {} else: section_data = section_data[index] - assert isinstance( - section_data, dict - ), f"Expected {section}/{index} to be a dict" + if not isinstance(section_data, dict): + raise ValueError(f"Expected {name} to be a dict") value = section_data.get(key, default) @@ -1470,7 +1506,7 @@ def get_depends_top_and_out(self, typ): if not self.is_output: matching_output = [ out - for out in self.meta.get("outputs", []) + for out in self.get_section("outputs") if out.get("name") == self.name() ] if matching_output: @@ -2266,9 +2302,8 @@ def pin_depends(self): @property def source_provided(self): - return not bool(self.meta.get("source")) or ( - os.path.isdir(self.config.work_dir) - and len(os.listdir(self.config.work_dir)) > 0 + return not self.get_section("source") or ( + os.path.isdir(self.config.work_dir) and os.listdir(self.config.work_dir) ) def reconcile_metadata_with_output_dict(self, output_metadata, output_dict): diff --git a/conda_build/render.py b/conda_build/render.py index c0f1d8be73..1e8ddae08a 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -721,17 +721,17 @@ def finalize_metadata( # if source/path is relative, then the output package makes no sense at all. The next # best thing is to hard-code the absolute path. This probably won't exist on any # system other than the original build machine, but at least it will work there. - if source_path := m.get_value("source/path"): - if not isabs(source_path): - m.meta["source"]["path"] = normpath(join(m.path, source_path)) + for source_dict in m.get_section("source"): + if (source_path := source_dict.get("path")) and not isabs(source_path): + source_dict["path"] = normpath(join(m.path, source_path)) elif ( - (git_url := m.get_value("source/git_url")) + (git_url := source_dict.get("git_url")) # absolute paths are not relative paths and not isabs(git_url) # real urls are not relative paths and ":" not in git_url ): - m.meta["source"]["git_url"] = normpath(join(m.path, git_url)) + source_dict["git_url"] = normpath(join(m.path, git_url)) m.meta.setdefault("build", {}) diff --git a/conda_build/source.py b/conda_build/source.py index c8d21a4c2e..5b9c9b889e 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -1027,16 +1027,11 @@ def provide(metadata): - unpack - apply patches (if any) """ - meta = metadata.get_section("source") + dicts = metadata.get_section("source") if not os.path.isdir(metadata.config.build_folder): os.makedirs(metadata.config.build_folder) git = None - if hasattr(meta, "keys"): - dicts = [meta] - else: - dicts = meta - try: for source_dict in dicts: folder = source_dict.get("folder") From 409f7e3d5de3147f0926f45a321998d7f9b26ecb Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 12 Dec 2023 15:26:19 -0600 Subject: [PATCH 4/9] Add MetaData tests --- conda_build/metadata.py | 12 +++++----- .../metadata/source_multiple/meta.yaml | 2 +- tests/test_metadata.py | 23 ++++++++++++++++++- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 262b689f8d..0f1bca520a 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -13,6 +13,7 @@ from collections import OrderedDict from functools import lru_cache from os.path import isfile, join +from typing import Literal, overload from bs4 import UnicodeDammit @@ -46,9 +47,6 @@ except AttributeError: Loader = yaml.Loader -if TYPE_CHECKING := False: - from typing import Literal, overload - class StringifyNumbersLoader(Loader): @classmethod @@ -1319,6 +1317,9 @@ def parse_until_resolved( @classmethod def fromstring(cls, metadata, config=None, variant=None): m = super().__new__(cls) + m.path = "" + m._meta_path = "" + m.requirements_path = "" config = config or Config(variant=variant) m.meta = parse(metadata, config=config, path="") m.config = config @@ -1335,8 +1336,7 @@ def fromdict(cls, metadata, config=None, variant=None): m._meta_path = "" m.requirements_path = "" m.meta = sanitize(metadata) - config = config or Config(variant=variant) - m.config = config + m.config = config or Config(variant=variant) m.undefined_jinja_vars = [] m.final = False return m @@ -1404,7 +1404,7 @@ def get_value(self, name, default=None, autotype=True): default = FIELDS[section][key]() section_data = self.get_section(section) - if isinstance(section_data, dict) and not index: + if isinstance(section_data, dict) and index: raise ValueError( f"Got non-zero index ({index}), but section {section} is not a list." ) diff --git a/tests/test-recipes/metadata/source_multiple/meta.yaml b/tests/test-recipes/metadata/source_multiple/meta.yaml index bbd2cb4f03..82c3a7a34d 100644 --- a/tests/test-recipes/metadata/source_multiple/meta.yaml +++ b/tests/test-recipes/metadata/source_multiple/meta.yaml @@ -3,7 +3,7 @@ package: version: 1.0 source: - - path: {{ environ.get('CONDA_BUILD_TEST_RECIPE_PATH') }} + - path: "{{ environ.get('CONDA_BUILD_TEST_RECIPE_PATH') }}" - git_url: https://github.com/conda/conda_build_test_recipe git_tag: 1.20.2 diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 0fd89a22c3..71e2d680c9 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -15,6 +15,8 @@ from conda_build import api from conda_build.config import Config from conda_build.metadata import ( + FIELDS, + OPTIONALLY_ITERABLE_FIELDS, MetaData, _hash_dependencies, get_selectors, @@ -23,7 +25,7 @@ ) from conda_build.utils import DEFAULT_SUBDIRS -from .utils import metadata_dir, thisdir +from .utils import metadata_dir, metadata_path, thisdir def test_uses_vcs_in_metadata(testing_workdir, testing_metadata): @@ -459,3 +461,22 @@ def test_get_selectors( # override with True values **{key: True for key in expected}, } + + +def test_fromstring(): + MetaData.fromstring((metadata_path / "source_multiple" / "meta.yaml").read_text()) + + +def test_fromdict(): + MetaData.fromdict( + yamlize((metadata_path / "source_multiple" / "meta.yaml").read_text()) + ) + + +def test_get_section(testing_metadata: MetaData): + for name in FIELDS: + section = testing_metadata.get_section(name) + if name in OPTIONALLY_ITERABLE_FIELDS: + assert isinstance(section, list) + else: + assert isinstance(section, dict) From 725051c675e717f1faa8c96af8454b1c30a4de29 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 12 Dec 2023 15:50:25 -0600 Subject: [PATCH 5/9] Touchup --- conda_build/build.py | 8 +++----- conda_build/source.py | 6 ++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index cfcd5f1d76..c007286474 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -1564,13 +1564,11 @@ def create_info_files(m, replacements, files, prefix): write_no_link(m, files) - sources = m.get_section("source") - with open(join(m.config.info_dir, "git"), "w", encoding="utf-8") as fo: - for src in sources: - if src.get("git_url"): + for source_dict in m.get_section("source"): + if source_dict.get("git_url"): source.git_info( - os.path.join(m.config.work_dir, src.get("folder", "")), + os.path.join(m.config.work_dir, source_dict.get("folder", "")), m.config.build_prefix, git=None, verbose=m.config.verbose, diff --git a/conda_build/source.py b/conda_build/source.py index 5b9c9b889e..85e64c8292 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -1027,13 +1027,11 @@ def provide(metadata): - unpack - apply patches (if any) """ - dicts = metadata.get_section("source") - if not os.path.isdir(metadata.config.build_folder): - os.makedirs(metadata.config.build_folder) + os.makedirs(metadata.config.build_folder, exist_ok=True) git = None try: - for source_dict in dicts: + for source_dict in metadata.get_section("source"): folder = source_dict.get("folder") src_dir = os.path.join(metadata.config.work_dir, folder if folder else "") if any(k in source_dict for k in ("fn", "url")): From 0e75945e033ec6998ef3b0180434999801ac4572 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 12 Dec 2023 23:02:10 -0600 Subject: [PATCH 6/9] Revert recipe change --- tests/test-recipes/metadata/source_multiple/meta.yaml | 2 +- tests/test_metadata.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-recipes/metadata/source_multiple/meta.yaml b/tests/test-recipes/metadata/source_multiple/meta.yaml index 82c3a7a34d..bbd2cb4f03 100644 --- a/tests/test-recipes/metadata/source_multiple/meta.yaml +++ b/tests/test-recipes/metadata/source_multiple/meta.yaml @@ -3,7 +3,7 @@ package: version: 1.0 source: - - path: "{{ environ.get('CONDA_BUILD_TEST_RECIPE_PATH') }}" + - path: {{ environ.get('CONDA_BUILD_TEST_RECIPE_PATH') }} - git_url: https://github.com/conda/conda_build_test_recipe git_tag: 1.20.2 diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 71e2d680c9..7ac5bbdf01 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -464,12 +464,12 @@ def test_get_selectors( def test_fromstring(): - MetaData.fromstring((metadata_path / "source_multiple" / "meta.yaml").read_text()) + MetaData.fromstring((metadata_path / "multiple_sources" / "meta.yaml").read_text()) def test_fromdict(): MetaData.fromdict( - yamlize((metadata_path / "source_multiple" / "meta.yaml").read_text()) + yamlize((metadata_path / "multiple_sources" / "meta.yaml").read_text()) ) From 2d640b74c9ea383d6ce98654db7e08d95c3aea5c Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 12 Dec 2023 23:04:54 -0600 Subject: [PATCH 7/9] Add news --- news/5112-fix-multiple-sources | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 news/5112-fix-multiple-sources diff --git a/news/5112-fix-multiple-sources b/news/5112-fix-multiple-sources new file mode 100644 index 0000000000..17c0f77398 --- /dev/null +++ b/news/5112-fix-multiple-sources @@ -0,0 +1,19 @@ +### Enhancements + +* Update `conda_build.metadata.MetaData.get_section` to always return lists for "source" and "outputs". (#5112) + +### Bug fixes + +* Fix finalizing recipes with multiple sources. (#5112) + +### Deprecations + +* + +### Docs + +* + +### Other + +* From 80b5ade810383f4a0339e1913d634ab233b73840 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 14 Dec 2023 12:03:23 -0600 Subject: [PATCH 8/9] Revert removing assertions --- conda_build/metadata.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 0f1bca520a..f25e57f280 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -1388,26 +1388,26 @@ def get_value(self, name, default=None, autotype=True): :return: The named value from meta.yaml """ names = name.split("/") + assert len(names) in (2, 3), "Bad field name: " + name if len(names) == 2: section, key = names index = None elif len(names) == 3: section, index, key = names - if section not in OPTIONALLY_ITERABLE_FIELDS: - raise ValueError(f"Section is not indexable: {section}") + assert section in OPTIONALLY_ITERABLE_FIELDS, ( + "Section is not a list: " + section + ) index = int(index) - else: - raise ValueError(f"Bad field name: {name}") # get correct default if autotype and default is None and FIELDS.get(section, {}).get(key): default = FIELDS[section][key]() section_data = self.get_section(section) - if isinstance(section_data, dict) and index: - raise ValueError( - f"Got non-zero index ({index}), but section {section} is not a list." - ) + if isinstance(section_data, dict): + assert ( + not index + ), f"Got non-zero index ({index}), but section {section} is not a list." elif isinstance(section_data, list): # The 'source' section can be written a list, in which case the name # is passed in with an index, e.g. get_value('source/0/git_url') @@ -1422,8 +1422,9 @@ def get_value(self, name, default=None, autotype=True): section_data = {} else: section_data = section_data[index] - if not isinstance(section_data, dict): - raise ValueError(f"Expected {name} to be a dict") + assert isinstance( + section_data, dict + ), f"Expected {section}/{index} to be a dict" value = section_data.get(key, default) From 826472e7267e4b68546bfc080b57ba801588cf79 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 14 Dec 2023 20:13:27 -0600 Subject: [PATCH 9/9] Update news/5112-fix-multiple-sources Co-authored-by: jaimergp --- news/5112-fix-multiple-sources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/5112-fix-multiple-sources b/news/5112-fix-multiple-sources index 17c0f77398..f988080a8b 100644 --- a/news/5112-fix-multiple-sources +++ b/news/5112-fix-multiple-sources @@ -4,7 +4,7 @@ ### Bug fixes -* Fix finalizing recipes with multiple sources. (#5112) +* Fix finalizing recipes with multiple sources. (#5111 via #5112) ### Deprecations