From 23132054b816161732433690458e722601e182bc Mon Sep 17 00:00:00 2001 From: Dmitry Arkhipov Date: Wed, 20 Mar 2019 16:56:02 +0300 Subject: [PATCH] delaying load of recipe until it is required by builders --- cpt/builds_generator.py | 7 ++-- cpt/lazy_objects.py | 90 +++++++++++++++++++++++++++++++++++++++++ cpt/packager.py | 50 +++++++---------------- cpt/profiles.py | 7 +++- 4 files changed, 114 insertions(+), 40 deletions(-) create mode 100644 cpt/lazy_objects.py diff --git a/cpt/builds_generator.py b/cpt/builds_generator.py index 4f2747f1..3656efe7 100644 --- a/cpt/builds_generator.py +++ b/cpt/builds_generator.py @@ -4,6 +4,7 @@ from conans.model.ref import ConanFileReference from conans.model.version import Version +from cpt.lazy_objects import LazyConanFileReference from cpt.tools import split_colon_env, transform_list_options_to_dict default_gcc_versions = ["4.9", "5", "6", "7", "8"] @@ -182,9 +183,9 @@ def __new__(cls, settings, options, env_vars, build_requires, reference): raise Exception("'env_vars' field has to be a dict") if not isinstance(build_requires, dict): raise Exception("'build_requires' field has to be a dict") - if reference is not None and not isinstance(reference, str) and \ - not isinstance(reference, ConanFileReference): - raise Exception("'reference' field has to be a string or ConanFileReference") + if reference is not None and \ + not isinstance(reference, (str, ConanFileReference, LazyConanFileReference)): + raise Exception("'reference' field has to be a string, ConanFileReference or LazyConanFileReference") if isinstance(reference, str): reference = ConanFileReference.loads(reference) diff --git a/cpt/lazy_objects.py b/cpt/lazy_objects.py new file mode 100644 index 00000000..9c12ca28 --- /dev/null +++ b/cpt/lazy_objects.py @@ -0,0 +1,90 @@ +import functools +import os + +from conans import __version__ as client_version +from conans.model.ref import ConanFileReference +from conans.model.version import Version + + +class LazyObject(object): + def __init__(self, api, cwd, recipe_name): + self._api = api + self._cwd = cwd + self._recipe_name = recipe_name + + @property + def recipe_path(self): + return os.path.join(self._cwd, self._recipe_name) + + def load_recipe(self): + path = self.recipe_path + if not os.path.exists(path): + return + + if Version(client_version) < Version("1.7.0"): + from conans.client.loader_parse import load_conanfile_class + return load_conanfile_class(path) + else: + return self._api._loader.load_class(path) + + +class LazyPackageOption(LazyObject): + def __init__(self, name, api, cwd, recipe_name, reference): + super(LazyPackageOption, self).__init__(api, cwd, recipe_name) + self._name = name + self._reference = reference + self._loaded = False + + def __str__(self): + if not self._loaded: + self._load() + return self._name + + def _load(self): + recipe = self.load_recipe() + if recipe and hasattr(recipe, "options") and recipe.options \ + and self._name in recipe.options: + self._name = "%s:shared" % self._reference.name + else: + self._name = "" + self._loaded = True + + +class LazyConanFileReference(LazyObject): + def __init__(self, api, cwd, recipe_name, reference_text, username, channel): + super(LazyConanFileReference, self).__init__(api, cwd, recipe_name) + self._reference_text = reference_text + self._username = username + self._channel = channel + self._reference = None + + def __getattr__(self, name): + if self._reference is None: + self._load() + + return getattr(self._reference, name) + + def __iter__(self): + return self.__getattr__("__iter__")() + + def _load(self): + if self._reference_text: + if "@" in self._reference_text: + reference = ConanFileReference.loads(self._reference_text) + else: + name, version = self._reference_text.split("/") + reference = ConanFileReference(name, version, self._username, self._channel) + else: + recipe = self.load_recipe() + if recipe is None: + raise Exception("Conanfile not found, specify a 'reference' " + "parameter with name and version") + + name, version = recipe.name, recipe.version + if name and version: + reference = ConanFileReference(name, version, self._username, self._channel) + else: + raise Exception("Specify a CONAN_REFERENCE or name and version " + "fields in the recipe") + + self._reference = reference diff --git a/cpt/packager.py b/cpt/packager.py index dd64be03..29d4468e 100644 --- a/cpt/packager.py +++ b/cpt/packager.py @@ -8,13 +8,13 @@ from conans import __version__ as client_version, tools from conans.client.conan_api import Conan from conans.client.runner import ConanRunner -from conans.model.ref import ConanFileReference from conans.model.version import Version from cpt import NEWEST_CONAN_SUPPORTED from cpt.auth import AuthManager from cpt.builds_generator import BuildConf, BuildGenerator from cpt.ci_manager import CIManager +from cpt.lazy_objects import LazyConanFileReference, LazyPackageOption from cpt.printer import Printer from cpt.profiles import get_profiles, save_profile_to_tmp from cpt.remotes import RemotesManager @@ -24,14 +24,6 @@ from cpt.uploader import Uploader -def load_cf_class(path, conan_api): - if Version(client_version) < Version("1.7.0"): - from conans.client.loader_parse import load_conanfile_class - return load_conanfile_class(path) - else: - return conan_api._loader.load_class(path) - - class PlatformInfo(object): """Easy mockable for testing""" @staticmethod @@ -161,26 +153,12 @@ def __init__(self, username=None, channel=None, runner=None, self.stable_channel = stable_channel or os.getenv("CONAN_STABLE_CHANNEL", "stable") self.stable_channel = self.stable_channel.rstrip() self.channel = self._get_channel(self.specified_channel, self.stable_channel, self.upload_only_when_tag) - self.partial_reference = reference or os.getenv("CONAN_REFERENCE", None) self.conanfile = conanfile or os.getenv("CONAN_CONANFILE", "conanfile.py") - - if self.partial_reference: - if "@" in self.partial_reference: - self.reference = ConanFileReference.loads(self.partial_reference) - else: - name, version = self.partial_reference.split("/") - self.reference = ConanFileReference(name, version, self.username, self.channel) - else: - if not os.path.exists(os.path.join(self.cwd, self.conanfile)): - raise Exception("Conanfile not found, specify a 'reference' " - "parameter with name and version") - - conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile), self.conan_api) - name, version = conanfile.name, conanfile.version - if name and version: - self.reference = ConanFileReference(name, version, self.username, self.channel) - else: - self.reference = None + self.reference = LazyConanFileReference( + self.conan_api, self.cwd, self.conanfile, + reference or os.getenv("CONAN_REFERENCE", None), + self.username, self.channel + ) self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE", None) @@ -388,14 +366,11 @@ def login(self, remote_name): def add_common_builds(self, shared_option_name=None, pure_c=True, dll_with_static_runtime=False, reference=None): - if not reference and not self.reference: - raise Exception("Specify a CONAN_REFERENCE or name and version fields in the recipe") - if shared_option_name is None: if os.path.exists(os.path.join(self.cwd, self.conanfile)): - conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile), self.conan_api) - if hasattr(conanfile, "options") and conanfile.options and "shared" in conanfile.options: - shared_option_name = "%s:shared" % self.reference.name + shared_option_name = LazyPackageOption( + "shared", self.conan_api, self.cwd, self.conanfile, self.reference + ) tmp = self.build_generator.get_builds(pure_c, shared_option_name, dll_with_static_runtime, reference or self.reference) @@ -407,8 +382,11 @@ def add(self, settings=None, options=None, env_vars=None, build_requires=None, r env_vars = env_vars or {} build_requires = build_requires or {} if reference: - reference = ConanFileReference.loads("%s@%s/%s" % (reference, - self.username, self.channel)) + reference = LazyConanFileReference( + self.conan_api, self.cwd, self.conanfile, + "%s@%s/%s" % (reference, self.username, self.channel), + self.username, self.channel + ) reference = reference or self.reference self._builds.append(BuildConf(settings, options, env_vars, build_requires, reference)) diff --git a/cpt/profiles.py b/cpt/profiles.py index 378354cd..8b823b47 100644 --- a/cpt/profiles.py +++ b/cpt/profiles.py @@ -7,6 +7,11 @@ from conans.util.files import save from conans import __version__ as conan_version + +def actual_options(options): + return [opt for opt in options.items() if str(opt[0])] + + def get_profiles(client_cache, build_config, base_profile_name=None): base_profile_text = "" @@ -31,7 +36,7 @@ def pairs_lines(items): return "\n".join(["%s=%s" % (k, v) for k, v in items]) settings = pairs_lines(sorted(build_config.settings.items())) - options = pairs_lines(build_config.options.items()) + options = pairs_lines(actual_options(build_config.options)) env_vars = pairs_lines(build_config.env_vars.items()) br_lines = "" for pattern, build_requires in build_config.build_requires.items():