diff --git a/conans/client/cache/cache.py b/conans/client/cache/cache.py index 5b921e28ac3..b21d4804c03 100644 --- a/conans/client/cache/cache.py +++ b/conans/client/cache/cache.py @@ -33,6 +33,7 @@ HOOKS_FOLDER = "hooks" TEMPLATES_FOLDER = "templates" GENERATORS_FOLDER = "generators" +MIDDLEWARE_FOLDER = "middleware" def _is_case_insensitive_os(): @@ -194,6 +195,10 @@ def settings_path(self): def generators_path(self): return os.path.join(self.cache_folder, GENERATORS_FOLDER) + @property + def middleware_path(self): + return os.path.join(self.cache_folder, MIDDLEWARE_FOLDER) + @property def default_profile_path(self): if os.path.isabs(self.config.default_profile): @@ -246,6 +251,17 @@ def generators(self): generators.append(generator) return generators + @property + def middleware(self): + """Returns a list of middleware paths inside the middleware folder""" + middleware = [] + if os.path.exists(self.middleware_path): + for path in os.listdir(self.middleware_path): + middleware_path = os.path.join(self.middleware_path, path) + if os.path.isfile(middleware_path) and middleware_path.endswith(".py"): + middleware.append(middleware_path) + return middleware + def delete_empty_dirs(self, deleted_refs): """ Method called by ConanRemover.remove() to clean up from the cache empty folders :param deleted_refs: The recipe references that the remove() has been removed diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 0508151ddc5..87b080b55b9 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -23,6 +23,7 @@ from conans.client.conanfile.package import run_package_method from conans.client.conf.required_version import check_required_conan_version from conans.client.generators import GeneratorManager +from conans.client.middleware import MiddlewareManager from conans.client.graph.graph import RECIPE_EDITABLE from conans.client.graph.graph_binaries import GraphBinariesAnalyzer from conans.client.graph.graph_manager import GraphManager @@ -201,8 +202,11 @@ def __init__(self, cache_folder, user_io, http_requester=None, runner=None, quie self.python_requires = ConanPythonRequire(self.proxy, self.range_resolver, self.generator_manager) self.pyreq_loader = PyRequireLoader(self.proxy, self.range_resolver) + self.middleware_manager = MiddlewareManager(self.proxy, self.range_resolver) self.loader = ConanFileLoader(self.runner, self.out, self.python_requires, - self.generator_manager, self.pyreq_loader, self.requester) + self.generator_manager, self.pyreq_loader, self.requester, + middleware_manager=self.middleware_manager) + self.loader.load_middleware_from_cache(self.cache) self.binaries_analyzer = GraphBinariesAnalyzer(self.cache, self.out, self.remote_manager) self.graph_manager = GraphManager(self.out, self.cache, self.remote_manager, self.loader, @@ -215,6 +219,7 @@ def load_remotes(self, remote_name=None, update=False, check_updates=False): self.python_requires.enable_remotes(update=update, check_updates=check_updates, remotes=remotes) self.pyreq_loader.enable_remotes(update=update, check_updates=check_updates, remotes=remotes) + self.middleware_manager.enable_remotes(update=update, check_updates=check_updates, remotes=remotes) return remotes diff --git a/conans/client/loader.py b/conans/client/loader.py index 477b6ed3604..d680c2bad6c 100644 --- a/conans/client/loader.py +++ b/conans/client/loader.py @@ -14,6 +14,7 @@ conanfile_exception_formatter from conans.model.conan_file import ConanFile from conans.model.conan_generator import Generator +from conans.model.conan_middleware import Middleware from conans.model.options import OptionsValues from conans.model.ref import ConanFileReference from conans.model.settings import Settings @@ -24,9 +25,10 @@ class ConanFileLoader(object): def __init__(self, runner, output, python_requires, generator_manager=None, pyreq_loader=None, - requester=None): + requester=None, middleware_manager=None): self._runner = runner self._generator_manager = generator_manager + self._middleware_manager = middleware_manager self._output = output self._pyreq_loader = pyreq_loader self._python_requires = python_requires @@ -34,20 +36,29 @@ def __init__(self, runner, output, python_requires, generator_manager=None, pyre self._cached_conanfile_classes = {} self._requester = requester + def apply_middleware(self, conanfile_path, conanfile, user=None, channel=None, + profile=None, consumer=False): + if not profile: + return conanfile + return self._middleware_manager.apply_middleware(conanfile_path, conanfile, user, channel, + consumer, profile, self) + def load_basic(self, conanfile_path, lock_python_requires=None, user=None, channel=None, - display=""): + display="", profile=None): """ loads a conanfile basic object without evaluating anything """ return self.load_basic_module(conanfile_path, lock_python_requires, user, channel, - display)[0] + display, profile)[0] def load_basic_module(self, conanfile_path, lock_python_requires=None, user=None, channel=None, - display=""): + display="", profile=None, consumer=False): """ loads a conanfile basic object without evaluating anything, returns the module too """ cached = self._cached_conanfile_classes.get(conanfile_path) if cached and cached[1] == lock_python_requires: - conanfile = cached[0](self._output, self._runner, display, user, channel) + recipe_class = self.apply_middleware(conanfile_path, cached[0], user, channel, + profile, consumer) + conanfile = recipe_class(self._output, self._runner, display, user, channel) conanfile._conan_requester = self._requester if hasattr(conanfile, "init") and callable(conanfile.init): with conanfile_exception_formatter(str(conanfile), "init"): @@ -83,6 +94,8 @@ def load_basic_module(self, conanfile_path, lock_python_requires=None, user=None if scm_data: conanfile.scm.update(scm_data) + conanfile = self.apply_middleware(conanfile_path, conanfile, user, channel, + profile, consumer) self._cached_conanfile_classes[conanfile_path] = (conanfile, lock_python_requires, module) result = conanfile(self._output, self._runner, display, user, channel) @@ -110,6 +123,35 @@ def load_generators(self, conanfile_path): if issubclass(attr, Generator) and attr != Generator: self._generator_manager.add(attr.__name__, attr, custom=True) + def load_middleware(self, conanfile_path): + """ Load middleware classes from a module. Any non-middleware classes + will be ignored. python_requires is not processed. + """ + """ Parses a python in-memory module and adds any middleware found + to the provided middleware list + @param conanfile_module: the module to be processed + """ + conanfile_module, module_id = _parse_conanfile(conanfile_path) + middleware = None + for name, attr in conanfile_module.__dict__.items(): + if (name.startswith("_") or not inspect.isclass(attr) or + attr.__dict__.get("__module__") != module_id): + continue + if issubclass(attr, Middleware) and attr != Middleware: + self._middleware_manager.add(attr.__name__, attr, custom=True) + if middleware is None: + middleware = attr + if middleware is None: + raise ConanException("No subclass of Middleware in the file") + return middleware, conanfile_module + + def load_middleware_from_cache(self, cache): + # Load custom middleware from the cache. + # Middleware loaded here from the cache will have precedence + # and overwrite possible middleware loaded from packages. + for middleware_path in cache.middleware: + self.load_middleware(middleware_path) + @staticmethod def _load_data(conanfile_path): data_path = os.path.join(os.path.dirname(conanfile_path), DATA_YML) @@ -123,10 +165,12 @@ def _load_data(conanfile_path): return data or {} - def load_named(self, conanfile_path, name, version, user, channel, lock_python_requires=None): + def load_named(self, conanfile_path, name, version, user, channel, lock_python_requires=None, + profile=None, consumer=False): """ loads the basic conanfile object and evaluates its name and version """ - conanfile, _ = self.load_basic_module(conanfile_path, lock_python_requires, user, channel) + conanfile, _ = self.load_basic_module(conanfile_path, lock_python_requires, user, channel, + profile=profile, consumer=consumer) # Export does a check on existing name & version if name: @@ -207,7 +251,7 @@ def load_consumer(self, conanfile_path, profile_host, name=None, version=None, u """ loads a conanfile.py in user space. Might have name/version or not """ conanfile = self.load_named(conanfile_path, name, version, user, channel, - lock_python_requires) + lock_python_requires, profile_host, consumer=True) ref = ConanFileReference(conanfile.name, conanfile.version, user, channel, validate=False) if str(ref): @@ -243,7 +287,7 @@ def load_conanfile(self, conanfile_path, profile, ref, lock_python_requires=None """ try: conanfile, _ = self.load_basic_module(conanfile_path, lock_python_requires, - ref.user, ref.channel, str(ref)) + ref.user, ref.channel, str(ref), profile) except Exception as e: raise ConanException("%s: Cannot load recipe.\n%s" % (str(ref), str(e))) @@ -272,7 +316,8 @@ def load_conanfile_txt(self, conan_txt_path, profile_host, ref=None): return conanfile def _parse_conan_txt(self, contents, path, display_name, profile): - conanfile = ConanFile(self._output, self._runner, display_name) + conanfile = self.apply_middleware(path, ConanFile, profile=profile, consumer=True) + conanfile = conanfile(self._output, self._runner, display_name) tmp_settings = profile.processed_settings.copy() package_settings_values = profile.package_settings_values if "&" in package_settings_values: diff --git a/conans/client/middleware/__init__.py b/conans/client/middleware/__init__.py new file mode 100644 index 00000000000..b277c5344b9 --- /dev/null +++ b/conans/client/middleware/__init__.py @@ -0,0 +1,178 @@ +import fnmatch +import os + +from conans.client.recorder.action_recorder import ActionRecorder +from conans.errors import ConanException +from conans.model.ref import ConanFileReference +from conans.model.requires import Requirement + + +class MiddlewareLoader(object): + def __init__(self, proxy, range_resolver): + self._proxy = proxy + self._range_resolver = range_resolver + self._cached_modules = {} + + def enable_remotes(self, check_updates=False, update=False, remotes=None): + self._check_updates = check_updates + self._update = update + self._remotes = remotes + + def resolve_middleware_requires(self, loader, conanfile, profile, user, channel, consumer=True): + ref = ConanFileReference(conanfile.name, conanfile.version, user, channel, validate=False) + ref_str = str(ref) + recipe_middleware_requires = [] + for pattern, refs in profile.middleware_requires.items(): + consumer = False + if ((consumer and pattern == "&") or + (not consumer and pattern == "&!") or + fnmatch.fnmatch(ref_str, pattern)): + for ref in refs: + if ref not in recipe_middleware_requires: + recipe_middleware_requires.append(ref) + self._resolve_middleware_requires(recipe_middleware_requires, loader) + return recipe_middleware_requires + + def _resolve_middleware_requires(self, middleware_refs, loader): + for ref in middleware_refs: + ref_str = str(ref) + if ref_str not in self._cached_modules: + self._cached_modules[ref_str] = self._load_middleware_conanfile(loader, ref) + + def _load_middleware_conanfile(self, loader, ref): + requirement = Requirement(ref) + self._range_resolver.resolve(requirement, "middleware", update=self._update, + remotes=self._remotes) + new_ref = requirement.ref + recipe = self._proxy.get_recipe(new_ref, self._check_updates, self._update, + remotes=self._remotes, recorder=ActionRecorder()) + path, _, _, new_ref = recipe + middleware, module = loader.load_middleware(path) + middleware.name = new_ref.name + # FIXME Conan 2.0 version should be a string, not a Version object + middleware.version = new_ref.version + + if getattr(middleware, "alias", None): + ref = ConanFileReference.loads(middleware.alias) + requirement = Requirement(ref) + alias = requirement.alias + if alias is not None: + ref = alias + return self._load_middleware_conanfile(loader, ref) + return middleware, module, new_ref, os.path.dirname(path) + + +class MiddlewareManager(object): + def __init__(self, proxy, range_resolver): + self._loader = MiddlewareLoader(proxy, range_resolver) + # Future built-in middleware go here. + self._middleware = {} + + def enable_remotes(self, *args, **kwargs): + self._loader.enable_remotes(*args, **kwargs) + + def add(self, name, middleware_class, custom=False): + if name not in self._middleware or custom: + self._middleware[name] = middleware_class + + def __contains__(self, name): + return name in self._middleware + + def __getitem__(self, key): + return self._middleware[key] + + @staticmethod + def _initialize_middleware(middleware, profile): + # Prepare the settings for the loaded middleware + # Mixing the global settings with the specified for that name if exist + tmp_settings = profile.processed_settings.copy() + package_settings_values = profile.package_settings_values + ref = ConanFileReference(middleware.name or middleware.__class__.__name__, + middleware.version or "1.0", + user=None, channel=None, validate=False) + ref_str = str(ref) + if package_settings_values: + # First, try to get a match directly by name (without needing *) + # TODO: Conan 2.0: We probably want to remove this, and leave a pure fnmatch + pkg_settings = package_settings_values.get(middleware.name) + + if middleware.develop and "&" in package_settings_values: + # "&" overrides the "name" scoped settings. + pkg_settings = package_settings_values.get("&") + + if pkg_settings is None: # If there is not exact match by package name, do fnmatch + for pattern, settings in package_settings_values.items(): + if fnmatch.fnmatchcase(ref_str, pattern): + pkg_settings = settings + break + if pkg_settings: + tmp_settings.update_values(pkg_settings) + + middleware.initialize(tmp_settings, profile.env_values, profile.buildenv) + middleware.conf = profile.conf.get_conanfile_conf(ref_str) + + def find_middleware(self, middleware_name): + try: + return self._middleware[middleware_name] + except KeyError: + available = list(self._middleware.keys()) + raise ConanException("Invalid middleware '%s'. Available types: %s" % + (middleware_name, ", ".join(available))) + + def new_middleware(self, middleware_name, profile, loader): + cache = profile.cached_middleware + if middleware_name in cache.middleware: + return cache.middleware[middleware_name] + middleware_class = self.find_middleware(middleware_name) + middleware = middleware_class(loader._output, middleware_name) + middleware.output.scope = middleware.display_name + self._initialize_middleware(middleware, profile) + cache.middleware[middleware_name] = middleware + return middleware + + def _apply_middleware(self, middleware, conanfile_path, conanfile, user, channel, profile, loader): + result = conanfile + applied = [] + for mw in middleware: + if mw.should_apply(result): + result = mw(result) + applied.append(mw) + if applied: + # This might be useful if multiple middleware needs to work together? + result._middleware = applied + return result + + def apply_middleware(self, conanfile_path, conanfile, user, channel, consumer, profile, loader): + if not profile: + return conanfile + if not profile.middleware: + return conanfile + # Don't re-apply middleware + if hasattr(conanfile, "_middleware"): + return conanfile + if conanfile_path in profile.cached_middleware: + return profile.cached_middleware[conanfile_path] + # Load middleware_requires first + self._loader.resolve_middleware_requires(loader, conanfile, profile, user, channel, consumer) + # Find middleware by conanfile and profile, then instantiate with profile settings + middleware = self.get_recipe_middleware(conanfile, profile, user, channel, consumer) + middleware = [self.new_middleware(mw, profile, loader) for mw in middleware] + # Subclass conanfile + result = self._apply_middleware(middleware, conanfile_path, conanfile, + user, channel, profile, loader) + profile.cached_middleware[conanfile_path] = result + return result + + def get_recipe_middleware(self, conanfile, profile, user, channel, consumer=True): + ref = ConanFileReference(conanfile.name, conanfile.version, user, channel, validate=False) + ref_str = str(ref) + recipe_middleware = [] + for pattern, middleware in profile.middleware.items(): + consumer = False + if ((consumer and pattern == "&") or + (not consumer and pattern == "&!") or + fnmatch.fnmatch(ref_str, pattern)): + for mw in middleware: + if mw not in recipe_middleware: + recipe_middleware.append(mw) + return recipe_middleware diff --git a/conans/client/middleware/cache.py b/conans/client/middleware/cache.py new file mode 100644 index 00000000000..bf3ee0f4ba7 --- /dev/null +++ b/conans/client/middleware/cache.py @@ -0,0 +1,19 @@ +class MiddlewareCache(object): + def __init__(self): + # Cached middleware instances with profile settings + self._middleware = {} + # Recipes wrapped with middleware + self._recipes = {} + + def __contains__(self, path): + return path in self._recipes + + def __getitem__(self, path): + return self._recipes[path] + + def __setitem__(self, path, recipe): + self._recipes[path] = recipe + + @property + def middleware(self): + return self._middleware diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index 1ff366141d8..79b90ecfec3 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -162,7 +162,7 @@ def _load_profile(text, profile_path, default_folder): # Current profile before update with parents (but parent variables already applied) doc = ConfigParser(profile_parser.profile_text, allowed_fields=["build_requires", "settings", "env", "options", "conf", - "buildenv"]) + "buildenv", "middleware_requires", "middleware"]) # Merge the inherited profile with the readed from current profile _apply_inner_profile(doc, inherited_profile) @@ -185,6 +185,28 @@ def _load_single_build_require(profile, line): profile.build_requires.setdefault(pattern, []).extend(refs) +def _load_single_middleware_require(profile, line): + + tokens = line.split(":", 1) + if len(tokens) == 1: + pattern, req_list = "*", line + else: + pattern, req_list = tokens + refs = [ConanFileReference.loads(reference.strip()) for reference in req_list.split(",")] + profile.middleware_requires.setdefault(pattern, []).extend(refs) + + +def _load_single_middleware(profile, line): + + tokens = line.split(":", 1) + if len(tokens) == 1: + pattern, mw_list = "*", line + else: + pattern, mw_list = tokens + middleware = [mw.strip() for mw in mw_list.split(",")] + profile.middleware.setdefault(pattern, []).extend(middleware) + + def _apply_inner_profile(doc, base_profile): """ @@ -222,6 +244,16 @@ def get_package_name_value(item): for req in doc.build_requires.splitlines(): _load_single_build_require(base_profile, req) + if doc.middleware_requires: + # FIXME CHECKS OF DUPLICATED? + for req in doc.middleware_requires.splitlines(): + _load_single_middleware_require(base_profile, req) + + if doc.middleware: + # FIXME CHECKS OF DUPLICATED? + for req in doc.middleware.splitlines(): + _load_single_middleware(base_profile, req) + if doc.options: base_profile.options.update(OptionsValues.loads(doc.options)) diff --git a/conans/model/__init__.py b/conans/model/__init__.py index b7a948e27df..58c068e529f 100644 --- a/conans/model/__init__.py +++ b/conans/model/__init__.py @@ -1 +1,2 @@ from .conan_generator import Generator +from .conan_middleware import Middleware diff --git a/conans/model/conan_middleware.py b/conans/model/conan_middleware.py new file mode 100644 index 00000000000..30201d5abaf --- /dev/null +++ b/conans/model/conan_middleware.py @@ -0,0 +1,129 @@ +import inspect +import os + +from conans.client.output import ScopedOutput +from conans.errors import ConanException, ConanInvalidConfiguration +from conans.model.values import Values +from conans.util.conan_v2_mode import conan_v2_error + + +def create_options(middleware): + # avoid circular imports + from conans.model.conan_file import create_options + return create_options(middleware) + +def create_settings(middleware, settings): + try: + defined_settings = getattr(middleware, "settings", None) + if isinstance(defined_settings, str): + defined_settings = [defined_settings] + current = defined_settings or {} + settings.constraint(current) + return settings + except Exception as e: + raise ConanInvalidConfiguration("The middleware %s is constraining settings. %s" % ( + middleware.display_name, str(e))) + + +class Middleware(object): + """ The base class for all middleware + """ + + name = None + version = None # Any str, can be "1.1" or whatever + url = None # The URL where this File is located, as github, to collaborate in package + # The license of the PACKAGE, just a shortcut, does not replace or + # change the actual license of the source code + license = None + author = None # Main maintainer/responsible for the package, any format + description = None + topics = None + homepage = None + + # Settings and Options + settings = None + options = None + default_options = None + + def __init__(self, output, display_name=""): + # an output stream (writeln, info, warn error) + self.output = ScopedOutput(display_name, output) + self.display_name = display_name + + def initialize(self, settings, env, buildenv=None): + self._conan_buildenv = buildenv + # User defined options + self.options = create_options(self) + self.settings = create_settings(self, settings) + + def config_options(self): + """ modify options, probably conditioned to some settings. This call is executed + before config_settings. E.g. + if self.settings.os == "Windows": + del self.options.shared # shared/static not supported in win + """ + + def configure(self): + """ modify settings, probably conditioned to some options. This call is executed + after config_options. E.g. + if self.options.header_only: + self.settings.clear() + This is also the place for conditional requirements + """ + + @staticmethod + def extend(a, b): + if a is None: + return b + elif b is None: + return a + elif isinstance(a, (tuple, list)) and isinstance(b, (tuple, list)): + return tuple(a) + tuple(b) + elif isinstance(a, dict) and isinstance(b, dict): + c = a.copy() + c.update(b) + return c + else: + raise TypeError("extend expects tuples/lists or dicts") + + @staticmethod + def is_binary(conanfile): + """ does this ConanFile class or instance produce binaries? + For conanfile.txt base will be None. + """ + if inspect.isclass(conanfile): + # We can't see values yet + settings = conanfile.settings + if not (settings and "os" in settings and "arch" in settings): + return False + elif conanfile: + try: + if conanfile.options.header_only: + # Header only + return False + except ConanException: + pass + try: + conanfile.settings.arch + conanfile.settings.os + except ConanException: + # arch or os is not required + return False + return True + + def should_apply(self, base): + """ should this middleware be applied to a given base class? + base is a subclass of ConanFile not an instance. + For conanfile.txt base will be None. + """ + return True + + def __call__(self, base): + """ apply this middleware to a given base class. + base is a subclass of ConanFile not an instance. + Returns a new subclass of ConanFile. + """ + return base + + def __repr__(self): + return self.display_name diff --git a/conans/model/profile.py b/conans/model/profile.py index f5112a41ee5..8dfc2baf0b0 100644 --- a/conans/model/profile.py +++ b/conans/model/profile.py @@ -9,6 +9,7 @@ from conans.model.options import OptionsValues from conans.model.ref import ConanFileReference from conans.model.values import Values +from conans.client.middleware.cache import MiddlewareCache class Profile(object): @@ -24,6 +25,9 @@ def __init__(self): self.build_requires = OrderedDict() # ref pattern: list of ref self.conf = ConfDefinition() self.buildenv = ProfileEnvironment() + self.middleware_requires = OrderedDict() # ref pattern: list of ref + self.middleware = OrderedDict() # ref pattern: list of str + self._cached_middleware = MiddlewareCache() # Cached processed values self.processed_settings = None # Settings with values, and smart completion @@ -31,6 +35,10 @@ def __init__(self): self._package_settings_values = None self.dev_reference = None # Reference of the package being develop + @property + def cached_middleware(self): + return self._cached_middleware + @property def user_options(self): if self._user_options is None: @@ -94,6 +102,16 @@ def dumps(self): result.append("[buildenv]") result.append(self.buildenv.dumps()) + if self.middleware_requires: + result.append("[middleware_requires]") + for pattern, req_list in self.middleware_requires.items(): + result.append("%s: %s" % (pattern, ", ".join(str(r) for r in req_list))) + + if self.middleware: + result.append("[middleware]") + for pattern, mw_list in self.middleware.items(): + result.append("%s: %s" % (pattern, ", ".join(str(mw) for mw in mw_list))) + return "\n".join(result).replace("\n\n", "\n") def compose_profile(self, other): @@ -118,6 +136,33 @@ def compose_profile(self, other): if not isinstance(req, ConanFileReference) else req existing[r.name] = req self.build_requires[pattern] = list(existing.values()) + # It is possible that middleware are repeated, or same package but different versions + for pattern, req_list in other.middleware_requires.items(): + existing_middleware_requires = self.middleware_requires.get(pattern) + existing = OrderedDict() + if existing_middleware_requires is not None: + for mw in existing_middleware_requires: + # TODO: Understand why sometimes they are str and other are ConanFileReference + r = ConanFileReference.loads(mw) \ + if not isinstance(mw, ConanFileReference) else mw + existing[r.name] = mw + for req in req_list: + r = ConanFileReference.loads(req) \ + if not isinstance(req, ConanFileReference) else req + existing[r.name] = req + self.middleware_requires[pattern] = list(existing.values()) + # It is possible that middleware are repeated + for pattern, mw_list in other.middleware.items(): + existing_middleware = self.middleware.get(pattern) + existing = OrderedDict() + if existing_middleware is not None: + for mw in existing_middleware: + existing[mw] = mw + for mw in mw_list: + existing[mw] = mw + self.middleware[pattern] = list(existing.values()) + # Rebuild middleware cache with new settings + self._cached_middleware = MiddlewareCache() self.conf.update_conf_definition(other.conf) self.buildenv.update_profile_env(other.buildenv) # Profile composition, last has priority