diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index 669bb8e8..5bd7628f 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -2009,54 +2009,19 @@ found:: Required include files have a lower precedence that optional include files. -.. _configuration-config-environment: - -environment -~~~~~~~~~~~ +alias +~~~~~ Type: Dictionary (String -> String) -Specifies default environment variables. Example:: - - environment: - # Number of make jobs is determined by the number of available processors - # (nproc). If desired it can be set to a specific number, e.g. "2". See - # classes/make.yaml for details. - MAKE_JOBS: "nproc" - -If the :ref:`policies-cleanEnvironment` policy is enabled then these variables -are subject to :ref:`configuration-principle-subst` with the current OS -environment. This allows to take over certain variables from the OS environment -in a controlled fashion. - -.. _configuration-config-whitelist: - -whitelist -~~~~~~~~~ - -Type: List of Strings - -Specifies a list of environment variable keys that should be passed unchanged -to all scripts during execution. The content of these variables are considered -invariants of the build. It is no error if any variable specified in this list -is not set. By default the following environment variables are passed to all -scripts: - -* Linux and other POSIX platforms: ``TERM``, ``SHELL``, ``USER``, ``HOME`` -* Windows: ``ALLUSERSPROFILE``, ``APPDATA``, ``COMMONPROGRAMFILES``, - ``COMMONPROGRAMFILES(X86)``, ``COMSPEC``, ``HOMEDRIVE``, ``HOMEPATH``, - ``LOCALAPPDATA``, ``PATH``, ``PATHEXT``, ``PROGRAMDATA``, ``PROGRAMFILES``, - ``PROGRAMFILES(X86)``, ``SYSTEMDRIVE``, ``SYSTEMROOT``, ``TEMP``, ``TMP``, - ``WINDIR`` -* MSYS2: Union of POSIX and Windows white list - -The names given with ``whitelist`` are *added* to the list and does not replace -the default list. +Aliases allow a string to be substituted for the first step of a +:ref:`relative location path `:: -Example:: + alias: + myApp: "host/files/group/app42" + allTests: "//*-unittest" - # Keep ssh-agent working - whitelist: ["SSH_AGENT_PID", "SSH_AUTH_SOCK"] +See :ref:`manpage-bobpaths-aliases` for the rules that apply to aliases. .. _configuration-config-archive: @@ -2189,88 +2154,20 @@ the anonymous access to the container can be used like this:: The ``flags: [download]`` makes sure that Bob does not try to upload artifacts in case other backends are configured too. -.. _configuration-config-scmDefaults: - -scmDefaults -~~~~~~~~~~~ - -Type: Dict of SCM dicts - -Default settings for SCMs are applied if the value is not set in the -recipe. Useable values are marked with `(*)` in (:ref:`configuration-recipes-scm`). - -Example:: - - scmDefaults: - git: - branch: "main" - singleBranch: True - retries: 3 - url: - extract: False - -.. _configuration-config-scmOverrides: - -scmOverrides -~~~~~~~~~~~~ - -Type: List of override specifications - -SCM overrides allow the user to alter any attribute of SCMs -(:ref:`configuration-recipes-scm`) without touching the recipes. They are quite -useful to change e.g. the server url or to override the branch of some SCMs. Overrides -are applied after string substitution. The general syntax looks like the following:: - - scmOverrides: - - - match: - url: "git@acme.com:/foo/repo.git" - del: [commit, tag] - set: - branch: develop - replace: - url: - pattern: "foo" - replacement: "bar" - if: !expr | - "${BOB_RECIPE_NAME}" == "foo" - -The ``scmOverrides`` key takes a list of one or more override specifications. -You can select overrides using a ``if`` expression. If ``if`` condition evaluates to true -the override is first matched via pattens that are in the ``match`` section. -All entries under ``match`` must be matching for the override to apply. The -right side of a match entry can use shell globbing patterns. - -If an override is matching the actions are then applied in the following order: - - * ``del``: The list of attributes that are removed. - * ``set``: The attributes and their values are taken, overwriting previous values. - * ``replace``: Performs a substitution based on regular expressions. This - section can hold any number of attributes with a ``pattern`` and a - ``replacement``. Each occurrence of ``pattern`` is replaced by - ``replacement``. - -All overrides values are mangled through :ref:`configuration-principle-subst`. Mangling is -performed during calculation of the checkoutStep so that the full environment for this step is -available for substitution. - -When an override is applied the ``overridden`` property of the SCM is set to -true. This property can be used with the ``matchScm`` function in package -queries to find packages whose SCM(s) have been overridden. - -alias -~~~~~ +.. _configuration-config-archive-prepend-append: -Type: Dictionary (String -> String) +archive{Prepend,Append} +~~~~~~~~~~~~~~~~~~~~~~~ -Aliases allow a string to be substituted for the first step of a -:ref:`relative location path `:: +Type: Dictionary or list of dictionaries - alias: - myApp: "host/files/group/app42" - allTests: "//*-unittest" +These keys receive the same archive specification(s) like the :ref:`configuration-config-archive` +keyword. Compared to the ``archive`` key, which replaces the currently configured +archives, the ``archivePrepend`` key prepends the given archive(s) to the current list and +``archiveAppend`` appends to it. See :ref:`configuration-config-archive` for more details. -See :ref:`manpage-bobpaths-aliases` for the rules that apply to aliases. +It is usually advisable to use these keywords instead of ``archive`` to enable +interoperability between projects, layers and the local user configuration. .. _configuration-config-commands: @@ -2356,6 +2253,26 @@ type "d3" or "dot" max_depth Integer =============== =================================================================== +.. _configuration-config-environment: + +environment +~~~~~~~~~~~ + +Type: Dictionary (String -> String) + +Specifies default environment variables. Example:: + + environment: + # Number of make jobs is determined by the number of available processors + # (nproc). If desired it can be set to a specific number, e.g. "2". See + # classes/make.yaml for details. + MAKE_JOBS: "nproc" + +If the :ref:`policies-cleanEnvironment` policy is enabled then these variables +are subject to :ref:`configuration-principle-subst` with the current OS +environment. This allows to take over certain variables from the OS environment +in a controlled fashion. + .. _configuration-config-hooks: hooks @@ -2397,10 +2314,8 @@ postBuildHook The return status of the hook is ignored. -.. _configuration-config-rootFilter: - -preMirror / fallbackMirror -~~~~~~~~~~~~~~~~~~~~~~~~~~ +{pre,fallback}Mirror[{Prepend,Append}] +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Type: Mirror-entry or list of mirror-entries @@ -2418,6 +2333,14 @@ between different methods e.g., use a HTTP URL SCM mirror for a git SCM. It is not possible to independently verify the equivalence of the mirror in such a case. +When used without suffix (i.e. ``preMirror`` or ``fallbackMirror``), the +currently configured list of mirrors is replaced. By using the ``Prepend`` +suffix, e.g. ``preMirrorPrepend``, the given mirror(s) is/are prepended to the +configured list of mirrors. Likewise, the ``Append`` suffix (e.g. +``fallbackMirrorAppend``) appends to the list of mirrors. It is advised to use +these suffixes instead of the bare ``preMirror``/``fallbackMirror`` to enable +interoperability between projects, layers and the local user configuration. + Each mirror entry specifies the SCM type (``scm``), a regular expression to match the URL (``url``) and a replacement URL (``mirror``). Optionally, it is possible to upload to the mirror (``upload``). Currently only the URL SCM is @@ -2436,7 +2359,7 @@ Examples:: A typical mirror configuration for the global user configuration could look like the following. It mirrors all remote URLs to a local directory:: - preMirror: + preMirrorPrepend: scm: url url: "https?://.*/(.*)" mirror: "~/.cache/bob/mirror/\\1" @@ -2445,6 +2368,8 @@ like the following. It mirrors all remote URLs to a local directory:: This will put all downloaded files into a caching directory of the current user. +.. _configuration-config-rootFilter: + rootFilter ~~~~~~~~~~ @@ -2487,6 +2412,75 @@ as ``/mnt`` inside the sandbox. The ``/mnt`` directory will be in ``$PATH`` before any other search directory of the sandbox but still after any used tool (if any). +.. _configuration-config-scmDefaults: + +scmDefaults +~~~~~~~~~~~ + +Type: Dict of SCM dicts + +Default settings for SCMs are applied if the value is not set in the +recipe. Useable values are marked with `(*)` in (:ref:`configuration-recipes-scm`). + +Example:: + + scmDefaults: + git: + branch: "main" + singleBranch: True + retries: 3 + url: + extract: False + +.. _configuration-config-scmOverrides: + +scmOverrides +~~~~~~~~~~~~ + +Type: List of override specifications + +SCM overrides allow the user to alter any attribute of SCMs +(:ref:`configuration-recipes-scm`) without touching the recipes. They are quite +useful to change e.g. the server url or to override the branch of some SCMs. Overrides +are applied after string substitution. The general syntax looks like the following:: + + scmOverrides: + - + match: + url: "git@acme.com:/foo/repo.git" + del: [commit, tag] + set: + branch: develop + replace: + url: + pattern: "foo" + replacement: "bar" + if: !expr | + "${BOB_RECIPE_NAME}" == "foo" + +The ``scmOverrides`` key takes a list of one or more override specifications. +You can select overrides using a ``if`` expression. If ``if`` condition evaluates to true +the override is first matched via pattens that are in the ``match`` section. +All entries under ``match`` must be matching for the override to apply. The +right side of a match entry can use shell globbing patterns. + +If an override is matching the actions are then applied in the following order: + + * ``del``: The list of attributes that are removed. + * ``set``: The attributes and their values are taken, overwriting previous values. + * ``replace``: Performs a substitution based on regular expressions. This + section can hold any number of attributes with a ``pattern`` and a + ``replacement``. Each occurrence of ``pattern`` is replaced by + ``replacement``. + +All overrides values are mangled through :ref:`configuration-principle-subst`. Mangling is +performed during calculation of the checkoutStep so that the full environment for this step is +available for substitution. + +When an override is applied the ``overridden`` property of the SCM is set to +true. This property can be used with the ``matchScm`` function in package +queries to find packages whose SCM(s) have been overridden. + .. _configuration-config-share: share @@ -2568,3 +2562,40 @@ queryMode ``nullfail`` An empty set of packages is always treated as an error. + +.. _configuration-config-whitelist: + +whitelist +~~~~~~~~~ + +Type: List of Strings + +Specifies a list of environment variable keys that should be passed unchanged +to all scripts during execution. The content of these variables are considered +invariants of the build. It is no error if any variable specified in this list +is not set. By default the following environment variables are passed to all +scripts: + +* Linux and other POSIX platforms: ``PATH``, ``TERM``, ``SHELL``, ``USER``, ``HOME`` +* Windows: ``ALLUSERSPROFILE``, ``APPDATA``, ``COMMONPROGRAMFILES``, + ``COMMONPROGRAMFILES(X86)``, ``COMSPEC``, ``HOMEDRIVE``, ``HOMEPATH``, + ``LOCALAPPDATA``, ``PATH``, ``PATHEXT``, ``PROGRAMDATA``, ``PROGRAMFILES``, + ``PROGRAMFILES(X86)``, ``SYSTEMDRIVE``, ``SYSTEMROOT``, ``TEMP``, ``TMP``, + ``WINDIR`` +* MSYS2: Union of POSIX and Windows white list + +The names given with ``whitelist`` are *added* to the list and does not replace +the default list. + +Example:: + + # Keep ssh-agent working + whitelist: ["SSH_AGENT_PID", "SSH_AUTH_SOCK"] + +whitelistRemove +~~~~~~~~~~~~~~~ + +Type: List of strings + +Remove the given names from the ``whitelist``. It is not an error to remove a +non-existing name. See :ref:`configuration-config-whitelist` for more details. diff --git a/pym/bob/archive.py b/pym/bob/archive.py index 4967ed9f..f50a39e3 100644 --- a/pym/bob/archive.py +++ b/pym/bob/archive.py @@ -1182,7 +1182,13 @@ def getArchiver(recipes, jenkins=None): archiveSpec = [jenkins] + archiveSpec else: archiveSpec = [jenkins, archiveSpec] + if isinstance(archiveSpec, list): - return MultiArchive([ getSingleArchiver(recipes, i) for i in archiveSpec ]) + if len(archiveSpec) == 0: + return DummyArchive() + elif len(archiveSpec) == 1: + return getSingleArchiver(recipes, archiveSpec[0]) + else: + return MultiArchive([ getSingleArchiver(recipes, i) for i in archiveSpec ]) else: return getSingleArchiver(recipes, archiveSpec) diff --git a/pym/bob/input.py b/pym/bob/input.py index a85ea4f1..86f31c9c 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -284,6 +284,8 @@ def onFinish(self, env, properties): class PluginSetting: + priority = 50 + """Base class for plugin settings. Plugins can be configured in the user configuration of a project. The @@ -365,13 +367,24 @@ def newFun(args, **kwargs): return newFun +class SentinelSetting(PluginSetting): + """Sentinel for "include" and "require" settings""" + + def __init__(self): + pass + + def merge(self, other): + pass + + class BuiltinSetting(PluginSetting): """Tiny wrapper to define Bob built-in settings""" - def __init__(self, schema, updater, mangle = False): + def __init__(self, schema, updater, mangle = False, priority=50): self.__schema = schema self.__updater = updater self.__mangle = mangle + self.priority = priority def merge(self, other): self.__updater(self.__schema.validate(other) if self.__mangle else other) @@ -2922,8 +2935,14 @@ def __init__(self): } def validate(self, data): - self.__validTypes.validate(data) - return self.__backends[data['backend']].validate(data) + if isinstance(data, dict): + self.__validTypes.validate(data) + return [ self.__backends[data['backend']].validate(data) ] + elif isinstance(data, list): + return [ self.__validTypes.validate(i) and self.__backends[i['backend']].validate(i) + for i in data ] + else: + raise schema.SchemaError(None, "Invalid archive specification!") class MountValidator: def __init__(self): @@ -3047,7 +3066,7 @@ def __init__(self): self.__aliases = {} self.__recipes = {} self.__classes = {} - self.__archive = { "backend" : "none" } + self.__archive = [] self.__rootFilter = [] self.__scmOverrides = [] self.__hooks = {} @@ -3149,8 +3168,14 @@ def __init__(self): self.__preMirrors = [] self.__fallbackMirrors = [] + def appendArchive(x): self.__archive.extend(x) + def prependArchive(x): self.__archive[0:0] = x def updateArchive(x): self.__archive = x + def appendPreMirror(x) : self.__preMirrors.extend(x) + def prependPreMirror(x) : self.__preMirrors[0:0] = x def updatePreMirror(x) : self.__preMirrors = x + def appendFallbackMirror(x) : self.__fallbackMirrors.extend(x) + def prependFallbackMirror(x) : self.__fallbackMirrors[0:0] = x def updateFallbackMirror(x) : self.__fallbackMirrors = x def updateWhiteList(x): @@ -3161,18 +3186,27 @@ def updateWhiteList(x): else: self.__whiteList.update(x) + def removeWhiteList(x): + if self.__platform == "win32": + # Convert to upper case on Windows. The Python interpreter does that + # too and the variables are considered case insensitive by Windows. + self.__whiteList.difference_update(i.upper() for i in x) + else: + self.__whiteList.difference_update(x) + + archiveValidator = ArchiveValidator() + self.__settings = { + "include" : SentinelSetting(), + "require" : SentinelSetting(), + "alias" : BuiltinSetting( schema.Schema({ schema.Regex(r'^[0-9A-Za-z_-]+$') : str }), lambda x: self.__aliases.update(x) ), - "archive" : BuiltinSetting( - schema.Or( - ArchiveValidator(), - schema.Schema( [ArchiveValidator()] ) - ), - updateArchive - ), + "archive" : BuiltinSetting(archiveValidator, updateArchive, True), + "archiveAppend" : BuiltinSetting(archiveValidator, appendArchive, True, 100), + "archivePrepend" : BuiltinSetting(archiveValidator, prependArchive, True, 100), "command" : BuiltinSetting( schema.Schema({ schema.Optional('dev') : self.BUILD_DEV_SCHEMA, @@ -3185,11 +3219,9 @@ def updateWhiteList(x): VarDefineValidator("environment"), lambda x: self.__defaultEnv.update(x) ), - "fallbackMirror" : BuiltinSetting( - self.MIRRORS_SCHEMA, - updateFallbackMirror, - True - ), + "fallbackMirror" : BuiltinSetting(self.MIRRORS_SCHEMA, updateFallbackMirror, True), + "fallbackMirrorAppend" : BuiltinSetting(self.MIRRORS_SCHEMA, appendFallbackMirror, True, 100), + "fallbackMirrorPrepend" : BuiltinSetting(self.MIRRORS_SCHEMA, prependFallbackMirror, True, 100), "hooks" : BuiltinSetting( schema.Schema({ schema.Optional('preBuildHook') : str, @@ -3197,11 +3229,9 @@ def updateWhiteList(x): }), lambda x: self.__buildHooks.update(x) ), - "preMirror" : BuiltinSetting( - self.MIRRORS_SCHEMA, - updatePreMirror, - True - ), + "preMirror" : BuiltinSetting(self.MIRRORS_SCHEMA, updatePreMirror, True), + "preMirrorAppend" : BuiltinSetting(self.MIRRORS_SCHEMA, appendPreMirror, True, 100), + "preMirrorPrepend" : BuiltinSetting(self.MIRRORS_SCHEMA, prependPreMirror, True, 100), "rootFilter" : BuiltinSetting( schema.Schema([str]), lambda x: self.__rootFilter.extend(x) @@ -3260,6 +3290,11 @@ def updateWhiteList(x): schema.Schema([ schema.Regex(r'^[^=]*$') ]), updateWhiteList ), + "whitelistRemove" : BuiltinSetting( + schema.Schema([ schema.Regex(r'^[^=]*$') ]), + removeWhiteList, + priority=100 + ), } def __addRecipe(self, recipe): @@ -3666,8 +3701,9 @@ def __parseUserConfig(self, fileName, relativeIncludes=None): if relativeIncludes is None: relativeIncludes = self.getPolicy("relativeIncludes") cfg = self.loadYaml(fileName, self.__userConfigSchema) - for (name, value) in cfg.items(): - if name != "include" and name != "require": self.__settings[name].merge(value) + # merge settings by priority + for (name, value) in sorted(cfg.items(), key=lambda i: self.__settings[i[0]].priority): + self.__settings[name].merge(value) for p in cfg.get("require", []): p = (os.path.join(os.path.dirname(fileName), p) if relativeIncludes else p) + ".yaml" if not os.path.isfile(p): diff --git a/test/unit/test_input_recipeset.py b/test/unit/test_input_recipeset.py index db946ed8..bdbbf882 100644 --- a/test/unit/test_input_recipeset.py +++ b/test/unit/test_input_recipeset.py @@ -229,6 +229,89 @@ def testDefaultRelativeIncludes(self): self.assertEqual(pruneBuiltin(recipeSet.defaultEnv()), {'FOO' : 'default', 'BAR' : 'lower', 'BAZ' : 'higher' }) + def testWhitelistRemove(self): + """Test whitelistRemove key""" + self.writeDefault({ + "whitelist" : [ "FOO", "BAR" ], + "whitelistRemove" : [ "BAR", "PATH" ], + }) + recipeSet = RecipeSet() + recipeSet.parse() + self.assertIn("FOO", recipeSet.envWhiteList()) + self.assertNotIn("BAR", recipeSet.envWhiteList()) + self.assertNotIn("PATH", recipeSet.envWhiteList()) + + def testArchiveAppendPrepend(self): + """Test mirrorAppend/Prepend keywords""" + self.writeDefault( + { + "archivePrepend" : [ + { + "backend" : "file", + "path" : "/foo/bar", + }, + { + "backend" : "http", + "url" : "http://bob.test/prepend", + }, + ], + "archive" : { + "backend" : "http", + "url" : "http://bob.test/main", + }, + "archiveAppend" : { + "backend" : "http", + "url" : "http://bob.test/append", + }, + }) + recipeSet = RecipeSet() + recipeSet.parse() + + self.assertEqual( + [ + { + "backend" : "file", + "path" : "/foo/bar", + }, + { + "backend" : "http", + "url" : "http://bob.test/prepend", + }, + { + "backend" : "http", + "url" : "http://bob.test/main", + }, + { + "backend" : "http", + "url" : "http://bob.test/append", + }, + ], recipeSet.archiveSpec()) + + def testMirrorsAppendPrepend(self): + """Test pre/fallbackMirrorAppend/Prepend keywords""" + for prefix in ["pre", "fallback"]: + self.writeDefault( + { + prefix+"MirrorPrepend" : [ + { "scm" : "url", "url" : "foo", "mirror" : "prepend-1" }, + { "scm" : "url", "url" : "bar", "mirror" : "prepend-2" }, + ], + prefix+"Mirror" : { "scm" : "url", "url" : "bar", "mirror" : "main" }, + prefix+"MirrorAppend" : { "scm" : "url", "url" : "bar", "mirror" : "append" }, + }) + + recipeSet = RecipeSet() + recipeSet.parse() + + self.assertEqual( + [ + { "scm" : "url", "url" : "foo", "mirror" : "prepend-1" }, + { "scm" : "url", "url" : "bar", "mirror" : "prepend-2" }, + { "scm" : "url", "url" : "bar", "mirror" : "main" }, + { "scm" : "url", "url" : "bar", "mirror" : "append" }, + ], + recipeSet.getPreMirrors() if prefix == "pre" else recipeSet.getFallbackMirrors()) + class TestProjectConfiguration(RecipesTmp, TestCase): def testInvalid(self):