Merge pull request #561 from rhubert/rh-layers

bob managed layers

contrib/bash-completion/bob

@@ -30,7 +30,7 @@ __bob_complete_dir()
 }
 
 __bob_commands="build dev clean graph help init jenkins ls project status \
-                query-scm query-recipe query-path query-meta show"
+                query-scm query-recipe query-path query-meta show layers"
 
 # Complete a Bob path
 #
@@ -79,10 +79,13 @@ __bob_complete_path()
    # Auto complete config files. They can be in directories and their '.yaml'
    # suffix is implicitly added by Bob. The file name might directly start
    # after '-c' making it a bit more complicated.
-   if [[ $prev = "-c" || $cur = -c?* ]] ; then
+   if [[ $prev = "-c" || $cur = -c?*  || $prev = "-lc" || $cur = -lc* ]] ; then
 	  if [[ $cur = -c?* ]] ; then
 	     prefix="-c"
 		 cur="${cur:2}"
+	  elif [[ $cur = -lc?* ]] ; then
+	     prefix="-lc"
+		 cur="${cur:3}"
 	  else
 		 prefix=
 	  fi
@@ -386,6 +389,21 @@ __bob_status()
   __bob_complete_path "--attic --develop --recursive --no-sandbox --release --sandbox --show-clean --show-overrides --verbose -D -c -r -v"
 }
 
+__bob_layers_status()
+{
+   __bob_complete_path "-c -D -v -lc --attic --no-attic"
+}
+
+__bob_layers_update()
+{
+   __bob_complete_path "-c -D -v -lc --attic --no-attic"
+}
+
+__bob_layers()
+{
+  __bob_subcommands "status update" "layers"
+}
+
 __bob_show()
 {
     if [[ "$prev" = "--format" ]] ; then

doc/conf.py

@@ -267,6 +267,7 @@ def __getattr__(cls, name):
     ('manpages/bob-graph', 'bob-graph', 'Generate dependency graph', ['Ralf Hubert'], 1),
     ('manpages/bob-init', 'bob-init', 'Initialize out-of-source build tree', ['Jan Klötzke'], 1),
     ('manpages/bob-jenkins', 'bob-jenkins', 'Configure Jenkins server', ['Jan Klötzke'], 1),
+    ('manpages/bob-layers', 'bob-layers', 'Manage Layers', ['BobBuildTool Team'], 1),
     ('manpages/bob-ls', 'bob-ls', 'List package hierarchy', ['Jan Klötzke'], 1),
     ('manpages/bobpaths', 'bobpaths', 'Specifying paths to Bob packages', ['Jan Klötzke'], 7),
     ('manpages/bob-project', 'bob-project', 'Create IDE project files', ['Jan Klötzke'], 1),

doc/manpages/bob-layers.rst

@@ -0,0 +1,76 @@
+.. _manpage-layers:
+
+bob-layers
+==========
+
+.. only:: not man
+
+   Name
+   ----
+
+   bob-layers - Handle layers
+
+Synopsis
+--------
+
+::
+
+    bob layers [-h] [-c CONFIGFILE] [-lc LAYERCONFIG] [-v] [-D DEFINES]
+               [--attic | --no-attic] 
+               {update,status}
+
+Description
+-----------
+
+Update layers or show their SCM-status. The following sub-commands are
+available:
+
+``update``
+    Updates the layers.
+
+``status``
+    Show the SCM-status of each layer and optionally list modifications. See
+    :ref:`bob status <manpage-bob-status>` for a description of the output
+    fields.
+
+Options
+-------
+
+``--attic``
+    Move layer workspace to attic if inline SCM switching is not possible.
+    (Default)
+
+``--no-attic``
+    Do not move layer workspace to attic if inline SCM switching is not possible.
+    Instead a build error is issued.
+
+``-c CONFIGFILE``
+    Use additional configuration file.
+
+    The ``.yaml`` suffix is appended automatically and the configuration file
+    is searched relative to the project root directory unless an absolute path
+    is given. Bob will parse these user configuration files after
+    *default.yaml*. They are using the same schema.
+
+    This option can be given multiple times. The files will be parsed in the
+    order as they appeared on the command line.
+
+``-lc LAYERCONFIG``
+    Use additional layer configuration file.
+
+    This is special kind of configuration file to control the layers checkout. Only
+    ``layersWhitelist`` and ``layersScmOverrides`` are supported.
+
+    The ``.yaml`` suffix is appended automatically and the configuration file
+    is searched relative to the project root directory unless an absolute path
+    is given.
+
+``-D VAR=VALUE``
+    Override default or set environment variable.
+
+    Sets the variable ``VAR`` to ``VALUE``. This overrides the value possibly
+    set by ``default.yaml``, config files passed by ``-c`` or any file that was
+    included by either of these files.
+
+``-v, --verbose``
+    Increase verbosity (may be specified multiple times)

doc/manpages/index.rst

@@ -14,6 +14,7 @@ Contents:
    bob-graph
    bob-init
    bob-jenkins
+   bob-layers
    bob-ls
    bobpaths
    bob-project

doc/manual/configuration.rst

@@ -1927,7 +1927,7 @@ equal.
 layers
 ~~~~~~
 
-Type: List of strings
+Type: List of strings or SCM-Dictionary or List of SCM-Dictionaries
 
 The ``layers`` section consists of a list of layer names that are then expected
 in the ``layers`` directory relative to the ``conig.yaml`` referencing them::
@@ -1949,8 +1949,40 @@ of lower precedence.
 
 See :ref:`configuration` for more information.
 
+Typically layers have their own VCS. To provide them to the root-recipes common
+VCS-methods like git-submodules can be used. Another possibility is to provide a
+SCM-Dictionary (see :ref:`configuration-recipes-scm`)::
+
+    layers:
+        - name: myapp
+          scm: git
+          url: [email protected]:myapp.git
+          commit: ...
+        - bsp
+
+Only `git`,`svn`,`url` and `cvs` scm's are supported for layers. During layers
+checkout the regular ``whitelist`` and ``scmOverrides`` settings are not used.
+Instead the checkout could be controlled by ``layersWhitelist`` and
+``layersScmOverrides``.
+
+If a scmSpec is given Bob takes care of the layer management:
+
+- layers are checked out / updated during bob-build (except build-only)
+- bob layers command to update / show status (see :ref:`manpage-layers`).
+
 .. _configuration-config-plugins:
 
+layersWhitelist
+~~~~~~~~~~~~~~~
+
+Whitelist for layers update only. See :ref:`configuration-config-whitelist`.
+
+layersScmOverrides
+~~~~~~~~~~~~~~~~~~
+
+:ref:`configuration-config-scmOverrides` used by layers checkout / update.
+Conditional overrides are not supported.
+
 plugins
 ~~~~~~~
 

pym/bob/builder.py

@@ -1474,7 +1474,7 @@ async def _downloadPackage(self, packageStep, depth, packageBuildId):
           * still same build-id -> done
           * build-id changed -> prune and try download, fall back to build
         """
-        layer = "/".join(packageStep.getPackage().getRecipe().getLayer())
+        layer = packageStep.getPackage().getRecipe().getLayer()
         layerDownloadMode = None
         if layer:
             for mode in self.__downloadLayerModes:

pym/bob/cmds/build/build.py

@@ -9,6 +9,7 @@
 from ...input import RecipeSet
 from ...intermediate import StepIR, PackageIR, RecipeIR, ToolIR, SandboxIR, \
     RecipeSetIR
+from ...layers import updateLayers
 from ...share import getShare
 from ...tty import setVerbosity, setTui, Warn
 from ...utils import copyTree, processDefines, EventLoopWrapper
@@ -171,6 +172,8 @@ def _downloadLayerArgument(arg):
         help="Override default environment variable")
     parser.add_argument('-c', dest="configFile", default=[], action='append',
         help="Use config File")
+    parser.add_argument('-lc', dest="layerConfig", default=[], action='append',
+        help="Additional layer config")
     parser.add_argument('-e', dest="white_list", default=[], action='append', metavar="NAME",
         help="Preserve environment variable")
     parser.add_argument('-E', dest="preserve_env", default=False, action='store_true',
@@ -224,6 +227,9 @@ def _downloadLayerArgument(arg):
         recipes.defineHook('developNameFormatter', LocalBuilder.developNameFormatter)
         recipes.defineHook('developNamePersister', None)
         recipes.setConfigFiles(args.configFile)
+        if args.build_mode != 'build-only':
+            setVerbosity(args.verbose)
+            updateLayers(recipes, loop, defines, args.verbose, args.attic, args.layerConfig)
         recipes.parse(defines)
 
         # if arguments are not passed on cmdline use them from default.yaml or set to default yalue

pym/bob/cmds/layers.py

@@ -0,0 +1,48 @@
+import argparse
+
+from ..input import RecipeSet
+from ..layers import Layers, updateLayers
+from ..tty import NORMAL, setVerbosity
+from ..utils import EventLoopWrapper, processDefines
+from .build.status import PackagePrinter
+
+def doLayers(argv, bobRoot):
+    parser = argparse.ArgumentParser(prog="bob layers", description='Handle layers')
+    parser.add_argument('action', type=str, choices=['update', 'status'], default="status",
+                        help="Action: [update, status]")
+    parser.add_argument('-c', dest="configFile", default=[], action='append',
+        help="Use config File.")
+    parser.add_argument('-lc', dest="layerConfig", default=[], action='append',
+        help="Additional layer config")
+    parser.add_argument('-v', '--verbose', default=NORMAL, action='count',
+        help="Increase verbosity (may be specified multiple times)")
+    parser.add_argument('-D', default=[], action='append', dest="defines",
+        help="Override default environment variable")
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--attic', action='store_true', default=True,
+        help="Move scm to attic if inline switch is not possible (default).")
+    group.add_argument('--no-attic', action='store_false', default=None, dest='attic',
+        help="Do not move to attic, instead fail the build.")
+
+    args = parser.parse_args(argv)
+
+    setVerbosity(args.verbose)
+
+    defines = processDefines(args.defines)
+
+    with EventLoopWrapper() as (loop, executor):
+        recipes = RecipeSet()
+        recipes.setConfigFiles(args.configFile)
+        if args.action == "update":
+            updateLayers(recipes, loop, defines, args.verbose,
+                         args.attic, args.layerConfig)
+
+        recipes.parse(defines)
+
+        layers = Layers(recipes, loop, defines, args.attic)
+        layers.setLayerConfig(args.layerConfig)
+        layers.collect(False, args.verbose)
+        if args.action == "status":
+            pp = PackagePrinter(args.verbose, False, False)
+            layers.status(pp.show)
+