Merge pull request #37 from avirshup/from_cache

Cache resolution from images with the same name

.gitignore

@@ -2,6 +2,7 @@ docker_makefiles
 Dockerfile.fail
 _docker_make_tmp
 dockerfile.fail
+MANIFEST
 
 # IntelliJ project files
 .idea

dockermake/__main__.py

@@ -25,16 +25,20 @@
 from . import errors
 
 
+RED_ERROR = termcolor.colored('FATAL ERROR:', 'red')
+
 def main():
     parser = cli.make_arg_parser()
     args = parser.parse_args()
 
-    try:
+    if args.debug:
         run(args)
-    except errors.UserException as exc:
-        red_error = termcolor.colored('FATAL ERROR:', 'red')
-        print(red_error, exc.args[0], file=sys.stderr)
-        sys.exit(exc.CODE)
+    else:
+        try:
+            run(args)
+        except (errors.UserException, errors.BuildError) as exc:
+            print(RED_ERROR, exc.args[0], file=sys.stderr)
+            sys.exit(exc.CODE)
 
 
 def _runargs(argstring):

dockermake/builds.py

@@ -43,6 +43,8 @@ def __init__(self, imagename, targetname, steps, sourcebuilds, from_image):
         self.from_image = from_image
 
     def write_dockerfile(self, output_dir):
+        """ Used only to write a Dockerfile that will NOT be built by docker-make
+        """
         if not os.path.exists(output_dir):
             os.makedirs(output_dir)
 
@@ -118,7 +120,7 @@ def build(self, client,
 
     def _get_stack_key(self, istep):
         names = [self.from_image]
-        for i in xrange(istep+1):
+        for i in range(istep+1):
             step = self.steps[i]
             if isinstance(step, FileCopyStep):
                 continue

dockermake/cli.py

@@ -47,6 +47,18 @@ def make_arg_parser():
     ca = parser.add_argument_group('Image caching')
     ca.add_argument('--pull', action='store_true',
                     help='Always try to pull updated FROM images')
+    ca.add_argument('--cache-repo',
+                    help='Repository to use for cached images. This allows you to invoke the '
+                    '`docker build --build-from` option for each image.'
+                    'For instance, running '
+                    '`docker-make foo bar --cache-repo docker.io/cache` will use '
+                    'docker.io/cache/foo as a cache for `foo` and docker.io/cache/bar as a cache'
+                    'for `bar`.',
+                    default='')
+    ca.add_argument('--cache-tag',
+                    help='Tag to use for cached images; '
+                         'can be used with the --cache-repo option (see above).',
+                    default='')
     ca.add_argument('--no-cache', action='store_true',
                     help="Rebuild every layer")
     ca.add_argument('--bust-cache', action='append',
@@ -60,8 +72,10 @@ def make_arg_parser():
                     help="Prepend this repository to all built images, e.g.\n"
                          "`docker-make hello-world -u quay.io/elvis` will tag the image "
                          "as `quay.io/elvis/hello-world`. You can add a ':' to the end to "
-                         "image names into tags:\n `docker-make -u quay.io/elvis/repo: hello-world` "
-                         "will create the image in the elvis repository: quay.io/elvis/repo:hello-world")
+                         "image names into tags:\n "
+                         "`docker-make -u quay.io/elvis/repo: hello-world` "
+                         "will create the "
+                         "image in the elvis repository: quay.io/elvis/repo:hello-world")
     rt.add_argument('--tag', '-t', type=str,
                     help='Tag all built images with this tag. If image names are ALREADY tags (i.e.,'
                          ' your repo name ends in a ":"), this will append the tag name with a dash. '
@@ -82,6 +96,7 @@ def make_arg_parser():
                     help="Print version and exit.")
     hh.add_argument('--help-yaml', action='store_true',
                     help="Print summary of YAML file format and exit.")
+    hh.add_argument('--debug', action='store_true')
 
     return parser
 

dockermake/errors.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 # Copyright 2017 Autodesk Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,7 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from io import StringIO
+import pprint
+from termcolor import cprint
 
 class UserException(Exception):
     """
@@ -68,3 +72,20 @@ class ParsingFailure(UserException):
 
 class MultipleIgnoreError(UserException):
     CODE = 51
+
+
+class BuildError(Exception):
+    CODE = 200
+
+    def __init__(self, dockerfile, item, build_args):
+        with open('dockerfile.fail', 'w') as dff:
+            print(dockerfile, file=dff)
+        with StringIO() as stream:
+            cprint('Docker build failure', 'red', attrs=['bold'], file=stream)
+            print(u'\n   -------- Docker daemon output --------', file=stream)
+            pprint.pprint(item, stream, indent=4)
+            print(u'   -------- Arguments to client.build --------', file=stream)
+            pprint.pprint(build_args, stream, indent=4)
+            print(u'This dockerfile was written to dockerfile.fail', file=stream)
+            stream.seek(0)
+            super(BuildError, self).__init__(stream.read())

dockermake/imagedefs.py

@@ -26,6 +26,7 @@
 from . import builds
 from . import staging
 from . import errors
+from . import utils
 
 RECOGNIZED_KEYS = set(('requires build_directory build copy_from FROM description _sourcefile'
                        ' FROM_DOCKERFILE ignore ignorefile')
@@ -43,13 +44,12 @@ def __init__(self, makefile_path):
         print('Copy cache directory: %s' % staging.TMPDIR)
         try:
             self.ymldefs = self.parse_yaml(self.makefile_path)
+        except errors.UserException:
+            raise
         except Exception as exc:
-            if isinstance(exc, errors.UserException):
-                raise
-            else:
-                raise errors.ParsingFailure('Failed to read file %s:\n' % self.makefile_path +
-                                            str(exc))
-        self.all_targets = self.ymldefs.pop('_ALL_', None)
+            raise errors.ParsingFailure('Failed to read file %s:\n' % self.makefile_path +
+                                        str(exc))
+        self.all_targets = self.ymldefs.pop('_ALL_', [])
         self._external_dockerfiles = {}
 
     def parse_yaml(self, filename):
@@ -93,6 +93,20 @@ def _check_yaml_and_paths(ymlfilepath, yamldefs):
                 if key in defn:
                     defn[key] = _get_abspath(pathroot, defn[key])
 
+            if 'copy_from' in defn:
+                if not isinstance(defn['copy_from'], dict):
+                    raise errors.ParsingFailure((
+                            'Syntax error in file "%s": \n' +
+                            'The "copy_from" field in image definition "%s" is not \n' 
+                            'a key:value list.') % (ymlfilepath, imagename))
+                for otherimg, value in defn.get('copy_from', {}).items():
+                    if not isinstance(value, dict):
+                        raise errors.ParsingFailure((
+                            'Syntax error in field:\n'
+                            '     %s . copy_from . %s\nin file "%s". \n'
+                            'All entries must be of the form "sourcepath: destpath"')%
+                                 (imagename, otherimg, ymlfilepath))
+
             # save the file path for logging
             defn['_sourcefile'] = relpath
 
@@ -107,7 +121,7 @@ def _check_yaml_and_paths(ymlfilepath, yamldefs):
                             'Field "%s" in image "%s" in file "%s" not recognized' %
                             (key, imagename, relpath))
 
-    def generate_build(self, image, targetname, rebuilds=None):
+    def generate_build(self, image, targetname, rebuilds=None, cache_repo='', cache_tag=''):
         """
         Separate the build into a series of one or more intermediate steps.
         Each specified build directory gets its own step
@@ -116,8 +130,14 @@ def generate_build(self, image, targetname, rebuilds=None):
             image (str): name of the image as defined in the dockermake.py file
             targetname (str): name to tag the final built image with
             rebuilds (List[str]): list of image layers to rebuild (i.e., without docker's cache)
+            cache_repo (str): repository to get images for caches in builds
+            cache_tag (str): tags to use from repository for caches in builds
         """
         from_image = self.get_external_base_image(image)
+        if cache_repo or cache_tag:
+            cache_from = utils.generate_name(image, cache_repo, cache_tag)
+        else:
+            cache_from = None
         if from_image is None:
             raise errors.NoBaseError("No base image found in %s's dependencies" % image)
         if isinstance(from_image, ExternalDockerfile):
@@ -137,12 +157,12 @@ def generate_build(self, image, targetname, rebuilds=None):
         for base_name in self.sort_dependencies(image):
             istep += 1
             buildname = 'dmkbuild_%s_%d' % (image, istep)
-            build_steps.append(dockermake.step.BuildStep(base_name,
-                                                         base_image,
-                                                         self.ymldefs[base_name],
-                                                         buildname,
-                                                         bust_cache=base_name in rebuilds,
-                                                         build_first=build_first))
+            build_steps.append(
+                    dockermake.step.BuildStep(
+                            base_name, base_image, self.ymldefs[base_name],
+                            buildname, bust_cache=base_name in rebuilds,
+                            build_first=build_first, cache_from=cache_from))
+
             base_image = buildname
             build_first = None
 
@@ -151,14 +171,16 @@ def generate_build(self, image, targetname, rebuilds=None):
                 for sourcepath, destpath in iteritems(files):
                     istep += 1
                     buildname = 'dmkbuild_%s_%d' % (image, istep)
-                    build_steps.append(dockermake.step.FileCopyStep(sourceimage, sourcepath,
-                                                                    base_image, destpath,
-                                                                    buildname,
-                                                                    self.ymldefs[base_name],
-                                                                    base_name))
+                    build_steps.append(
+                            dockermake.step.FileCopyStep(
+                                    sourceimage, sourcepath, destpath,
+                                    base_name, base_image, self.ymldefs[base_name],
+                                    buildname, bust_cache=base_name in rebuilds,
+                                    build_first=build_first, cache_from=cache_from))
                     base_image = buildname
 
-        sourcebuilds = [self.generate_build(img, img) for img in sourceimages]
+        sourcebuilds = [self.generate_build(img, img, cache_repo=cache_repo, cache_tag=cache_tag)
+                        for img in sourceimages]
 
         return builds.BuildTarget(imagename=image,
                                   targetname=targetname,

dockermake/staging.py

@@ -47,13 +47,15 @@ class StagedFile(object):
         sourceimage (str): name of the image to copy from
         sourcepath (str): path in the source image
         destpath (str): path in the target image
+        cache_from (str or list): use this(these) image(s) to resolve build cache
     """
-    def __init__(self, sourceimage, sourcepath, destpath):
+    def __init__(self, sourceimage, sourcepath, destpath, cache_from=None):
         self.sourceimage = sourceimage
         self.sourcepath = sourcepath
         self.destpath = destpath
         self._sourceobj = None
         self._cachedir = None
+        self.cache_from = cache_from
 
     def stage(self, startimage, newimage):
         """ Copies the file from source to target
@@ -62,8 +64,6 @@ def stage(self, startimage, newimage):
             startimage (str): name of the image to stage these files into
             newimage (str): name of the created image
         """
-        from .step import BuildError
-
         client = utils.get_client()
         cprint('  Copying file from "%s:/%s" \n                 to "%s://%s/"'
                % (self.sourceimage, self.sourcepath, startimage, self.destpath),
@@ -106,12 +106,15 @@ def stage(self, startimage, newimage):
                          tag=newimage,
                          decode=True)
 
+        if self.cache_from:
+            buildargs['cache_from'] = self.cache_from
+
         # Build and show logs
         stream = client.api.build(**buildargs)
         try:
             utils.stream_docker_logs(stream, newimage)
         except ValueError as e:
-            raise BuildError(dockerfile, e.args[0], build_args=buildargs)
+            raise errors.BuildError(dockerfile, e.args[0], build_args=buildargs)
 
     def _setcache(self, client):
         if self._sourceobj is None:  # get image and set up cache if necessary