Build cache

cget/cli.py

@@ -87,8 +87,10 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def
 @click.option('--debug', is_flag=True, help="Install debug version")
 @click.option('--release', is_flag=True, help="Install release version")
 @click.option('--insecure', is_flag=True, help="Don't use https urls")
+@click.option('--use-build-cache', is_flag=True, help="Cache builds")
+@click.option('--recipe-deps-only', is_flag=True, help="only use dependencies from recipes (speeds up cached builds a lot)")
 @click.argument('pkgs', nargs=-1, type=click.STRING)
-def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure):
+def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache, recipe_deps_only):
     """ Install packages """
     if debug and release:
         click.echo("ERROR: debug and release are not supported together")
@@ -102,7 +104,16 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato
     for pbu in util.flat([prefix.from_file(file), pbs]):
         pb = pbu.merge_defines(define)
         with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)):
-            click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure))
+            click.echo(prefix.install(
+                pb,
+                test=test,
+                test_all=test_all,
+                update=update,
+                generator=generator,
+                insecure=insecure,
+                use_build_cache=use_build_cache,
+                recipe_deps_only=recipe_deps_only
+            ))
 
 @cli.command(name='ignore')
 @use_prefix

cget/package.py

@@ -1,4 +1,4 @@
-import base64, copy, argparse, six
+import base64, copy, argparse, six, dirhash, hashlib
 
 def encode_url(url):
     x = six.b(url[url.find('://')+3:])
@@ -31,6 +31,14 @@ def get_src_dir(self):
             return self.url[7:] # Remove "file://"
         raise TypeError()
 
+    def calc_hash(self):
+        if self.recipe:
+            print("calculating dirshash of recipe '%s' package '%s'" % (self.recipe, self.to_name()))
+            return dirhash.dirhash(self.recipe, "sha1")
+        elif self.url:
+            print("calculating hash of url '%s' package '%s'" % (self.url, self.to_name()))
+            return hashlib.sha1(self.url.encode("utf-8")).hexdigest()
+        raise Exception("no url or recipe: %s" % self.__dict__)
 
 def fname_to_pkg(fname):
     if fname.startswith('_url_'): return PackageSource(name=decode_url(fname), fname=fname)

cget/prefix.py

@@ -1,4 +1,4 @@
-import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools
+import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools, hashlib
 
 from cget.builder import Builder
 from cget.package import fname_to_pkg
@@ -205,7 +205,7 @@ def get_unlink_deps_directory(self, name, *dirs):
     def parse_src_file(self, name, url, start=None):
         f = util.actual_path(url, start)
         self.log('parse_src_file actual_path:', start, f)
-        if os.path.exists(f): return PackageSource(name=name, url='file://' + f)
+        if os.path.isfile(f): return PackageSource(name=name, url='file://' + f)
         return None
 
     def parse_src_recipe(self, name, url):
@@ -223,6 +223,15 @@ def parse_src_github(self, name, url):
         if name is None: name = p
         return PackageSource(name=name, url=url)
 
+    def hash_pkg(self, pkg):
+        pkg_src = self.parse_pkg_src(pkg)
+        result = pkg_src.calc_hash()
+        pkg_build = self.parse_pkg_build(pkg)
+        if pkg_build.requirements:
+            for dependency in self.from_file(pkg_build.requirements):
+                result = hashlib.sha1((result + self.hash_pkg(dependency)).encode("utf-8")).hexdigest()
+        return result
+
     @returns(PackageSource)
     @params(pkg=PACKAGE_SOURCE_TYPES)
     def parse_pkg_src(self, pkg, start=None, no_recipe=False):
@@ -239,14 +248,14 @@ def parse_pkg_src(self, pkg, start=None, no_recipe=False):
     @returns(PackageBuild)
     @params(pkg=PACKAGE_SOURCE_TYPES)
     def parse_pkg_build(self, pkg, start=None, no_recipe=False):
-        if isinstance(pkg, PackageBuild): 
+        if isinstance(pkg, PackageBuild):
             pkg.pkg_src = self.parse_pkg_src(pkg.pkg_src, start, no_recipe)
             if pkg.pkg_src.recipe: pkg = self.from_recipe(pkg.pkg_src.recipe, pkg)
             if pkg.cmake: pkg.cmake = find_cmake(pkg.cmake, start)
             return pkg
         else:
             pkg_src = self.parse_pkg_src(pkg, start, no_recipe)
-            if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, pkg_src.name)
+            if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, name=pkg_src.name)
             else: return PackageBuild(pkg_src)
 
     def from_recipe(self, recipe, pkg=None, name=None):
@@ -256,7 +265,7 @@ def from_recipe(self, recipe, pkg=None, name=None):
         self.check(lambda:p.pkg_src is not None)
         requirements = os.path.join(recipe, "requirements.txt")
         if os.path.exists(requirements): p.requirements = requirements
-        p.pkg_src.recipe = None
+        p.pkg_src.recipe = recipe
         # Use original name
         if pkg: p.pkg_src.name = pkg.pkg_src.name
         elif name: p.pkg_src.name = name
@@ -285,17 +294,52 @@ def from_file(self, file, url=None, no_recipe=False):
     def write_parent(self, pb, track=True):
         if track and pb.parent is not None: util.mkfile(self.get_deps_directory(pb.to_fname()), pb.parent, pb.parent)
 
-    def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False):
-        for dependent in self.from_file(pb.requirements or os.path.join(d, 'requirements.txt'), pb.pkg_src.url):
+    def install_deps(
+        self,
+        pb,
+        src_dir=None, 
+        test=False,
+        test_all=False,
+        generator=None,
+        insecure=False,
+        use_build_cache=False,
+        recipe_deps_only=False
+    ):
+        if pb.requirements:
+            dependents = self.from_file(pb.requirements, pb.pkg_src.url)
+        elif src_dir:
+            dependents = self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url)
+        else:
+            return
+        for dependent in dependents:
             transient = dependent.test or dependent.build
             testing = test or test_all
             installable = not dependent.test or dependent.test == testing
             if installable: 
-                self.install(dependent.of(pb), test_all=test_all, generator=generator, track=not transient, insecure=insecure)
+                self.install(
+                    dependent.of(pb),
+                    test_all=test_all,
+                    generator=generator,
+                    track=not transient,
+                    insecure=insecure,
+                    use_build_cache=use_build_cache,
+                    recipe_deps_only=recipe_deps_only
+                )
 
     @returns(six.string_types)
     @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool)
-    def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False):
+    def install(
+        self,
+        pb,
+        test=False,
+        test_all=False,
+        generator=None,
+        update=False,
+        track=True,
+        insecure=False,
+        use_build_cache=False,
+        recipe_deps_only=False
+    ):
         pb = self.parse_pkg_build(pb)
         pkg_dir = self.get_package_directory(pb.to_fname())
         unlink_dir = self.get_unlink_directory(pb.to_fname())
@@ -311,26 +355,59 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False,
             self.write_parent(pb, track=track)
             if update: self.remove(pb)
             else: return "Package {} already installed".format(pb.to_name())
-        with self.create_builder(uuid.uuid4().hex, tmp=True) as builder:
-            # Fetch package
-            src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure)
-            # Install any dependencies first
-            self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure)
-            # Setup cmake file
-            if pb.cmake: 
-                target = os.path.join(src_dir, 'CMakeLists.txt')
-                if os.path.exists(target):
-                    os.rename(target, os.path.join(src_dir, builder.cmake_original_file))
-                shutil.copyfile(pb.cmake, target)
-            # Configure and build
-            builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant)
-            builder.build(variant=pb.variant)
-            # Run tests if enabled
-            if test or test_all: builder.test(variant=pb.variant)
-            # Install
-            builder.build(target='install', variant=pb.variant)
-            if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix)
-            else: util.copy_dir(install_dir, self.prefix)
+        package_hash = self.hash_pkg(pb)
+        print("package %s hash %s" % (pb.to_name(), package_hash))
+        build_cache_prefix = "builds/%s" % pb.to_name()
+        need_build = True
+        if recipe_deps_only:
+            self.install_deps(
+                pb,
+                test=test,
+                test_all=test_all,
+                generator=generator,
+                insecure=insecure,
+                use_build_cache=use_build_cache,
+                recipe_deps_only=True
+            )
+            if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir):
+                print("retreived Package {} from cache".format(pb.to_name()))
+                need_build = False
+        if need_build:
+            with self.create_builder(uuid.uuid4().hex, tmp=True) as builder:
+                # Fetch package
+                src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure)
+                # Install any dependencies first
+                if not recipe_deps_only:
+                    self.install_deps(
+                        pb,
+                        src_dir=src_dir,
+                        test=test,
+                        test_all=test_all,
+                        generator=generator,
+                        insecure=insecure,
+                        use_build_cache=use_build_cache,
+                        recipe_deps_only=False
+                    )
+                if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir):
+                    print("retreived Package {} from cache".format(pb.to_name()))
+                else:
+                    # Setup cmake file
+                    if pb.cmake:
+                        target = os.path.join(src_dir, 'CMakeLists.txt')
+                        if os.path.exists(target):
+                            os.rename(target, os.path.join(src_dir, builder.cmake_original_file))
+                        shutil.copyfile(pb.cmake, target)
+                    # Configure and build
+                    builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant)
+                    builder.build(variant=pb.variant)
+                    # Run tests if enabled
+                    if test or test_all: builder.test(variant=pb.variant)
+                    # Install
+                    builder.build(target='install', variant=pb.variant)
+                    if use_build_cache:
+                        util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir)
+        if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix)
+        else: util.copy_dir(install_dir, self.prefix)
         self.write_parent(pb, track=track)
         return "Successfully installed {}".format(pb.to_name())
 

cget/util.py

@@ -1,4 +1,4 @@
-import click, os, sys, shutil, json, six, hashlib, ssl
+import click, os, sys, shutil, json, six, hashlib, ssl, filelock
 
 if sys.version_info[0] < 3:
     try:
@@ -87,6 +87,43 @@ def mkfile(d, file, content, always_write=True):
         write_to(p, content)
     return p
 
+def cache_lock():
+    cache_base_dir = get_cache_path()
+    mkdir(cache_base_dir)
+    return filelock.FileLock(os.path.join(cache_base_dir, "lock"))
+
+def zipdir(src_dir, tgt_file):
+    print("zipping '%s' to '%s" % (src_dir, tgt_file))
+    zipf = zipfile.ZipFile(tgt_file, 'w', zipfile.ZIP_DEFLATED)
+    for root, dirs, files in os.walk(src_dir):
+        for file in files:
+            zipf.write(
+                os.path.join(root, file),
+                os.path.relpath(
+                    os.path.join(root, file),
+                    os.path.join(src_dir)
+                )
+            )
+    zipf.close()
+
+def zip_dir_to_cache(prefix, key, src_dir):
+    with cache_lock():
+        cache_dir = get_cache_path(prefix)
+        zipfile_path = os.path.join(cache_dir, key + ".zip")
+        mkdir(cache_dir)
+        zipdir(src_dir, zipfile_path)
+
+def unzip_dir_from_cache(prefix, key, tgt_dir):
+    with cache_lock():
+        cache_dir = get_cache_path(prefix)
+        zipfile_path = os.path.join(cache_dir, key + ".zip")
+        if os.path.exists(zipfile_path):
+            f = zipfile.ZipFile(zipfile_path, "r")
+            f.extractall(tgt_dir)
+            return True
+        else:
+            return False
+
 def ls(p, predicate=lambda x:True):
     if os.path.exists(p):
         return (d for d in os.listdir(p) if predicate(os.path.join(p, d)))
@@ -106,15 +143,17 @@ def adjust_path(p):
     return p
 
 def add_cache_file(key, f):
-    mkdir(get_cache_path(key))
-    shutil.copy2(f, get_cache_path(key, os.path.basename(f)))
+    with cache_lock():
+        mkdir(get_cache_path(key))
+        shutil.copy2(f, get_cache_path(key, os.path.basename(f)))
 
 def get_cache_file(key):
-    p = get_cache_path(key)
-    if os.path.exists(p):
-        return os.path.join(p, next(ls(p)))
-    else:
-        return None
+    with cache_lock():
+        p = get_cache_path(key)
+        if os.path.exists(p):
+            return os.path.join(p, next(ls(p)))
+        else:
+            return None
 
 def delete_dir(path):
     if path is not None and os.path.exists(path): shutil.rmtree(adjust_path(path))

requirements.txt

@@ -1,3 +1,5 @@
 click>=6.6
 # PyYAML
 six>=1.10
+dirhash>=0.2.1
+filelock>=2.0.13