Remove generated files before build and detect modified files

lbuild/api.py

@@ -21,6 +21,8 @@
 from lbuild.config import ConfigNode
 from lbuild.utils import listify, listrify
 from lbuild.logger import CallCounter
+from lbuild.exception import LbuildApiBuildlogNotFoundException, LbuildApiModifiedFilesException
+from pathlib import Path
 
 
 class Builder:
@@ -121,7 +123,7 @@ def validate(self, modules=None, complete=True):
         self.parser.validate_modules(build_modules, complete)
         return (build_modules, CallCounter.levels)
 
-    def build(self, outpath, modules=None, simulate=False, use_symlinks=False):
+    def build(self, outpath, modules=None, simulate=False, use_symlinks=False, write_buildlog=True):
         """
         Build the given set of modules.
 
@@ -133,8 +135,45 @@ def build(self, outpath, modules=None, simulate=False, use_symlinks=False):
             simulate -- If set to True simulate the build process. In
                 that case no output will be generated.
         """
+        buildlogname = self.config.filename + ".log"
+        try:
+            self.clean(buildlogname)
+        except LbuildApiBuildlogNotFoundException:
+            pass
+
         build_modules = self._filter_modules(modules)
         buildlog = BuildLog(outpath)
         lbuild.environment.SYMLINK_ON_COPY = use_symlinks
         self.parser.build_modules(build_modules, buildlog, simulate=simulate)
+        if write_buildlog and not simulate:
+            Path(buildlogname).write_bytes(buildlog.to_xml(to_string=True, path=self.cwd))
         return buildlog
+
+    def clean(self, buildlog, force=False):
+        buildlogfile = Path(buildlog)
+        if not buildlogfile.exists():
+            raise LbuildApiBuildlogNotFoundException(buildlogfile)
+        buildlog = BuildLog.from_xml(buildlogfile.read_bytes(), path=self.cwd)
+
+        unmodified, modified, missing = buildlog.compare_outpath()
+        if not force and len(modified):
+            raise LbuildApiModifiedFilesException(buildlogfile, modified)
+
+        removed = []
+        dirs = set()
+        for filename in sorted(unmodified + modified + [str(buildlogfile)]):
+            dirs.add(os.path.dirname(filename))
+            try:
+                os.remove(filename)
+                removed.append(filename)
+            except OSError:
+                pass
+
+        dirs = sorted(list(dirs), key=lambda d: d.count("/"), reverse=True)
+        for directory in dirs:
+            try:
+                os.removedirs(directory)
+            except OSError:
+                pass
+
+        return removed

lbuild/buildlog.py

@@ -14,6 +14,7 @@
 import collections
 import threading
 from os.path import join, relpath, dirname, normpath, basename, isabs, abspath
+from pathlib import Path
 
 import lxml.etree
 import lbuild.utils
@@ -22,7 +23,6 @@
 
 LOGGER = logging.getLogger('lbuild.buildlog')
 
-
 class Operation:
     """
     Representation of a build operation.
@@ -46,6 +46,22 @@ def __init__(self, module_name, outpath, module_path,
 
         self.filename_in = abspath(filename_in)
         self.filename_out = abspath(filename_out)
+        self.filename_module = None
+
+        self.hash_in = None
+        self.hash_out = None
+        self.hash_module = None
+
+        self.mtime_in = None
+        self.mtime_out = None
+        self.mtime_module = None
+
+    def compute_hash(self, module_filename=None):
+        self.mtime_in, self.hash_in = lbuild.utils.hash_file(self.filename_in)
+        self.mtime_out, self.hash_out = lbuild.utils.hash_file(self.filename_out)
+        self.mtime_module, self.hash_module = lbuild.utils.hash_file(module_filename)
+        if self.hash_module is not None:
+            self.filename_module = relpath(module_filename)
 
     def local_filename_in(self, relative_to=None):
         path = self.inpath
@@ -161,6 +177,7 @@ def log(self, module, filename_in: str, filename_out: str, time=None, metadata=N
         with self.__lock:
             operation = Operation(module.fullname, self.outpath, module._filepath,
                                   filename_in, filename_out, time, metadata)
+            operation.compute_hash(module._filename)
             LOGGER.debug(str(operation))
 
             previous = self._build_files.get(filename_out, None)
@@ -173,7 +190,8 @@ def log(self, module, filename_in: str, filename_out: str, time=None, metadata=N
 
         return operation
 
-    def log_unsafe(self, modulename, filename_in, filename_out, time=None, metadata=None):
+    def log_unsafe(self, modulename, filename_in, filename_out,
+                   time=None, metadata=None, compute_hash=True):
         """
         Log an lbuild internal operation.
 
@@ -184,6 +202,7 @@ def log_unsafe(self, modulename, filename_in, filename_out, time=None, metadata=
         """
         operation = Operation(modulename, self.outpath, self.outpath,
                               filename_in, filename_out, time, metadata)
+        if compute_hash: operation.compute_hash();
         with self.__lock:
             self._operations[modulename].append(operation)
 
@@ -218,18 +237,57 @@ def operations(self):
         operations = (o for olists in operations for o in olists)
         return sorted(operations, key=lambda o: (o.module_name, o.filename_in, o.filename_out))
 
+    def compare_outpath(self):
+        unmodified = []
+        modified = []
+        missing = []
+        for op in self.operations:
+            destname = op.local_filename_out()
+            if os.path.exists(destname):
+                mtime_out = int(os.path.getmtime(destname))
+                if (op.mtime_out != mtime_out and
+                    op.hash_out != lbuild.utils.hash_file(destname)[1]):
+                        modified.append(destname)
+                else:
+                    unmodified.append(destname)
+            else:
+                missing.append(destname)
+
+        return (unmodified, modified, missing)
+
     @staticmethod
     def from_xml(string, path):
         rootnode = lxml.etree.fromstring(string)
         outpath = join(abspath(path), rootnode.find("outpath").text)
         buildlog = BuildLog(outpath)
 
         for opnode in rootnode.iterfind("operation"):
-            module_name = opnode.find("module").text
-            source = join(outpath, opnode.find("source").text)
-            destination = join(outpath, opnode.find("destination").text)
-            operation = Operation(module_name, outpath, path, source, destination)
-            buildlog._operations[module_name].append(operation)
+            module = opnode.find("module")
+            module_hash = module.get("hash", None)
+            module_mtime = module.get("mtime", None)
+            module = module.get("name")
+
+            source = opnode.find("source")
+            source_hash = source.get("hash", None)
+            source_mtime = source.get("modified", None)
+            source = join(outpath, source.text)
+
+            destination = opnode.find("destination")
+            destination_hash = destination.get("hash", None)
+            destination_mtime = destination.get("modified", None)
+            destination = join(outpath, destination.text)
+
+            operation = Operation(module, outpath, path, source, destination)
+
+            operation.hash_module = module_hash
+            operation.hash_in = source_hash
+            operation.hash_out = destination_hash
+
+            operation.mtime_module = None if module_mtime is None else int(module_mtime)
+            operation.mtime_in = None if source_mtime is None else int(source_mtime)
+            operation.mtime_out = None if destination_mtime is None else int(destination_mtime)
+
+            buildlog._operations[module].append(operation)
 
         return buildlog
 
@@ -238,23 +296,45 @@ def to_xml(self, path, to_string=True):
         Convert the complete build log into a XML representation.
         """
         rootnode = lxml.etree.Element("buildlog")
+        extended_format=False
 
         with self.__lock:
+            versionnode = lxml.etree.SubElement(rootnode, "version")
+            versionnode.text = "2.0"
             outpathnode = lxml.etree.SubElement(rootnode, "outpath")
             outpathnode.text = relpath(self.outpath, path)
             for operation in self.operations:
                 operationnode = lxml.etree.SubElement(rootnode, "operation")
 
                 modulenode = lxml.etree.SubElement(operationnode, "module")
-                modulenode.text = operation.module_name
+                if extended_format:
+                    if operation.mtime_module is not None:
+                        modulenode.set("modified", str(operation.mtime_module))
+                    if operation.hash_module is not None:
+                        modulenode.set("hash", operation.hash_module)
+                    if operation.filename_module is not None:
+                        modulenode.text = operation.filename_module
+                modulenode.set("name", operation.module_name)
+
                 srcnode = lxml.etree.SubElement(operationnode, "source")
+                if extended_format:
+                    if operation.mtime_in is not None:
+                        srcnode.set("modified", str(operation.mtime_in))
+                    if operation.hash_in is not None:
+                        srcnode.set("hash", operation.hash_in)
                 srcnode.text = relpath(operation.filename_in, path)
+
                 destnode = lxml.etree.SubElement(operationnode, "destination")
+                if operation.mtime_out is not None:
+                    destnode.set("modified", str(operation.mtime_out))
+                if operation.hash_out is not None:
+                    destnode.set("hash", operation.hash_out)
                 destnode.text = relpath(operation.filename_out, path)
 
-                if operation.time is not None:
-                    timenode = lxml.etree.SubElement(operationnode, "time")
-                    timenode.text = "{:.3f} ms".format(operation.time * 1000)
+                if extended_format:
+                    if operation.time is not None:
+                        timenode = lxml.etree.SubElement(operationnode, "time")
+                        timenode.text = "{:.3f} ms".format(operation.time * 1000)
 
         if to_string:
             return lxml.etree.tostring(rootnode,

lbuild/environment.py

@@ -44,6 +44,8 @@ def _copyfile(sourcepath, destpath, fn_copy=None):
     if fn_copy is None:
         fn_copy = default_fn_copy
     if not SIMULATE:
+        if not os.path.exists(os.path.dirname(destpath)):
+            os.makedirs(os.path.dirname(destpath), exist_ok=True)
         fn_copy(sourcepath, destpath)
 
 
@@ -76,8 +78,6 @@ def _copytree(logger, src, dst, ignore=None,
                 _copytree(logger, sourcepath, destpath, ignore, fn_listdir, fn_isdir, fn_copy)
             else:
                 starttime = time.time()
-                if not os.path.exists(dst):
-                    os.makedirs(dst, exist_ok=True)
                 _copyfile(sourcepath, destpath, fn_copy)
                 endtime = time.time()
                 total = endtime - starttime
@@ -176,8 +176,6 @@ def log_copy(src, dest, operation_time):
                 _copytree(log_copy, src, destpath, wrap_ignore,
                           fn_listdir, fn_isdir, fn_copy)
             else:
-                if not os.path.exists(os.path.dirname(destpath)):
-                    os.makedirs(os.path.dirname(destpath), exist_ok=True)
                 _copyfile(src, destpath, fn_copy)
 
                 endtime = time.time()
@@ -227,8 +225,6 @@ def log_copy(src, dest, operation_time):
                       destpath,
                       wrap_ignore)
         else:
-            if not os.path.exists(os.path.dirname(destpath)):
-                os.makedirs(os.path.dirname(destpath), exist_ok=True)
             _copyfile(srcpath, destpath)
 
             endtime = time.time()
@@ -284,11 +280,11 @@ def template(self, src, dest=None, substitutions=None, filters=None, metadata=No
 
         outfile_name = self.outpath(dest)
 
-        # Create folder structure if it doesn't exists
         if not SIMULATE:
+            # Create folder structure if it doesn't exists
             if not os.path.exists(os.path.dirname(outfile_name)):
                 os.makedirs(os.path.dirname(outfile_name), exist_ok=True)
-
+            # Write template output to file
             with open(outfile_name, 'w', encoding="utf-8") as outfile:
                 outfile.write(output)
 

lbuild/exception.py

@@ -353,6 +353,21 @@ def __init__(self, module, file, conflict): # RepositoryInit
         super().__init__(msg)
 
 
+# =============================== API EXCEPTIONS ==============================
+class LbuildApiBuildlogNotFoundException(LbuildException):
+    def __init__(self, buildlog):
+        msg = "Buildlog '{}' not found!".format(_hl(_rel(buildlog)))
+        super().__init__(msg)
+        self.buildlog = buildlog
+
+class LbuildApiModifiedFilesException(LbuildException):
+    def __init__(self, buildlog, modified):
+        msg = ("Buildlog '{}' shows these generated files were modified:\n\n{}"
+               .format(_hl(_rel(buildlog)), _bp(_rel(m) for m in modified)))
+        super().__init__(msg)
+        self.buildlog = buildlog
+        self.modified = modified
+
 # =========================== REPOSITORY EXCEPTIONS ===========================
 class LbuildRepositoryNoNameException(LbuildDumpConfigException):
     def __init__(self, parser, repo): # RepositoryInit