diff --git a/lbuild/api.py b/lbuild/api.py index f0f95f9..f80d078 100644 --- a/lbuild/api.py +++ b/lbuild/api.py @@ -21,6 +21,8 @@ from lbuild.config import ConfigNode from lbuild.utils import listify, listrify from lbuild.logger import CallCounter +from lbuild.exception import LbuildApiBuildlogNotFoundException, LbuildApiModifiedFilesException +from pathlib import Path class Builder: @@ -121,7 +123,7 @@ def validate(self, modules=None, complete=True): self.parser.validate_modules(build_modules, complete) return (build_modules, CallCounter.levels) - def build(self, outpath, modules=None, simulate=False, use_symlinks=False): + def build(self, outpath, modules=None, simulate=False, use_symlinks=False, write_buildlog=True): """ Build the given set of modules. @@ -133,8 +135,45 @@ def build(self, outpath, modules=None, simulate=False, use_symlinks=False): simulate -- If set to True simulate the build process. In that case no output will be generated. """ + buildlogname = self.config.filename + ".log" + try: + self.clean(buildlogname) + except LbuildApiBuildlogNotFoundException: + pass + build_modules = self._filter_modules(modules) buildlog = BuildLog(outpath) lbuild.environment.SYMLINK_ON_COPY = use_symlinks self.parser.build_modules(build_modules, buildlog, simulate=simulate) + if write_buildlog and not simulate: + Path(buildlogname).write_bytes(buildlog.to_xml(to_string=True, path=self.cwd)) return buildlog + + def clean(self, buildlog, force=False): + buildlogfile = Path(buildlog) + if not buildlogfile.exists(): + raise LbuildApiBuildlogNotFoundException(buildlogfile) + buildlog = BuildLog.from_xml(buildlogfile.read_bytes(), path=self.cwd) + + unmodified, modified, missing = buildlog.compare_outpath() + if not force and len(modified): + raise LbuildApiModifiedFilesException(buildlogfile, modified) + + removed = [] + dirs = set() + for filename in sorted(unmodified + modified + [str(buildlogfile)]): + dirs.add(os.path.dirname(filename)) + try: + os.remove(filename) + removed.append(filename) + except OSError: + pass + + dirs = sorted(list(dirs), key=lambda d: d.count("/"), reverse=True) + for directory in dirs: + try: + os.removedirs(directory) + except OSError: + pass + + return removed diff --git a/lbuild/buildlog.py b/lbuild/buildlog.py index d2ec284..cbea1ee 100644 --- a/lbuild/buildlog.py +++ b/lbuild/buildlog.py @@ -14,6 +14,7 @@ import collections import threading from os.path import join, relpath, dirname, normpath, basename, isabs, abspath +from pathlib import Path import lxml.etree import lbuild.utils @@ -22,7 +23,6 @@ LOGGER = logging.getLogger('lbuild.buildlog') - class Operation: """ Representation of a build operation. @@ -46,6 +46,22 @@ def __init__(self, module_name, outpath, module_path, self.filename_in = abspath(filename_in) self.filename_out = abspath(filename_out) + self.filename_module = None + + self.hash_in = None + self.hash_out = None + self.hash_module = None + + self.mtime_in = None + self.mtime_out = None + self.mtime_module = None + + def compute_hash(self, module_filename=None): + self.mtime_in, self.hash_in = lbuild.utils.hash_file(self.filename_in) + self.mtime_out, self.hash_out = lbuild.utils.hash_file(self.filename_out) + self.mtime_module, self.hash_module = lbuild.utils.hash_file(module_filename) + if self.hash_module is not None: + self.filename_module = relpath(module_filename) def local_filename_in(self, relative_to=None): path = self.inpath @@ -161,6 +177,7 @@ def log(self, module, filename_in: str, filename_out: str, time=None, metadata=N with self.__lock: operation = Operation(module.fullname, self.outpath, module._filepath, filename_in, filename_out, time, metadata) + operation.compute_hash(module._filename) LOGGER.debug(str(operation)) previous = self._build_files.get(filename_out, None) @@ -173,7 +190,8 @@ def log(self, module, filename_in: str, filename_out: str, time=None, metadata=N return operation - def log_unsafe(self, modulename, filename_in, filename_out, time=None, metadata=None): + def log_unsafe(self, modulename, filename_in, filename_out, + time=None, metadata=None, compute_hash=True): """ Log an lbuild internal operation. @@ -184,6 +202,7 @@ def log_unsafe(self, modulename, filename_in, filename_out, time=None, metadata= """ operation = Operation(modulename, self.outpath, self.outpath, filename_in, filename_out, time, metadata) + if compute_hash: operation.compute_hash(); with self.__lock: self._operations[modulename].append(operation) @@ -218,6 +237,24 @@ def operations(self): operations = (o for olists in operations for o in olists) return sorted(operations, key=lambda o: (o.module_name, o.filename_in, o.filename_out)) + def compare_outpath(self): + unmodified = [] + modified = [] + missing = [] + for op in self.operations: + destname = op.local_filename_out() + if os.path.exists(destname): + mtime_out = int(os.path.getmtime(destname)) + if (op.mtime_out != mtime_out and + op.hash_out != lbuild.utils.hash_file(destname)[1]): + modified.append(destname) + else: + unmodified.append(destname) + else: + missing.append(destname) + + return (unmodified, modified, missing) + @staticmethod def from_xml(string, path): rootnode = lxml.etree.fromstring(string) @@ -225,11 +262,32 @@ def from_xml(string, path): buildlog = BuildLog(outpath) for opnode in rootnode.iterfind("operation"): - module_name = opnode.find("module").text - source = join(outpath, opnode.find("source").text) - destination = join(outpath, opnode.find("destination").text) - operation = Operation(module_name, outpath, path, source, destination) - buildlog._operations[module_name].append(operation) + module = opnode.find("module") + module_hash = module.get("hash", None) + module_mtime = module.get("mtime", None) + module = module.get("name") + + source = opnode.find("source") + source_hash = source.get("hash", None) + source_mtime = source.get("modified", None) + source = join(outpath, source.text) + + destination = opnode.find("destination") + destination_hash = destination.get("hash", None) + destination_mtime = destination.get("modified", None) + destination = join(outpath, destination.text) + + operation = Operation(module, outpath, path, source, destination) + + operation.hash_module = module_hash + operation.hash_in = source_hash + operation.hash_out = destination_hash + + operation.mtime_module = None if module_mtime is None else int(module_mtime) + operation.mtime_in = None if source_mtime is None else int(source_mtime) + operation.mtime_out = None if destination_mtime is None else int(destination_mtime) + + buildlog._operations[module].append(operation) return buildlog @@ -238,23 +296,45 @@ def to_xml(self, path, to_string=True): Convert the complete build log into a XML representation. """ rootnode = lxml.etree.Element("buildlog") + extended_format=False with self.__lock: + versionnode = lxml.etree.SubElement(rootnode, "version") + versionnode.text = "2.0" outpathnode = lxml.etree.SubElement(rootnode, "outpath") outpathnode.text = relpath(self.outpath, path) for operation in self.operations: operationnode = lxml.etree.SubElement(rootnode, "operation") modulenode = lxml.etree.SubElement(operationnode, "module") - modulenode.text = operation.module_name + if extended_format: + if operation.mtime_module is not None: + modulenode.set("modified", str(operation.mtime_module)) + if operation.hash_module is not None: + modulenode.set("hash", operation.hash_module) + if operation.filename_module is not None: + modulenode.text = operation.filename_module + modulenode.set("name", operation.module_name) + srcnode = lxml.etree.SubElement(operationnode, "source") + if extended_format: + if operation.mtime_in is not None: + srcnode.set("modified", str(operation.mtime_in)) + if operation.hash_in is not None: + srcnode.set("hash", operation.hash_in) srcnode.text = relpath(operation.filename_in, path) + destnode = lxml.etree.SubElement(operationnode, "destination") + if operation.mtime_out is not None: + destnode.set("modified", str(operation.mtime_out)) + if operation.hash_out is not None: + destnode.set("hash", operation.hash_out) destnode.text = relpath(operation.filename_out, path) - if operation.time is not None: - timenode = lxml.etree.SubElement(operationnode, "time") - timenode.text = "{:.3f} ms".format(operation.time * 1000) + if extended_format: + if operation.time is not None: + timenode = lxml.etree.SubElement(operationnode, "time") + timenode.text = "{:.3f} ms".format(operation.time * 1000) if to_string: return lxml.etree.tostring(rootnode, diff --git a/lbuild/environment.py b/lbuild/environment.py index f171697..4d9b462 100644 --- a/lbuild/environment.py +++ b/lbuild/environment.py @@ -44,6 +44,8 @@ def _copyfile(sourcepath, destpath, fn_copy=None): if fn_copy is None: fn_copy = default_fn_copy if not SIMULATE: + if not os.path.exists(os.path.dirname(destpath)): + os.makedirs(os.path.dirname(destpath), exist_ok=True) fn_copy(sourcepath, destpath) @@ -76,8 +78,6 @@ def _copytree(logger, src, dst, ignore=None, _copytree(logger, sourcepath, destpath, ignore, fn_listdir, fn_isdir, fn_copy) else: starttime = time.time() - if not os.path.exists(dst): - os.makedirs(dst, exist_ok=True) _copyfile(sourcepath, destpath, fn_copy) endtime = time.time() total = endtime - starttime @@ -176,8 +176,6 @@ def log_copy(src, dest, operation_time): _copytree(log_copy, src, destpath, wrap_ignore, fn_listdir, fn_isdir, fn_copy) else: - if not os.path.exists(os.path.dirname(destpath)): - os.makedirs(os.path.dirname(destpath), exist_ok=True) _copyfile(src, destpath, fn_copy) endtime = time.time() @@ -227,8 +225,6 @@ def log_copy(src, dest, operation_time): destpath, wrap_ignore) else: - if not os.path.exists(os.path.dirname(destpath)): - os.makedirs(os.path.dirname(destpath), exist_ok=True) _copyfile(srcpath, destpath) endtime = time.time() @@ -284,11 +280,11 @@ def template(self, src, dest=None, substitutions=None, filters=None, metadata=No outfile_name = self.outpath(dest) - # Create folder structure if it doesn't exists if not SIMULATE: + # Create folder structure if it doesn't exists if not os.path.exists(os.path.dirname(outfile_name)): os.makedirs(os.path.dirname(outfile_name), exist_ok=True) - + # Write template output to file with open(outfile_name, 'w', encoding="utf-8") as outfile: outfile.write(output) diff --git a/lbuild/exception.py b/lbuild/exception.py index d37dc8f..c58e13b 100644 --- a/lbuild/exception.py +++ b/lbuild/exception.py @@ -353,6 +353,21 @@ def __init__(self, module, file, conflict): # RepositoryInit super().__init__(msg) +# =============================== API EXCEPTIONS ============================== +class LbuildApiBuildlogNotFoundException(LbuildException): + def __init__(self, buildlog): + msg = "Buildlog '{}' not found!".format(_hl(_rel(buildlog))) + super().__init__(msg) + self.buildlog = buildlog + +class LbuildApiModifiedFilesException(LbuildException): + def __init__(self, buildlog, modified): + msg = ("Buildlog '{}' shows these generated files were modified:\n\n{}" + .format(_hl(_rel(buildlog)), _bp(_rel(m) for m in modified))) + super().__init__(msg) + self.buildlog = buildlog + self.modified = modified + # =========================== REPOSITORY EXCEPTIONS =========================== class LbuildRepositoryNoNameException(LbuildDumpConfigException): def __init__(self, parser, repo): # RepositoryInit diff --git a/lbuild/main.py b/lbuild/main.py index 7fff01f..b60e51f 100644 --- a/lbuild/main.py +++ b/lbuild/main.py @@ -16,13 +16,14 @@ import traceback import textwrap +from pathlib import Path + import lbuild.logger import lbuild.vcs.common from lbuild.format import format_option_short_description - from lbuild.api import Builder -__version__ = '1.13.0' +__version__ = '1.14.0' class InitAction: @@ -315,8 +316,13 @@ def register(self, argument_parser): @staticmethod def perform(args, builder): - buildlog = builder.build(args.path, args.modules, simulate=args.simulate, - use_symlinks=args.symlink) + try: + buildlog = builder.build(args.path, args.modules, simulate=args.simulate, + use_symlinks=args.symlink, write_buildlog=args.buildlog) + except lbuild.exception.LbuildApiModifiedFilesException as error: + raise lbuild.exception.LbuildException(str(error) + + "\nA build may overwrite these files, run '{}' to remove them anyways.".format( + lbuild.exception._hl("lbuild clean --force"))) if args.simulate: ostream = [] @@ -324,13 +330,6 @@ def perform(args, builder): ostream.append(operation.local_filename_out()) return "\n".join(sorted(ostream)) - if args.buildlog: - configfilename = args.config - logfilename = configfilename + ".log" - buildlog.log_unsafe("lbuild", "buildlog.xml.in", logfilename) - with open(logfilename, "wb") as logfile: - logfile.write(buildlog.to_xml(to_string=True, path=os.getcwd())) - return "" @@ -343,38 +342,32 @@ def register(self, argument_parser): parser.add_argument( "--buildlog", dest="buildlog", - default="project.xml.log", + default=next(Path(os.getcwd()).glob("*.xml.log"), "project.xml.log"), help="Use the given buildlog to identify the files to remove.") + parser.add_argument( + "--force", + dest="force_clean", + action="store_true", + default=False, + help="Remove modified files without error.") parser.set_defaults(execute_action=self.perform) @staticmethod def perform(args, builder): - ostream = [] - if os.path.exists(args.buildlog): - with open(args.buildlog, "rb") as logfile: - buildlog = lbuild.buildlog.BuildLog.from_xml(logfile.read(), path=os.getcwd()) - else: - builder.load(args.repositories) - buildlog = builder.build(args.path, simulate=True) - - dirs = set() - filenames = [op.local_filename_out() for op in buildlog.operations] - for filename in sorted(filenames): - ostream.append("Removing " + filename) - dirs.add(os.path.dirname(filename)) - try: - os.remove(filename) - except OSError: - pass - - dirs = sorted(list(dirs), key=lambda d: d.count("/"), reverse=True) - for directory in dirs: - try: - os.removedirs(directory) - except OSError: - pass - - return "\n".join(ostream) + # builder.load(args.repositories) + # buildlog = builder.build(args.path, simulate=True) + try: + removed = builder.clean(args.buildlog, args.force_clean) + except lbuild.exception.LbuildApiModifiedFilesException as error: + raise lbuild.exception.LbuildException(str(error) + + "\nRun '{}' to remove these files anyways.".format( + lbuild.exception._hl("lbuild clean --force"))) + except lbuild.exception.LbuildApiBuildlogNotFoundException as error: + raise lbuild.exception.LbuildException(str(error) + + "\nRun with '{}' or manually delete the generated files.".format( + lbuild.exception._hl("lbuild clean --buildlog path/to/buildlog.xml.log"))) + + return "\n".join("Removing '{}'".format(os.path.relpath(f)) for f in removed) class DependenciesAction(ManipulationActionBase): diff --git a/lbuild/utils.py b/lbuild/utils.py index 492c2ad..6065acc 100644 --- a/lbuild/utils.py +++ b/lbuild/utils.py @@ -12,8 +12,10 @@ import os import sys import uuid +import time import shutil import fnmatch +import hashlib import importlib.util import importlib.machinery @@ -203,3 +205,13 @@ def _is_pathname_valid(pathname: str) -> bool: except TypeError as exc: return False return True + +def hash_file(filename): + if os.path.exists(filename): + with open(filename, "rb") as fileobj: + return (int(os.path.getmtime(filename)), + hashlib.md5(fileobj.read().strip()).hexdigest()) + return (None, None) + +def hash_content(content): + return (int(time.time()), hashlib.md5(file_or_content).hexdigest()) diff --git a/test/buildlog_test.py b/test/buildlog_test.py index a8ba73d..1636490 100644 --- a/test/buildlog_test.py +++ b/test/buildlog_test.py @@ -65,14 +65,15 @@ def test_should_generate_xml(self): self.assertEqual(b""" + 2.0 . - repo:module1 + m1/in1 out1 - repo:module2 + m2/in2 out2