From 732985d57ba4f4d12ecdb659db40b6e86d7950b2 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Thu, 8 Feb 2024 16:33:36 -0800 Subject: [PATCH] Format files in projects/cross/*.py Arguably a lot of this is uglier due to the added newlines but it is the de-facto style for python, so we should follow that. --- pycheribuild/projects/cross/apache.py | 82 +- pycheribuild/projects/cross/bash.py | 13 +- .../projects/cross/benchmark_mixin.py | 50 +- pycheribuild/projects/cross/benchmarks.py | 208 +++-- pycheribuild/projects/cross/bodiagsuite.py | 63 +- pycheribuild/projects/cross/boost.py | 5 +- .../projects/cross/cheri_exercises.py | 85 +- pycheribuild/projects/cross/cheribsd.py | 833 ++++++++++++------ pycheribuild/projects/cross/cheritest.py | 10 +- pycheribuild/projects/cross/compiler_rt.py | 30 +- .../projects/cross/crosscompileproject.py | 12 +- .../projects/cross/current_directory.py | 13 +- pycheribuild/projects/cross/dbus.py | 23 +- pycheribuild/projects/cross/device_model.py | 9 +- pycheribuild/projects/cross/dlmalloc.py | 27 +- pycheribuild/projects/cross/doom.py | 26 +- pycheribuild/projects/cross/drm_kmod.py | 31 +- pycheribuild/projects/cross/ffmpeg.py | 60 +- pycheribuild/projects/cross/freertos.py | 156 ++-- pycheribuild/projects/cross/freetype.py | 8 +- pycheribuild/projects/cross/gdb.py | 90 +- pycheribuild/projects/cross/git.py | 6 +- pycheribuild/projects/cross/gkermit.py | 3 +- pycheribuild/projects/cross/glib.py | 13 +- pycheribuild/projects/cross/gmp.py | 9 +- .../projects/cross/juliet_test_suite.py | 86 +- pycheribuild/projects/cross/kde.py | 546 +++++++++--- pycheribuild/projects/cross/littlekernel.py | 66 +- .../projects/cross/llvm_test_suite.py | 19 +- pycheribuild/projects/cross/mesa.py | 49 +- pycheribuild/projects/cross/mpfr.py | 9 +- pycheribuild/projects/cross/mrs.py | 40 +- pycheribuild/projects/cross/neomutt.py | 7 +- pycheribuild/projects/cross/newlib.py | 99 ++- pycheribuild/projects/cross/nginx.py | 55 +- pycheribuild/projects/cross/opensbi.py | 43 +- pycheribuild/projects/cross/openssl.py | 5 +- pycheribuild/projects/cross/picolibc.py | 4 +- pycheribuild/projects/cross/pkg.py | 7 +- pycheribuild/projects/cross/postgres.py | 40 +- pycheribuild/projects/cross/python.py | 35 +- pycheribuild/projects/cross/qt5.py | 277 +++--- pycheribuild/projects/cross/rlbox.py | 5 +- pycheribuild/projects/cross/ros2.py | 41 +- pycheribuild/projects/cross/rtems.py | 18 +- .../projects/cross/simple_benchmark.py | 37 +- pycheribuild/projects/cross/snmalloc.py | 34 +- pycheribuild/projects/cross/sqlite.py | 5 +- pycheribuild/projects/cross/u_boot.py | 22 +- pycheribuild/projects/cross/wayland.py | 123 ++- pycheribuild/projects/cross/webkit.py | 62 +- pycheribuild/projects/cross/x11.py | 174 ++-- pycheribuild/projects/cross/zlib.py | 7 +- 53 files changed, 2503 insertions(+), 1277 deletions(-) diff --git a/pycheribuild/projects/cross/apache.py b/pycheribuild/projects/cross/apache.py index 768060de7..938473466 100644 --- a/pycheribuild/projects/cross/apache.py +++ b/pycheribuild/projects/cross/apache.py @@ -42,8 +42,7 @@ class BuildPcre(CrossCompileAutotoolsProject): target = "pcre" - repository = SubversionRepository("svn://vcs.pcre.org/pcre", - default_branch="code/trunk") + repository = SubversionRepository("svn://vcs.pcre.org/pcre", default_branch="code/trunk") native_install_dir = DefaultInstallDir.BOOTSTRAP_TOOLS @@ -54,8 +53,7 @@ def configure(self, **kwargs): class BuildApr(CrossCompileAutotoolsProject): target = "apr" - repository = GitRepository("https://github.com/CTSRD-CHERI/apr.git", - default_branch="cheri") + repository = GitRepository("https://github.com/CTSRD-CHERI/apr.git", default_branch="cheri") dependencies = ("libexpat",) @@ -63,12 +61,14 @@ class BuildApr(CrossCompileAutotoolsProject): def setup(self): super().setup() - self.configure_args.extend([ - "--enable-threads", - "--enable-posix-shm", - "--with-devrandom", - "--with-expat=" + str(BuildExpat.get_install_dir(self)), - ]) + self.configure_args.extend( + [ + "--enable-threads", + "--enable-posix-shm", + "--with-devrandom", + "--with-expat=" + str(BuildExpat.get_install_dir(self)), + ] + ) if self.build_type.is_debug: self.configure_args.append("--enable-debug") @@ -79,17 +79,18 @@ def setup(self): if not self.compiling_for_host(): # Can't determine these when cross-compiling - self.configure_environment.update(ac_cv_file__dev_zero="yes", - ac_cv_mmap__dev_zero="yes", - # XXX: This might be yes on Linux - ac_cv_func_setpgrp_void="no", - ac_cv_struct_rlimit="yes", - ac_cv_func_sem_open="yes", - apr_cv_process_shared_works="yes", - apr_cv_mutex_robust_shared="yes", - # XXX: This might be yes on Linux - apr_cv_tcp_nodelay_with_cork="no", - ) + self.configure_environment.update( + ac_cv_file__dev_zero="yes", + ac_cv_mmap__dev_zero="yes", + # XXX: This might be yes on Linux + ac_cv_func_setpgrp_void="no", + ac_cv_struct_rlimit="yes", + ac_cv_func_sem_open="yes", + apr_cv_process_shared_works="yes", + apr_cv_mutex_robust_shared="yes", + # XXX: This might be yes on Linux + apr_cv_tcp_nodelay_with_cork="no", + ) def configure(self, **kwargs): self.run_cmd("./buildconf", cwd=self.source_dir) @@ -98,21 +99,22 @@ def configure(self, **kwargs): class BuildApache(CrossCompileAutotoolsProject): target = "apache" - repository = GitRepository("https://github.com/CTSRD-CHERI/apache-httpd.git", - default_branch="2.4.x-cheri") + repository = GitRepository("https://github.com/CTSRD-CHERI/apache-httpd.git", default_branch="2.4.x-cheri") dependencies = ("apr", "pcre") def setup(self): super().setup() - self.configure_args.extend([ - "--enable-layout=FreeBSD", - "--enable-http", - "--enable-mod-ssl", - "--with-expat=" + str(BuildExpat.get_install_dir(self)), - "--with-pcre=" + str(BuildPcre.get_install_dir(self)), - "--with-apr=" + str(BuildApr.get_install_dir(self)), - ]) + self.configure_args.extend( + [ + "--enable-layout=FreeBSD", + "--enable-http", + "--enable-mod-ssl", + "--with-expat=" + str(BuildExpat.get_install_dir(self)), + "--with-pcre=" + str(BuildPcre.get_install_dir(self)), + "--with-apr=" + str(BuildApr.get_install_dir(self)), + ] + ) if self.build_type.is_debug: self.configure_args.append("--enable-debugger-mode") @@ -130,12 +132,16 @@ def configure(self, **kwargs): # gen_test_char rules in server/ assume a native build if not self.compiling_for_host(): - self.run_cmd(str(self.host_CC), "-DCROSS_COMPILE", "-c", - self.source_dir / "server" / "gen_test_char.c", - "-o", "gen_test_char.lo", - cwd=self.build_dir / "server") - self.run_cmd(str(self.host_CC), "gen_test_char.lo", "-o", - "gen_test_char", cwd=self.build_dir / "server") + self.run_cmd( + str(self.host_CC), + "-DCROSS_COMPILE", + "-c", + self.source_dir / "server" / "gen_test_char.c", + "-o", + "gen_test_char.lo", + cwd=self.build_dir / "server", + ) + self.run_cmd(str(self.host_CC), "gen_test_char.lo", "-o", "gen_test_char", cwd=self.build_dir / "server") class BuildSSLProc(CrossCompileCMakeProject): @@ -163,4 +169,4 @@ def setup(self): super().setup() self.configure_args.append( "--with-sslproc=" + str(BuildSSLProc.get_install_dir(self)), - ) + ) diff --git a/pycheribuild/projects/cross/bash.py b/pycheribuild/projects/cross/bash.py index 517714f58..1e6eea723 100644 --- a/pycheribuild/projects/cross/bash.py +++ b/pycheribuild/projects/cross/bash.py @@ -31,12 +31,12 @@ class BuildBash(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/bash", - default_branch="cheri") + repository = GitRepository("https://github.com/CTSRD-CHERI/bash", default_branch="cheri") cross_install_dir = DefaultInstallDir.ROOTFS_OPTBASE path_in_rootfs = "/usr/local" - set_as_root_shell = BoolConfigOption("set-as-root-shell", show_help=True, - help="Set root's shell to bash (in the target rootfs)") + set_as_root_shell = BoolConfigOption( + "set-as-root-shell", show_help=True, help="Set root's shell to bash (in the target rootfs)" + ) def setup(self): super().setup() @@ -63,12 +63,13 @@ def install(self, **kwargs): self.create_symlink(Path("/usr/local/bin/bash"), self.destdir / "bin/bash", relative=False) self.add_unique_line_to_file(self.destdir / "etc/shells", "/usr/local/bin/bash") if self.set_as_root_shell: + def rewrite(old): new = [] for line in old: - fields = line.split(':') + fields = line.split(":") if len(fields) == 10 and fields[0] == "root": - line = ':'.join(fields[0:9] + ["/usr/local/bin/bash"]) + line = ":".join(fields[0:9] + ["/usr/local/bin/bash"]) new.append(line) return new diff --git a/pycheribuild/projects/cross/benchmark_mixin.py b/pycheribuild/projects/cross/benchmark_mixin.py index adc94e0e2..91fd610d1 100644 --- a/pycheribuild/projects/cross/benchmark_mixin.py +++ b/pycheribuild/projects/cross/benchmark_mixin.py @@ -48,8 +48,9 @@ # We also build benchmarks for hybrid to see whether those compilation flags change the results class BenchmarkMixin(_BenchmarkMixinBase): - supported_architectures = (CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID_FOR_PURECAP_ROOTFS + - CompilationTargets.ALL_NATIVE) + supported_architectures = ( + CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID_FOR_PURECAP_ROOTFS + CompilationTargets.ALL_NATIVE + ) default_build_type = BuildType.RELEASE prefer_full_lto_over_thin_lto = True @@ -59,10 +60,15 @@ def optimization_flags(self): return ["-O3"] return super().optimization_flags - def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str]" = None, - benchmark_script: "Optional[str]" = None, - benchmark_script_args: "Optional[list[str]]" = None, - extra_runbench_args: "Optional[list[str]]" = None): + def run_fpga_benchmark( + self, + benchmarks_dir: Path, + *, + output_file: "Optional[str]" = None, + benchmark_script: "Optional[str]" = None, + benchmark_script_args: "Optional[list[str]]" = None, + extra_runbench_args: "Optional[list[str]]" = None, + ): assert benchmarks_dir is not None assert output_file is not None, "output_file must be set to a valid value" xtarget = self.crosscompile_target @@ -80,12 +86,16 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str basic_args = [] if self.config.benchmark_with_qemu: from ...projects.build_qemu import BuildQEMU + qemu_path = BuildQEMU.qemu_binary(self) qemu_ssh_socket = find_free_port() if not qemu_path.exists(): self.fatal("QEMU binary", qemu_path, "doesn't exist") - basic_args += ["--use-qemu-instead-of-fpga", "--qemu-path=" + str(qemu_path), - "--qemu-ssh-port=" + str(qemu_ssh_socket.port)] + basic_args += [ + "--use-qemu-instead-of-fpga", + "--qemu-path=" + str(qemu_path), + "--qemu-ssh-port=" + str(qemu_ssh_socket.port), + ] elif not self.compiling_for_mips(include_purecap=True): self.fatal("run_fpga_benchmark has not been updated for RISC-V/AArch64") return @@ -101,11 +111,13 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str env_var = "LD_64_PRELOAD" else: env_var = "LD_PRELOAD" - pre_cmd = "export {}={};".format(env_var, - shlex.quote("/tmp/benchdir/" + self.config.benchmark_ld_preload.name)) + pre_cmd = "export {}={};".format( + env_var, shlex.quote("/tmp/benchdir/" + self.config.benchmark_ld_preload.name) + ) if env_var == "LD_64C_PRELOAD": - pre_cmd += "export {}={};".format("LD_CHERI_PRELOAD", - shlex.quote("/tmp/benchdir/" + self.config.benchmark_ld_preload.name)) + pre_cmd += "export {}={};".format( + "LD_CHERI_PRELOAD", shlex.quote("/tmp/benchdir/" + self.config.benchmark_ld_preload.name) + ) runbench_args.append("--pre-command=" + pre_cmd) if self.config.benchmark_fpga_extra_args: basic_args.extend(self.config.benchmark_fpga_extra_args) @@ -115,6 +127,7 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str runbench_args.append("--interact") from ...projects.cross.cheribsd import BuildCheriBsdMfsKernel, ConfigPlatform + if self.config.benchmark_with_qemu: # When benchmarking with QEMU we always spawn a new instance target = self.target_info @@ -128,10 +141,12 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str # use a bitfile from jenkins. TODO: add option for overriding assert xtarget.is_riscv(include_purecap=True) basic_args.append("--jenkins-bitfile") - mfs_kernel = BuildCheriBsdMfsKernel.get_instance_for_cross_target(xtarget.get_rootfs_target(), self.config, - caller=self) - kernel_config = mfs_kernel.default_kernel_config(ConfigPlatform.GFE, - benchmark=not self.config.benchmark_with_debug_kernel) + mfs_kernel = BuildCheriBsdMfsKernel.get_instance_for_cross_target( + xtarget.get_rootfs_target(), self.config, caller=self + ) + kernel_config = mfs_kernel.default_kernel_config( + ConfigPlatform.GFE, benchmark=not self.config.benchmark_with_debug_kernel + ) kernel_image = mfs_kernel.get_kernel_install_path(kernel_config) basic_args.append("--kernel-img=" + str(kernel_image)) else: @@ -150,4 +165,5 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str qemu_ssh_socket.socket.close() self.run_cmd( [str(cheribuild_path / "vcu118-bsd-boot.py"), *basic_args, "-vvvvv", "runbench", *runbench_args], - give_tty_control=True) + give_tty_control=True, + ) diff --git a/pycheribuild/projects/cross/benchmarks.py b/pycheribuild/projects/cross/benchmarks.py index 058db9767..d558293c0 100644 --- a/pycheribuild/projects/cross/benchmarks.py +++ b/pycheribuild/projects/cross/benchmarks.py @@ -67,13 +67,20 @@ class BuildMibench(BenchmarkMixin, CrossCompileProject): @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.benchmark_size = cls.add_config_option("benchmark-size", choices=("small", "large"), default="large", - kind=str, help="Size of benchmark input data to use") + cls.benchmark_size = cls.add_config_option( + "benchmark-size", + choices=("small", "large"), + default="large", + kind=str, + help="Size of benchmark input data to use", + ) @property def bundle_dir(self): - return Path(self.build_dir, self.crosscompile_target.generic_target_suffix + - self.build_configuration_suffix() + "-bundle") + return Path( + self.build_dir, + self.crosscompile_target.generic_target_suffix + self.build_configuration_suffix() + "-bundle", + ) @property def benchmark_version(self): @@ -88,18 +95,21 @@ def benchmark_version(self): def compile(self, **kwargs): new_env = dict() if not self.compiling_for_host(): - new_env = dict(MIPS_SDK=self.target_info.sdk_root_dir, - CHERI128_SDK=self.target_info.sdk_root_dir, - CHERI256_SDK=self.target_info.sdk_root_dir, - CHERI_SDK=self.target_info.sdk_root_dir) + new_env = dict( + MIPS_SDK=self.target_info.sdk_root_dir, + CHERI128_SDK=self.target_info.sdk_root_dir, + CHERI256_SDK=self.target_info.sdk_root_dir, + CHERI_SDK=self.target_info.sdk_root_dir, + ) with self.set_env(**new_env): # We can't fall back to /usr/bin/ar here since that breaks on MacOS if not self.compiling_for_host(): self.make_args.set(AR=str(self.sdk_bindir / "llvm-ar") + " rc") self.make_args.set(AR2=str(self.sdk_bindir / "llvm-ranlib")) self.make_args.set(RANLIB=str(self.sdk_bindir / "llvm-ranlib")) - self.make_args.set(MIPS_SYSROOT=self.sdk_sysroot, CHERI128_SYSROOT=self.sdk_sysroot, - CHERI256_SYSROOT=self.sdk_sysroot) + self.make_args.set( + MIPS_SYSROOT=self.sdk_sysroot, CHERI128_SYSROOT=self.sdk_sysroot, CHERI256_SYSROOT=self.sdk_sysroot + ) self.make_args.set(ADDITIONAL_CFLAGS=self.commandline_to_str(self.default_compiler_flags)) self.make_args.set(ADDITIONAL_LDFLAGS=self.commandline_to_str(self.default_ldflags)) @@ -144,9 +154,11 @@ def run_tests(self): return # testing, not benchmarking -> run only once: (-s small / -s large?) test_command = "cd '/build/{dirname}' && ./run_jenkins-bluehive.sh -d0 -r1 -s {size} {version}".format( - dirname=self.bundle_dir.name, size=self.benchmark_size, version=self.benchmark_version) - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), mount_builddir=True) + dirname=self.bundle_dir.name, size=self.benchmark_size, version=self.benchmark_version + ) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", "--test-command", test_command, "--test-timeout", str(120 * 60), mount_builddir=True + ) def run_benchmarks(self): if not self.compiling_for_mips(include_purecap=True): @@ -158,11 +170,19 @@ def run_benchmarks(self): if not (benchmark_dir / "run_jenkins-bluehive.sh").exists(): self.fatal("Created invalid benchmark bundle...") num_iterations = self.config.benchmark_iterations or 10 - self.run_fpga_benchmark(benchmark_dir, output_file=self.default_statcounters_csv_name, - benchmark_script_args=["-d1", "-r" + str(num_iterations), - "-s", self.benchmark_size, - "-o", self.default_statcounters_csv_name, - self.benchmark_version]) + self.run_fpga_benchmark( + benchmark_dir, + output_file=self.default_statcounters_csv_name, + benchmark_script_args=[ + "-d1", + "-r" + str(num_iterations), + "-s", + self.benchmark_size, + "-o", + self.default_statcounters_csv_name, + self.benchmark_version, + ], + ) class BuildMiBenchNew(BuildLLVMTestSuiteBase): @@ -171,19 +191,19 @@ class BuildMiBenchNew(BuildLLVMTestSuiteBase): def setup(self): super().setup() - self.add_cmake_options(TEST_SUITE_SUBDIRS="MultiSource/Benchmarks/MiBench", - TEST_SUITE_COPY_DATA=True) + self.add_cmake_options(TEST_SUITE_SUBDIRS="MultiSource/Benchmarks/MiBench", TEST_SUITE_COPY_DATA=True) def compile(self, **kwargs): super().compile(**kwargs) - self.install_file(self.source_dir / "MultiSource/lit.local.cfg", - self.build_dir / "MultiSource/lit.local.cfg", force=True) + self.install_file( + self.source_dir / "MultiSource/lit.local.cfg", self.build_dir / "MultiSource/lit.local.cfg", force=True + ) def install(self, **kwargs): root_dir = str(self.build_dir / "MultiSource/Benchmarks/MiBench") for curdir, dirnames, filenames in os.walk(root_dir): # We don't run some benchmarks (e.g. consumer-typeset or consumer-lame) yet - for ignored_dirname in ('CMakeFiles', 'consumer-typeset', 'consumer-lame', 'office-ispell', 'telecomm-gsm'): + for ignored_dirname in ("CMakeFiles", "consumer-typeset", "consumer-lame", "office-ispell", "telecomm-gsm"): if ignored_dirname in dirnames: dirnames.remove(ignored_dirname) relpath = os.path.relpath(curdir, root_dir) @@ -208,10 +228,12 @@ class BuildOlden(BenchmarkMixin, CrossCompileProject): def compile(self, **kwargs): new_env = dict() if not self.compiling_for_host(): - new_env = dict(MIPS_SDK=self.target_info.sdk_root_dir, - CHERI128_SDK=self.target_info.sdk_root_dir, - CHERI256_SDK=self.target_info.sdk_root_dir, - CHERI_SDK=self.target_info.sdk_root_dir) + new_env = dict( + MIPS_SDK=self.target_info.sdk_root_dir, + CHERI128_SDK=self.target_info.sdk_root_dir, + CHERI256_SDK=self.target_info.sdk_root_dir, + CHERI_SDK=self.target_info.sdk_root_dir, + ) with self.set_env(**new_env): if not self.compiling_for_host(): self.make_args.set(SYSROOT_DIRNAME=self.cross_sysroot_path.name) @@ -228,8 +250,9 @@ def compile(self, **kwargs): self.fatal("Unknown target: ", self.crosscompile_target) # copy asan libraries and the run script to the bin dir to ensure that we can run with --test from the # build directory. - self.install_file(self.source_dir / "run_jenkins-bluehive.sh", - self.build_dir / "bin/run_jenkins-bluehive.sh", force=True) + self.install_file( + self.source_dir / "run_jenkins-bluehive.sh", self.build_dir / "bin/run_jenkins-bluehive.sh", force=True + ) if self.compiling_for_mips(include_purecap=False) and self.use_asan: self.copy_asan_dependencies(self.build_dir / "bin/lib") @@ -267,9 +290,9 @@ def run_tests(self): return # testing, not benchmarking -> run only once: (-s small / -s large?) test_command = f"cd /build/bin && ./run_jenkins-bluehive.sh -d0 -r1 {self.test_arch_suffix}" - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), - mount_builddir=True) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", "--test-command", test_command, "--test-timeout", str(120 * 60), mount_builddir=True + ) def run_benchmarks(self): if not self.compiling_for_mips(include_purecap=True): @@ -282,10 +305,17 @@ def run_benchmarks(self): if not (benchmark_dir / "run_jenkins-bluehive.sh").exists(): self.fatal("Created invalid benchmark bundle...") num_iterations = self.config.benchmark_iterations or 15 - self.run_fpga_benchmark(benchmark_dir, output_file=self.default_statcounters_csv_name, - benchmark_script_args=["-d1", "-r" + str(num_iterations), "-o", - self.default_statcounters_csv_name, - self.test_arch_suffix]) + self.run_fpga_benchmark( + benchmark_dir, + output_file=self.default_statcounters_csv_name, + benchmark_script_args=[ + "-d1", + "-r" + str(num_iterations), + "-o", + self.default_statcounters_csv_name, + self.test_arch_suffix, + ], + ) class BuildSpec2006New(BuildLLVMTestSuiteBase): @@ -297,8 +327,9 @@ class BuildSpec2006New(BuildLLVMTestSuiteBase): @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.spec_iso_path = cls.add_optional_path_option("iso-path", altname="spec-sources", - help="Path to the SPEC2006 ISO image or extracted sources") + cls.spec_iso_path = cls.add_optional_path_option( + "iso-path", altname="spec-sources", help="Path to the SPEC2006 ISO image or extracted sources" + ) cls.fast_benchmarks_only = cls.add_bool_option("fast-benchmarks-only", default=False) cls.workload = cls.add_config_option("workload", choices=("test", "train", "ref"), default="test") cls.benchmark_override = cls.add_list_option("benchmarks", help="override the list of benchmarks to run") @@ -328,7 +359,7 @@ def __init__(self, *args, **kwargs): "471.omnetpp", # 3 runs = 0:05:09 -> ~1:45min per run "473.astar", # 3 runs = 0:31:41 -> ~10:30 mins per run "483.xalancbmk", # 3 runs = 0:00:55 -> ~20 secs per run" - ] + ] self.complete_benchmark_list = [*self.working_benchmark_list, "400.perlbench", "403.gcc", "429.mcf"] self.fast_list = ["471.omnetpp", "483.xalancbmk", "456.hmmer", "462.libquantum"] if self.benchmark_override: @@ -345,16 +376,19 @@ def setup(self): super().setup() # Only build spec2006 # self.add_cmake_options(TEST_SUITE_SUBDIRS="External/SPEC/CINT2006;External/SPEC/CFP2006", - self.add_cmake_options(TEST_SUITE_SUBDIRS="External/SPEC/CINT2006", - TEST_SUITE_COPY_DATA=True, - TEST_SUITE_RUN_TYPE=self.workload, - TEST_SUITE_SPEC2006_ROOT=self.extracted_spec_sources) + self.add_cmake_options( + TEST_SUITE_SUBDIRS="External/SPEC/CINT2006", + TEST_SUITE_COPY_DATA=True, + TEST_SUITE_RUN_TYPE=self.workload, + TEST_SUITE_SPEC2006_ROOT=self.extracted_spec_sources, + ) def _check_broken_bsdtar(self, bsdtar: Path) -> "tuple[bool, tuple[int, ...]]": if self.config.pretend and not bsdtar.exists(): return False, (0, 0, 0) - bsdtar_version = get_program_version(bsdtar, regex=rb"bsdtar\s+(\d+)\.(\d+)\.?(\d+)? \- libarchive", - config=self.config) + bsdtar_version = get_program_version( + bsdtar, regex=rb"bsdtar\s+(\d+)\.(\d+)\.?(\d+)? \- libarchive", config=self.config + ) # At least version 3.3.2 of libarchive fails to extract the SPEC ISO image correctly (at least two files # are missing). This does not appear to be a problem with Ubuntu 18.04's version 3.2.2. return (3, 3) <= bsdtar_version < (3, 5), bsdtar_version @@ -370,9 +404,13 @@ def extract_spec_iso_image(self): bsdtar = libarchive_path / "bin/bsdtar" bsdtar_broken, bsdtar_version = self._check_broken_bsdtar(bsdtar) if bsdtar_broken: - self.fatal("The installed version of libarchive (", ".".join(map(str, bsdtar_version)), - ") has a bug that results in some files not being extracted from the SPEC ISO.", - fixit_hint="Please update bsdtar to at least 3.5.0", sep="") + self.fatal( + "The installed version of libarchive (", + ".".join(map(str, bsdtar_version)), + ") has a bug that results in some files not being extracted from the SPEC ISO.", + fixit_hint="Please update bsdtar to at least 3.5.0", + sep="", + ) self.run_cmd(bsdtar, "xf", self.spec_iso_path, "-C", self.extracted_spec_sources, cwd=self.build_dir) # Some of the files in that archive are not user-writable; go pave @@ -394,7 +432,7 @@ def install(self, **kwargs): def install_benchmark_dir(self, root_dir: str): for curdir, dirnames, filenames in os.walk(root_dir): # We don't run some benchmarks (e.g. consumer-typeset or consumer-lame) yet - for ignored_dirname in ('CMakeFiles', ): + for ignored_dirname in ("CMakeFiles",): if ignored_dirname in dirnames: dirnames.remove(ignored_dirname) relpath = os.path.relpath(curdir, root_dir) @@ -427,8 +465,10 @@ def setup_config_options(cls, **kwargs): @property def bundle_dir(self): - return Path(self.build_dir, "lmbench-" + self.crosscompile_target.generic_target_suffix + - self.build_configuration_suffix() + "-bundle") + return Path( + self.build_dir, + "lmbench-" + self.crosscompile_target.generic_target_suffix + self.build_configuration_suffix() + "-bundle", + ) @property def benchmark_version(self): @@ -443,9 +483,11 @@ def benchmark_version(self): def compile(self, **kwargs): new_env = dict() if not self.compiling_for_host(): - new_env = dict(MIPS_SDK=self.target_info.sdk_root_dir, - CHERI128_SDK=self.target_info.sdk_root_dir, - CHERI_SDK=self.target_info.sdk_root_dir) + new_env = dict( + MIPS_SDK=self.target_info.sdk_root_dir, + CHERI128_SDK=self.target_info.sdk_root_dir, + CHERI_SDK=self.target_info.sdk_root_dir, + ) with self.set_env(**new_env): self.make_args.set(CC="clang") if not self.compiling_for_host(): @@ -461,14 +503,11 @@ def compile(self, **kwargs): def _create_benchmark_dir(self, install_dir: Path): self.makedirs(install_dir) - self.clean_directory(install_dir / "bin", keep_root=False, - ensure_dir_exists=False) - self.clean_directory(install_dir / "scripts", keep_root=False, - ensure_dir_exists=False) + self.clean_directory(install_dir / "bin", keep_root=False, ensure_dir_exists=False) + self.clean_directory(install_dir / "scripts", keep_root=False, ensure_dir_exists=False) self.copy_directory(self.build_dir / "bin", install_dir / "bin") self.copy_directory(self.build_dir / "scripts", install_dir / "scripts") - self.install_file(self.source_dir / "src" / "Makefile", - install_dir / "src" / "Makefile") + self.install_file(self.source_dir / "src" / "Makefile", install_dir / "src" / "Makefile") self.install_file(self.source_dir / "Makefile", install_dir / "Makefile") def install(self, **kwargs): @@ -484,8 +523,9 @@ def run_tests(self): return # testing, not benchmarking -> run only once test_command = f"cd '/build/{self.bundle_dir.name}' && ./run_jenkins-bluehive.sh -d0 -r1 -s" - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), mount_builddir=True) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", "--test-command", test_command, "--test-timeout", str(120 * 60), mount_builddir=True + ) class BuildUnixBench(BenchmarkMixin, CrossCompileProject): @@ -505,13 +545,18 @@ class BuildUnixBench(BenchmarkMixin, CrossCompileProject): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.fixed_iterations = cls.add_bool_option( - "fixed-iterations", default=False, - help="Run benchmarks for given number of iterations instead of duration.") + "fixed-iterations", default=False, help="Run benchmarks for given number of iterations instead of duration." + ) @property def bundle_dir(self): - return Path(self.build_dir, "unixbench-" + self.crosscompile_target.generic_target_suffix + - self.build_configuration_suffix() + "-bundle") + return Path( + self.build_dir, + "unixbench-" + + self.crosscompile_target.generic_target_suffix + + self.build_configuration_suffix() + + "-bundle", + ) @property def benchmark_version(self): @@ -526,9 +571,11 @@ def benchmark_version(self): def compile(self, **kwargs): new_env = dict() if not self.compiling_for_host(): - new_env = dict(MIPS_SDK=self.target_info.sdk_root_dir, - CHERI128_SDK=self.target_info.sdk_root_dir, - CHERI_SDK=self.target_info.sdk_root_dir) + new_env = dict( + MIPS_SDK=self.target_info.sdk_root_dir, + CHERI128_SDK=self.target_info.sdk_root_dir, + CHERI_SDK=self.target_info.sdk_root_dir, + ) with self.set_env(**new_env): self.make_args.set(CC="clang") if self.compiling_for_mips(include_purecap=True): @@ -550,8 +597,7 @@ def compile(self, **kwargs): def _create_benchmark_dir(self, install_dir: Path): self.makedirs(install_dir) - self.clean_directory(install_dir / "pgms", keep_root=False, - ensure_dir_exists=False) + self.clean_directory(install_dir / "pgms", keep_root=False, ensure_dir_exists=False) self.copy_directory(self.build_dir / "UnixBench" / "pgms", install_dir / "pgms") self.install_file(self.source_dir / "run.sh", install_dir / "run.sh") @@ -569,16 +615,21 @@ class NetPerfBench(BenchmarkMixin, CrossCompileAutotoolsProject): # Keep the old bundles when cleaning _extra_git_clean_excludes = ["--exclude=*-bundle"] # The makefiles here can't support any other tagets: - supported_architectures = (CompilationTargets.CHERIBSD_RISCV_NO_CHERI, - CompilationTargets.CHERIBSD_RISCV_HYBRID, - CompilationTargets.CHERIBSD_RISCV_PURECAP) + supported_architectures = ( + CompilationTargets.CHERIBSD_RISCV_NO_CHERI, + CompilationTargets.CHERIBSD_RISCV_HYBRID, + CompilationTargets.CHERIBSD_RISCV_PURECAP, + ) @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.hw_counters = cls.add_config_option("enable-hw-counters", - choices=("pmc", "statcounters"), default="statcounters", - help="Use hardware performance counters") + cls.hw_counters = cls.add_config_option( + "enable-hw-counters", + choices=("pmc", "statcounters"), + default="statcounters", + help="Use hardware performance counters", + ) def configure(self, **kwargs): self.configure_args.append("--enable-unixdomain") @@ -588,8 +639,7 @@ def configure(self, **kwargs): super().configure(**kwargs) def process(self): - if (self.compiling_for_riscv(include_purecap=True) and - self.hw_counters == "pmc"): + if self.compiling_for_riscv(include_purecap=True) and self.hw_counters == "pmc": self.fatal("hwpmc not supported on riscv") return super().process() diff --git a/pycheribuild/projects/cross/bodiagsuite.py b/pycheribuild/projects/cross/bodiagsuite.py index 2a1adc2a0..ed962c406 100644 --- a/pycheribuild/projects/cross/bodiagsuite.py +++ b/pycheribuild/projects/cross/bodiagsuite.py @@ -43,8 +43,9 @@ class BuildBODiagSuite(CrossCompileCMakeProject): target = "bodiagsuite" - repository = GitRepository("https://github.com/CTSRD-CHERI/bodiagsuite", - old_urls=[b"https://github.com/nwf/bodiagsuite"]) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/bodiagsuite", old_urls=[b"https://github.com/nwf/bodiagsuite"] + ) default_install_dir = DefaultInstallDir.DO_NOT_INSTALL default_build_type = BuildType.DEBUG default_use_asan = True @@ -69,19 +70,27 @@ def build_dir_suffix(self): @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.use_valgrind = cls.add_bool_option("use-valgrind", help="Run tests using valgrind (native only)", - only_add_for_targets=(CompilationTargets.NATIVE,)) - cls.use_stack_protector = cls.add_bool_option("use-stack-protector", - help="Compile tests with stack-protector (non-CHERI only)") - cls.use_fortify_source = cls.add_bool_option("use-fortify-source", - help="Compile tests with _DFORTIFY_SOURCE=2 (no effect on " - "FreeBSD)") - cls.use_softboundcets = cls.add_bool_option("use-softboundcets", - help="Compile tests with SoftBoundCETS (native only)", - only_add_for_targets=(CompilationTargets.NATIVE,)) - cls.use_effectivesan = cls.add_bool_option("use-effectivesan", - help="Compile tests with EffectiveSan (native only)", - only_add_for_targets=(CompilationTargets.NATIVE,)) + cls.use_valgrind = cls.add_bool_option( + "use-valgrind", + help="Run tests using valgrind (native only)", + only_add_for_targets=(CompilationTargets.NATIVE,), + ) + cls.use_stack_protector = cls.add_bool_option( + "use-stack-protector", help="Compile tests with stack-protector (non-CHERI only)" + ) + cls.use_fortify_source = cls.add_bool_option( + "use-fortify-source", help="Compile tests with _DFORTIFY_SOURCE=2 (no effect on " "FreeBSD)" + ) + cls.use_softboundcets = cls.add_bool_option( + "use-softboundcets", + help="Compile tests with SoftBoundCETS (native only)", + only_add_for_targets=(CompilationTargets.NATIVE,), + ) + cls.use_effectivesan = cls.add_bool_option( + "use-effectivesan", + help="Compile tests with EffectiveSan (native only)", + only_add_for_targets=(CompilationTargets.NATIVE,), + ) @property def CC(self): # noqa: N802 @@ -136,8 +145,9 @@ def process(self): assert "-lsoftboundcets_rt" in self.default_ldflags # FIXME: add option to disable FORTIFY_SOURCE if self.build_type != BuildType.DEBUG: - self.warning("BODiagsuite contains undefined behaviour that might be optimized away unless you compile" - " at -O0.") + self.warning( + "BODiagsuite contains undefined behaviour that might be optimized away unless you compile" " at -O0." + ) self.ask_for_confirmation("Are you sure you want to continue?") super().process() @@ -187,10 +197,19 @@ def run_tests(self): extra_args.extend(["--junit-testsuite-name", testsuite_prefix]) if self.compiling_for_host(): - extra_args.extend(["--test-native", "--bmake-path", bmake, - "--jobs", str(self.config.make_jobs), - "--build-dir", self.build_dir]) + extra_args.extend( + [ + "--test-native", + "--bmake-path", + bmake, + "--jobs", + str(self.config.make_jobs), + "--build-dir", + self.build_dir, + ] + ) self.run_cmd(self.get_test_script_path("run_bodiagsuite.py"), *extra_args) else: - self.target_info.run_cheribsd_test_script("run_bodiagsuite.py", *extra_args, - mount_sourcedir=False, mount_builddir=True) + self.target_info.run_cheribsd_test_script( + "run_bodiagsuite.py", *extra_args, mount_sourcedir=False, mount_builddir=True + ) diff --git a/pycheribuild/projects/cross/boost.py b/pycheribuild/projects/cross/boost.py index f50db36db..ea48e2acd 100644 --- a/pycheribuild/projects/cross/boost.py +++ b/pycheribuild/projects/cross/boost.py @@ -39,5 +39,6 @@ def setup(self): included_libraries = [ "range", # used by kactivitymanagerd ] - self.add_cmake_options(BOOST_EXCLUDE_LIBRARIES=";".join(skipped_libraries), - BOOST_INCLUDE_LIBRARIES=";".join(included_libraries)) + self.add_cmake_options( + BOOST_EXCLUDE_LIBRARIES=";".join(skipped_libraries), BOOST_INCLUDE_LIBRARIES=";".join(included_libraries) + ) diff --git a/pycheribuild/projects/cross/cheri_exercises.py b/pycheribuild/projects/cross/cheri_exercises.py index c318d8bf5..111d1e1d0 100644 --- a/pycheribuild/projects/cross/cheri_exercises.py +++ b/pycheribuild/projects/cross/cheri_exercises.py @@ -47,8 +47,7 @@ class BuildCheriExercises(CrossCompileProject): target = "cheri-exercises" repository = GitRepository("https://github.com/CTSRD-CHERI/cheri-exercises.git") - supported_architectures = (CompilationTargets.CHERIBSD_RISCV_PURECAP, - CompilationTargets.CHERIBSD_MORELLO_PURECAP) + supported_architectures = (CompilationTargets.CHERIBSD_RISCV_PURECAP, CompilationTargets.CHERIBSD_MORELLO_PURECAP) default_install_dir = DefaultInstallDir.ROOTFS_OPTBASE path_in_rootfs = "/opt/cheri-exercises" @@ -58,82 +57,102 @@ def __init__(self, *args, **kwargs): def _compile_file(self, output: Path, *args, target_override: "Optional[CrossCompileTarget]" = None): assert isinstance(self.target_info, CheriBSDTargetInfo) - target_flags = self.target_info.get_essential_compiler_and_linker_flags(xtarget=target_override, - default_flags_only=True) + target_flags = self.target_info.get_essential_compiler_and_linker_flags( + xtarget=target_override, default_flags_only=True + ) warning_flags = ["-Wall", "-Wcheri"] - self.run_cmd([str(self.CC), *target_flags, *warning_flags, "-g", "-fuse-ld=lld", "-o", output, *args], - print_verbose_only=False) + self.run_cmd( + [str(self.CC), *target_flags, *warning_flags, "-g", "-fuse-ld=lld", "-o", output, *args], + print_verbose_only=False, + ) self.compiled_files.append(output) def _compile_for_cheri_and_non_cheri(self, output_name_prefix: str, *src_and_args): non_cheri_target = self.crosscompile_target.get_non_cheri_target() - self._compile_file(self.build_dir / (output_name_prefix + "-" + non_cheri_target.generic_arch_suffix), - *src_and_args, target_override=non_cheri_target) + self._compile_file( + self.build_dir / (output_name_prefix + "-" + non_cheri_target.generic_arch_suffix), + *src_and_args, + target_override=non_cheri_target, + ) self._compile_file(self.build_dir / (output_name_prefix + "-cheri"), *src_and_args) def compile(self, **kwargs): # Compile and run RISC-V and CHERI-RISC-V programs self._compile_for_cheri_and_non_cheri( - "print-pointer", self.source_dir / "src/exercises/compile-and-run/print-pointer.c") - self._compile_file(self.build_dir / "print-capability", - self.source_dir / "src/exercises/compile-and-run/print-capability.c") + "print-pointer", self.source_dir / "src/exercises/compile-and-run/print-pointer.c" + ) + self._compile_file( + self.build_dir / "print-capability", self.source_dir / "src/exercises/compile-and-run/print-capability.c" + ) # Exercise sundry inter-object buffer overflows (needs -G0) self._compile_for_cheri_and_non_cheri( "buffer-overflow-global", - self.source_dir / "src/exercises/buffer-overflow-global/buffer-overflow-global.c", "-G0") + self.source_dir / "src/exercises/buffer-overflow-global/buffer-overflow-global.c", + "-G0", + ) self._compile_for_cheri_and_non_cheri( - "buffer-overflow-heap", - self.source_dir / "src/exercises/buffer-overflow-heap/buffer-overflow-heap.c", "-G0") + "buffer-overflow-heap", self.source_dir / "src/exercises/buffer-overflow-heap/buffer-overflow-heap.c", "-G0" + ) self._compile_for_cheri_and_non_cheri( "buffer-overflow-stack", - self.source_dir / "src/exercises/buffer-overflow-stack/buffer-overflow-stack.c", "-G0") + self.source_dir / "src/exercises/buffer-overflow-stack/buffer-overflow-stack.c", + "-G0", + ) # Exercise a subobject buffer overflow self._compile_for_cheri_and_non_cheri( - "subobject-bounds", - self.source_dir / "src/exercises/subobject-bounds/buffer-overflow-subobject.c") + "subobject-bounds", self.source_dir / "src/exercises/subobject-bounds/buffer-overflow-subobject.c" + ) self._compile_file( self.build_dir / "subobject-bounds-cheri-subobject-safe", self.source_dir / "src/exercises/subobject-bounds/buffer-overflow-subobject.c", - "-Xclang", "-cheri-bounds=subobject-safe") # compile another version with subobject bounds + "-Xclang", + "-cheri-bounds=subobject-safe", + ) # compile another version with subobject bounds # Corrupt a data pointer by improperly manipulating it. self._compile_for_cheri_and_non_cheri( - "corrupt-pointer", self.source_dir / "src/exercises/cheri-tags/corrupt-pointer.c") + "corrupt-pointer", self.source_dir / "src/exercises/cheri-tags/corrupt-pointer.c" + ) # Corrupt a control-flow pointer using a subobject buffer overflow self._compile_for_cheri_and_non_cheri( - "buffer-overflow-fnptr", self.source_dir / "src/exercises/control-flow-pointer/buffer-overflow-fnptr.c") + "buffer-overflow-fnptr", self.source_dir / "src/exercises/control-flow-pointer/buffer-overflow-fnptr.c" + ) # Exercise integer-pointer type confusion bug self._compile_for_cheri_and_non_cheri( - "union-int-ptr", self.source_dir / "src/exercises/type-confusion/union-int-ptr.c") + "union-int-ptr", self.source_dir / "src/exercises/type-confusion/union-int-ptr.c" + ) # Demonstrate pointer injection self._compile_for_cheri_and_non_cheri( - "long-over-pipe", self.source_dir / "src/exercises/pointer-injection/long-over-pipe.c") + "long-over-pipe", self.source_dir / "src/exercises/pointer-injection/long-over-pipe.c" + ) self._compile_for_cheri_and_non_cheri( - "ptr-over-pipe", self.source_dir / "src/exercises/pointer-injection/ptr-over-pipe.c") + "ptr-over-pipe", self.source_dir / "src/exercises/pointer-injection/ptr-over-pipe.c" + ) # Demonstrate pointer revocation self._compile_file( self.build_dir / "pointer-revocation-temporal-control", - self.source_dir / "src/exercises/pointer-revocation/temporal-control.c") + self.source_dir / "src/exercises/pointer-revocation/temporal-control.c", + ) self._compile_file( self.build_dir / "pointer-revocation-temporal-control-caprevoke", self.source_dir / "src/exercises/pointer-revocation/temporal-control.c", - "-DCAPREVOKE") + "-DCAPREVOKE", + ) # Demonstrate various CheriABI properties self._compile_for_cheri_and_non_cheri( - "kern-read-over", self.source_dir / "src/exercises/cheriabi/kern-read-over.c") + "kern-read-over", self.source_dir / "src/exercises/cheriabi/kern-read-over.c" + ) - self._compile_for_cheri_and_non_cheri( - "perm-vmem", self.source_dir / "src/exercises/cheriabi/perm-vmem.c") + self._compile_for_cheri_and_non_cheri("perm-vmem", self.source_dir / "src/exercises/cheriabi/perm-vmem.c") - self._compile_for_cheri_and_non_cheri( - "print-more", self.source_dir / "src/exercises/cheriabi/print-more.c") + self._compile_for_cheri_and_non_cheri("print-more", self.source_dir / "src/exercises/cheriabi/print-more.c") # TODO: Demonstrate pointer revocation (however that needs caprevoke) @@ -147,8 +166,10 @@ def install(self, **kwargs): self.install_file(i, self.install_dir / i.name, print_verbose_only=False) # Also install them to the hybrid rootfs: hybrid_target = self.crosscompile_target.get_cheri_hybrid_target() - hybrid_install_dir = self.target_info.get_rootfs_project( - t=Project, xtarget=hybrid_target, caller=self).install_dir / self.path_in_rootfs[1:] + hybrid_install_dir = ( + self.target_info.get_rootfs_project(t=Project, xtarget=hybrid_target, caller=self).install_dir + / self.path_in_rootfs[1:] + ) self.makedirs(hybrid_install_dir) if self.with_clean: self.clean_directory(hybrid_install_dir, ensure_dir_exists=True) diff --git a/pycheribuild/projects/cross/cheribsd.py b/pycheribuild/projects/cross/cheribsd.py index e903354ef..cbe3ad2b3 100644 --- a/pycheribuild/projects/cross/cheribsd.py +++ b/pycheribuild/projects/cross/cheribsd.py @@ -68,6 +68,7 @@ def inner(config: CheriConfig, project: Project): if not isinstance(project, BuildCHERIBSD) and xtarget.is_hybrid_or_purecap_cheri(): raise ValueError(f"{project.target} should not build for CHERI architectures") return config.output_root / (prefix + project.build_configuration_suffix(xtarget)) + return ComputedDefaultValue(function=inner, as_string="$INSTALL_ROOT/" + prefix + "-") @@ -111,8 +112,19 @@ class CheriBSDConfig: Cheribuild configuration descriptor for a CheriBSD kernel configuration file """ - def __init__(self, kernconf: str, platforms: "set[ConfigPlatform]", kernel_abi=KernelABI.NOCHERI, default=False, - nocaprevoke=False, mfsroot=False, debug=False, benchmark=False, fuzzing=False, fett=False): + def __init__( + self, + kernconf: str, + platforms: "set[ConfigPlatform]", + kernel_abi=KernelABI.NOCHERI, + default=False, + nocaprevoke=False, + mfsroot=False, + debug=False, + benchmark=False, + fuzzing=False, + fett=False, + ): self.kernconf = kernconf self.platforms = platforms self.kernel_abi = kernel_abi @@ -127,12 +139,14 @@ def __init__(self, kernconf: str, platforms: "set[ConfigPlatform]", kernel_abi=K def __repr__(self) -> str: flags = [key for key, val in self.__dict__.items() if isinstance(val, bool) and val] return "CheriBSDConfig({kernconf} {platform}:{kernel_abi} [{flags}])".format( - kernconf=self.kernconf, platform=self.platforms, kernel_abi=self.kernel_abi.value, flags=" ".join(flags)) + kernconf=self.kernconf, platform=self.platforms, kernel_abi=self.kernel_abi.value, flags=" ".join(flags) + ) class KernelConfigFactory: - kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict([(k, None) for k in ( - "kabi_name", "nocaprevoke", "platform_name", "flags")]) + kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict( + [(k, None) for k in ("kabi_name", "nocaprevoke", "platform_name", "flags")] + ) separator: str = "_" platform_name_map: "dict[ConfigPlatform, Optional[str]]" = {} @@ -151,8 +165,16 @@ def get_platform_name(self, platforms: "set[ConfigPlatform]") -> Optional[str]: return self.platform_name_map[platform] assert False, "Should not be reached..." - def get_flag_names(self, platforms: "set[ConfigPlatform]", kernel_abi: KernelABI, mfsroot=False, fuzzing=False, - benchmark=False, default=False, nocaprevoke=False): + def get_flag_names( + self, + platforms: "set[ConfigPlatform]", + kernel_abi: KernelABI, + mfsroot=False, + fuzzing=False, + benchmark=False, + default=False, + nocaprevoke=False, + ): flags = [] if mfsroot: flags.append(f"MFS{self.separator}ROOT") @@ -186,8 +208,9 @@ def make_config(self, platforms: "set[ConfigPlatform]", kernel_abi, base_context class RISCVKernelConfigFactory(KernelConfigFactory): - kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict([(k, None) for k in ( - "kabi_name", "nocaprevoke", "platform_name", "flags")]) + kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict( + [(k, None) for k in ("kabi_name", "nocaprevoke", "platform_name", "flags")] + ) separator: str = "-" platform_name_map: "dict[ConfigPlatform, Optional[str]]" = { ConfigPlatform.QEMU: "QEMU", @@ -195,16 +218,27 @@ class RISCVKernelConfigFactory(KernelConfigFactory): ConfigPlatform.AWS: None, } - def get_flag_names(self, platforms: "set[ConfigPlatform]", kernel_abi: KernelABI, default=False, nocaprevoke=False, - mfsroot=False, debug=False, benchmark=False, fuzzing=False, fett=False): + def get_flag_names( + self, + platforms: "set[ConfigPlatform]", + kernel_abi: KernelABI, + default=False, + nocaprevoke=False, + mfsroot=False, + debug=False, + benchmark=False, + fuzzing=False, + fett=False, + ): if ConfigPlatform.GFE in platforms: # Suppress mfsroot flag as it is implied for GFE configurations mfsroot = False flags = [] if fett: flags.append("FETT") - flags += super().get_flag_names(platforms, kernel_abi, mfsroot=mfsroot, fuzzing=fuzzing, benchmark=benchmark, - nocaprevoke=nocaprevoke) + flags += super().get_flag_names( + platforms, kernel_abi, mfsroot=mfsroot, fuzzing=fuzzing, benchmark=benchmark, nocaprevoke=nocaprevoke + ) return flags def make_all(self) -> "list[CheriBSDConfig]": @@ -215,12 +249,14 @@ def make_all(self) -> "list[CheriBSDConfig]": configs.append(self.make_config({ConfigPlatform.QEMU}, kernel_abi, benchmark=True, default=True)) configs.append(self.make_config({ConfigPlatform.QEMU}, kernel_abi, mfsroot=True, default=True)) configs.append( - self.make_config({ConfigPlatform.QEMU}, kernel_abi, mfsroot=True, benchmark=True, default=True)) + self.make_config({ConfigPlatform.QEMU}, kernel_abi, mfsroot=True, benchmark=True, default=True) + ) # Generate FPGA kernels for kernel_abi in KernelABI: configs.append(self.make_config({ConfigPlatform.GFE}, kernel_abi, mfsroot=True, default=True)) configs.append( - self.make_config({ConfigPlatform.GFE}, kernel_abi, mfsroot=True, benchmark=True, default=True)) + self.make_config({ConfigPlatform.GFE}, kernel_abi, mfsroot=True, benchmark=True, default=True) + ) configs.append(self.make_config({ConfigPlatform.AWS}, kernel_abi, fett=True)) configs.append(self.make_config({ConfigPlatform.AWS}, kernel_abi, fett=True, benchmark=True)) @@ -231,20 +267,25 @@ def make_all(self) -> "list[CheriBSDConfig]": for kernel_abi in KernelABI: configs.append(self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, default=True)) configs.append( - self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, benchmark=True, default=True)) + self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, benchmark=True, default=True) + ) configs.append( - self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, mfsroot=True, default=True)) + self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, mfsroot=True, default=True) + ) configs.append( - self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, benchmark=True, mfsroot=True, - default=True)) + self.make_config( + {ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, benchmark=True, mfsroot=True, default=True + ) + ) configs.append(self.make_config({ConfigPlatform.GFE}, kernel_abi, nocaprevoke=True, mfsroot=True)) return configs class AArch64KernelConfigFactory(KernelConfigFactory): - kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict([(k, None) for k in ( - "platform_name", "kabi_name", "nocaprevoke", "flags")]) + kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict( + [(k, None) for k in ("platform_name", "kabi_name", "nocaprevoke", "flags")] + ) separator: str = "-" platform_name_map: "dict[ConfigPlatform, Optional[str]]" = { ConfigPlatform.QEMU: "GENERIC", @@ -264,22 +305,37 @@ def make_all(self) -> "list[CheriBSDConfig]": # Generate QEMU/FVP kernels for kernel_abi in KernelABI: configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True)) - configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, - benchmark=True)) - configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, - mfsroot=True)) + configs.append( + self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, benchmark=True) + ) + configs.append( + self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, mfsroot=True) + ) # Caprevoke kernels for kernel_abi in KernelABI: - configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, - nocaprevoke=True)) - configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, - nocaprevoke=True, benchmark=True)) + configs.append( + self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, nocaprevoke=True) + ) + configs.append( + self.make_config( + {ConfigPlatform.QEMU, ConfigPlatform.FVP}, + kernel_abi, + default=True, + nocaprevoke=True, + benchmark=True, + ) + ) return configs -def filter_kernel_configs(configs: "list[CheriBSDConfig]", *, platform: "Optional[ConfigPlatform]", - kernel_abi: Optional[KernelABI], **filter_kwargs) -> "typing.Sequence[CheriBSDConfig]": +def filter_kernel_configs( + configs: "list[CheriBSDConfig]", + *, + platform: "Optional[ConfigPlatform]", + kernel_abi: Optional[KernelABI], + **filter_kwargs, +) -> "typing.Sequence[CheriBSDConfig]": """ Helper function to filter kernel configuration lists. Keyword filter arguments are mapped to CheriBSDConfig properties. @@ -346,17 +402,22 @@ def get_default(cls, xtarget, platform: ConfigPlatform, kernel_abi: KernelABI, * It is a fatal failure if 0 or more than one configurations exist. """ configs = cls.get_configs(xtarget, platform=platform, kernel_abi=kernel_abi, default=True, **filter_kwargs) - assert len(configs) != 0, f"No matching default kernel configuration for xtarget={xtarget}, " \ - f"platform={platform.name}, kernel_abi={kernel_abi.name}, " \ - f"filter_kwargs={filter_kwargs}" - assert len(configs) == 1, f"Too many default kernel configurations {configs} for xtarget={xtarget}, " \ - f"platform={platform.name}, kernel_abi={kernel_abi.name}, " \ - f"filter_kwargs={filter_kwargs}" + assert len(configs) != 0, ( + f"No matching default kernel configuration for xtarget={xtarget}, " + f"platform={platform.name}, kernel_abi={kernel_abi.name}, " + f"filter_kwargs={filter_kwargs}" + ) + assert len(configs) == 1, ( + f"Too many default kernel configurations {configs} for xtarget={xtarget}, " + f"platform={platform.name}, kernel_abi={kernel_abi.name}, " + f"filter_kwargs={filter_kwargs}" + ) return configs[0] @classmethod - def get_configs(cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_abi: "Optional[KernelABI]", - **filter_kwargs): + def get_configs( + cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_abi: "Optional[KernelABI]", **filter_kwargs + ): """ Return all configurations for a combination of target, platform and kernel ABI. This filters out all the specialized configuration flags defaulting all of them @@ -371,14 +432,16 @@ def get_configs(cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_ab return cls.get_all_configs(xtarget, platform=platform, kernel_abi=kernel_abi, **filter_kwargs) @classmethod - def get_all_configs(cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_abi: "Optional[KernelABI]", - **filter_kwargs): + def get_all_configs( + cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_abi: "Optional[KernelABI]", **filter_kwargs + ): """ Return all available configurations for a combination of target, group and kernel ABI filtered using kwargs. """ - return filter_kernel_configs(cls.get_target_configs(xtarget), platform=platform, kernel_abi=kernel_abi, - **filter_kwargs) + return filter_kernel_configs( + cls.get_target_configs(xtarget), platform=platform, kernel_abi=kernel_abi, **filter_kwargs + ) class BuildFreeBSDBase(Project): @@ -425,20 +488,27 @@ def crossbuild(self) -> bool: def setup_config_options(cls, kernel_only_target=False, **kwargs) -> None: super().setup_config_options(**kwargs) cls.extra_make_args = cls.add_list_option( - "build-options", default=cls.default_extra_make_options, metavar="OPTIONS", + "build-options", + default=cls.default_extra_make_options, + metavar="OPTIONS", help="Additional make options to be passed to make when building FreeBSD/CheriBSD. See `man src.conf` " - "for more info.", show_help=True) - cls.debug_kernel = cls.add_bool_option("debug-kernel", help="Build the kernel with -O0 and verbose boot output", - show_help=False) + "for more info.", + show_help=True, + ) + cls.debug_kernel = cls.add_bool_option( + "debug-kernel", help="Build the kernel with -O0 and verbose boot output", show_help=False + ) if kernel_only_target: cls.minimal = False cls.build_tests = False return # The remaining options only affect the userspace build if "minimal" not in cls.__dict__: - cls.minimal = cls.add_bool_option("minimal", show_help=False, - help="Don't build all of FreeBSD, just what is needed for running most " - "CHERI tests/benchmarks") + cls.minimal = cls.add_bool_option( + "minimal", + show_help=False, + help="Don't build all of FreeBSD, just what is needed for running most " "CHERI tests/benchmarks", + ) def check_system_dependencies(self) -> None: super().check_system_dependencies() @@ -455,8 +525,10 @@ def setup(self) -> None: if self.crossbuild: # Use the script that I added for building on Linux/MacOS: - self.make_args.set_command(self.source_dir / "tools/build/make.py", - early_args=["--bootstrap-toolchain"] if self.use_bootstrapped_toolchain else []) + self.make_args.set_command( + self.source_dir / "tools/build/make.py", + early_args=["--bootstrap-toolchain"] if self.use_bootstrapped_toolchain else [], + ) self.make_args.set( DB_FROM_SRC=True, # don't use the system passwd file @@ -468,8 +540,19 @@ def setup(self) -> None: self.make_args.set_with_options(CLEAN=False) if self.minimal: - self.make_args.set_with_options(MAN=False, KERBEROS=False, SVN=False, SVNLITE=False, MAIL=False, ZFS=False, - SENDMAIL=False, EXAMPLES=False, LOCALES=False, NLS=False, CDDL=False) + self.make_args.set_with_options( + MAN=False, + KERBEROS=False, + SVN=False, + SVNLITE=False, + MAIL=False, + ZFS=False, + SENDMAIL=False, + EXAMPLES=False, + LOCALES=False, + NLS=False, + CDDL=False, + ) # tests off by default because they take a long time and often seems to break # the creation of disk-image (METALOG is invalid) @@ -489,9 +572,15 @@ def setup(self) -> None: for option in self.extra_make_args: if not self.crosscompile_target.is_hybrid_or_purecap_cheri() and "CHERI_" in option: - self.warning("Should not be adding CHERI specific make option", option, "for", self.target, - " -- consider setting separate", self.get_config_option_name("extra_make_args"), - "in the config file.") + self.warning( + "Should not be adding CHERI specific make option", + option, + "for", + self.target, + " -- consider setting separate", + self.get_config_option_name("extra_make_args"), + "in the config file.", + ) if "=" in option: key, value = option.split("=", 1) args = {key: value} @@ -548,20 +637,32 @@ def get_rootfs_dir(cls, caller, cross_target: "Optional[CrossCompileTarget]" = N return cls.get_install_dir(caller, cross_target) @classmethod - def setup_config_options(cls, bootstrap_toolchain=False, use_upstream_llvm: Optional[bool] = None, - debug_info_by_default=True, kernel_only_target=False, **kwargs) -> None: + def setup_config_options( + cls, + bootstrap_toolchain=False, + use_upstream_llvm: Optional[bool] = None, + debug_info_by_default=True, + kernel_only_target=False, + **kwargs, + ) -> None: super().setup_config_options(kernel_only_target=kernel_only_target, **kwargs) if cls._xtarget: # KERNCONF always depends on the target, so we don't inherit this config option. The only exception is # the global --kernel-config option that is provided for convenience and backwards compat. cls.kernel_config = cls.add_config_option( - "kernel-config", metavar="CONFIG", show_help=True, extra_fallback_config_names=["kernel-config"], + "kernel-config", + metavar="CONFIG", + show_help=True, + extra_fallback_config_names=["kernel-config"], default=ComputedDefaultValue( - function=lambda _, p: - p.default_kernel_config() if p.has_default_buildkernel_kernel_config else None, - as_string="target-dependent, usually GENERIC"), + function=lambda _, p: p.default_kernel_config() + if p.has_default_buildkernel_kernel_config + else None, + as_string="target-dependent, usually GENERIC", + ), use_default_fallback_config_names=False, # - help="The kernel configuration to use for `make buildkernel`") + help="The kernel configuration to use for `make buildkernel`", + ) if cls._xtarget is not None and cls._xtarget.is_hybrid_or_purecap_cheri(): # When targeting CHERI we have to use CHERI LLVM @@ -579,25 +680,40 @@ def setup_config_options(cls, bootstrap_toolchain=False, use_upstream_llvm: Opti else: # Prefer using system clang for FreeBSD builds rather than a self-built snapshot of LLVM since that might # have new warnings that break the -Werror build. - cls.build_toolchain = typing.cast(CompilerType, cls.add_config_option( - "toolchain", kind=CompilerType, default=CompilerType.DEFAULT_COMPILER, - enum_choice_strings=[t.value for t in CompilerType], - help="The toolchain to use for building FreeBSD. When set to 'custom', the 'toolchain-path' option " - "must also be set")) + cls.build_toolchain = typing.cast( + CompilerType, + cls.add_config_option( + "toolchain", + kind=CompilerType, + default=CompilerType.DEFAULT_COMPILER, + enum_choice_strings=[t.value for t in CompilerType], + help="The toolchain to use for building FreeBSD. When set to 'custom', the 'toolchain-path' option " + "must also be set", + ), + ) cls._cross_toolchain_root = cls.add_optional_path_option( - "toolchain-path", help="Path to the cross toolchain tools") + "toolchain-path", help="Path to the cross toolchain tools" + ) # override in CheriBSD - cls.linker_for_world = cls.add_config_option("linker-for-world", default="lld", choices=["bfd", "lld"], - help="The linker to use for world") - cls.linker_for_kernel = cls.add_config_option("linker-for-kernel", default="lld", choices=["bfd", "lld"], - help="The linker to use for the kernel") - - cls.with_debug_info = cls.add_bool_option("debug-info", default=debug_info_by_default, show_help=True, - help="pass make flags for building with debug info") - cls.with_debug_files = cls.add_bool_option("debug-files", default=True, - help="Use split DWARF debug files if building with debug info") + cls.linker_for_world = cls.add_config_option( + "linker-for-world", default="lld", choices=["bfd", "lld"], help="The linker to use for world" + ) + cls.linker_for_kernel = cls.add_config_option( + "linker-for-kernel", default="lld", choices=["bfd", "lld"], help="The linker to use for the kernel" + ) + + cls.with_debug_info = cls.add_bool_option( + "debug-info", + default=debug_info_by_default, + show_help=True, + help="pass make flags for building with debug info", + ) + cls.with_debug_files = cls.add_bool_option( + "debug-files", default=True, help="Use split DWARF debug files if building with debug info" + ) cls.fast_rebuild = cls.add_bool_option( - "fast", help="Skip some (usually) unnecessary build steps to speed up rebuilds") + "fast", help="Skip some (usually) unnecessary build steps to speed up rebuilds" + ) if kernel_only_target: cls.build_lib32 = False @@ -605,26 +721,39 @@ def setup_config_options(cls, bootstrap_toolchain=False, use_upstream_llvm: Opti cls.with_manpages = False return # The remaining options only affect the userspace build - subdir_default = ComputedDefaultValue(function=lambda config, proj: config.freebsd_subdir, - as_string="the value of the global --freebsd-subdir options") + subdir_default = ComputedDefaultValue( + function=lambda config, proj: config.freebsd_subdir, + as_string="the value of the global --freebsd-subdir options", + ) cls.explicit_subdirs_only = cls.add_list_option( - "subdir", metavar="SUBDIRS", show_help=True, default=subdir_default, + "subdir", + metavar="SUBDIRS", + show_help=True, + default=subdir_default, help="Only build subdirs SUBDIRS instead of the full tree. Useful for quickly rebuilding individual" - " programs/libraries. If more than one dir is passed, they will be processed in order. Note: This" - " will break if not all dependencies have been built.") + " programs/libraries. If more than one dir is passed, they will be processed in order. Note: This" + " will break if not all dependencies have been built.", + ) cls.keep_old_rootfs = cls.add_bool_option( - "keep-old-rootfs", help="Don't remove the whole old rootfs directory. This can speed up installing but" - " may cause strange errors so is off by default.") - cls.with_manpages = cls.add_bool_option("with-manpages", help="Also install manpages. This is off by default" - " since they can just be read from the host.") - cls.build_drm_kmod = cls.add_bool_option("build-drm-kmod", help="Also build drm-kmod during buildkernel", - show_help=False) + "keep-old-rootfs", + help="Don't remove the whole old rootfs directory. This can speed up installing but" + " may cause strange errors so is off by default.", + ) + cls.with_manpages = cls.add_bool_option( + "with-manpages", + help="Also install manpages. This is off by default" " since they can just be read from the host.", + ) + cls.build_drm_kmod = cls.add_bool_option( + "build-drm-kmod", help="Also build drm-kmod during buildkernel", show_help=False + ) if cls._xtarget is None or not cls._xtarget.cpu_architecture.is_32bit(): cls.build_lib32 = cls.add_bool_option( - "build-lib32", default=False, - help="Build the 32-bit compatibility userspace libraries (if supported for the current architecture)") + "build-lib32", + default=False, + help="Build the 32-bit compatibility userspace libraries (if supported for the current architecture)", + ) else: # XXX: this is not correct if we were to support a CHERI-64 userspace assert not cls._xtarget.is_hybrid_or_purecap_cheri() @@ -743,20 +872,31 @@ def _try_find_compatible_system_clang(self) -> "tuple[Optional[Path], Optional[s if cc_info.is_clang and not cc_info.is_apple_clang and cc_info.version >= min_version: compiler_path = cc_info.path else: - return (None, "Could not find a system installation of clang.", - "Please install a recent upstream clang or use the 'custom' or 'upstream-llvm' toolchain " - "option.") + return ( + None, + "Could not find a system installation of clang.", + "Please install a recent upstream clang or use the 'custom' or 'upstream-llvm' toolchain " + "option.", + ) self.info("Checking if", compiler_path, "can be used to build FreeBSD...") cc_info = self.get_compiler_info(compiler_path) if cc_info.is_apple_clang: - return (None, "Cannot build FreeBSD with Apple clang.", - "Please install a recent upstream clang (e.g. with homebrew) or use the 'custom' " - "or 'upstream-llvm' toolchain option.") + return ( + None, + "Cannot build FreeBSD with Apple clang.", + "Please install a recent upstream clang (e.g. with homebrew) or use the 'custom' " + "or 'upstream-llvm' toolchain option.", + ) if cc_info.version < min_version: - return (None, "Cannot build FreeBSD with Clang older than " + ".".join(map(str, min_version)) + - ". Found clang = " + str(compiler_path), - "Please install a recent upstream clang (e.g. with homebrew) or use the 'custom' " - "or 'upstream-llvm' toolchain option.") + return ( + None, + "Cannot build FreeBSD with Clang older than " + + ".".join(map(str, min_version)) + + ". Found clang = " + + str(compiler_path), + "Please install a recent upstream clang (e.g. with homebrew) or use the 'custom' " + "or 'upstream-llvm' toolchain option.", + ) # Note: FreeBSD installs shell script wrappers for clang, so we can't just use # Path(compiler_path).resolve().parent.parent since that will try to use /usr/local/bin/clang. Instead # we print the resource dir (/lib/clang/) and go up three levels from there. @@ -785,8 +925,7 @@ def setup(self) -> None: def build_toolchain_root_dir(self) -> "Optional[Path]": if self.build_toolchain == CompilerType.BOOTSTRAPPED: return self.objdir / "tmp/usr" - elif self.build_toolchain in (CompilerType.UPSTREAM_LLVM, CompilerType.CHERI_LLVM, - CompilerType.MORELLO_LLVM): + elif self.build_toolchain in (CompilerType.UPSTREAM_LLVM, CompilerType.CHERI_LLVM, CompilerType.MORELLO_LLVM): return BuildLLVMMonoRepoBase.get_install_dir_for_type(self, self.build_toolchain) elif self.build_toolchain == CompilerType.SYSTEM_LLVM: system_clang_root, errmsg, fixit = self._try_find_compatible_system_clang() @@ -795,8 +934,11 @@ def build_toolchain_root_dir(self) -> "Optional[Path]": return system_clang_root elif self.build_toolchain == CompilerType.CUSTOM: if self._cross_toolchain_root is None: - self.fatal("Requested custom toolchain but", self.get_config_option_name("_cross_toolchain_root"), - "is not set.") + self.fatal( + "Requested custom toolchain but", + self.get_config_option_name("_cross_toolchain_root"), + "is not set.", + ) return self._cross_toolchain_root else: assert self.build_toolchain == CompilerType.DEFAULT_COMPILER @@ -814,10 +956,14 @@ def _setup_cross_toolchain_config(self) -> None: self.cross_toolchain_config.set_with_options( # TODO: should we have an option to include a compiler in the target system? - GCC=False, CLANG=False, LLD=False, # Take a long time and not needed in the target system + GCC=False, + CLANG=False, + LLD=False, # Take a long time and not needed in the target system LLDB=False, # may be useful but means we need to build LLVM # Bootstrap compiler/ linker are not needed: - GCC_BOOTSTRAP=False, CLANG_BOOTSTRAP=False, LLD_BOOTSTRAP=False, + GCC_BOOTSTRAP=False, + CLANG_BOOTSTRAP=False, + LLD_BOOTSTRAP=False, ) if not self.build_lib32: # takes a long time and usually not needed. @@ -838,7 +984,9 @@ def _setup_cross_toolchain_config(self) -> None: if not xccinfo.is_clang: self.ask_for_confirmation("Cross compiler is not clang, are you sure you want to continue?") self.cross_toolchain_config.set_env( - XCC=self.CC, XCXX=self.CXX, XCPP=self.CPP, + XCC=self.CC, + XCXX=self.CXX, + XCPP=self.CPP, X_COMPILER_TYPE=xccinfo.compiler, # This is needed otherwise the build assumes it should build with $CC ) if not self.use_llvm_binutils: @@ -898,8 +1046,10 @@ def _setup_arch_specific_options(self) -> str: # TODO: should probably set that inside CheriBSD makefiles instead if self.target_info.is_cheribsd(): target_flags += " -mcpu=beri" - self.cross_toolchain_config.set_with_options(RESCUE=False, # Won't compile with CHERI clang yet - BOOT=False) # bootloaders won't link with LLD yet + self.cross_toolchain_config.set_with_options( + RESCUE=False, # Won't compile with CHERI clang yet + BOOT=False, + ) # bootloaders won't link with LLD yet elif self.compiling_for_riscv(include_purecap=True): target_flags = "" else: @@ -965,6 +1115,7 @@ def kernel_make_args_for_config(self, kernconfs: "list[str]", extra_make_args) - kernel_options.remove_var("X" + binutil_name) if self.build_drm_kmod: from .drm_kmod import BuildDrmKMod + drm_kmod = BuildDrmKMod.get_instance(self) kernel_options.set(LOCAL_MODULES=drm_kmod.source_dir.name, LOCAL_MODULES_DIR=drm_kmod.source_dir.parent) kernel_options.set(KERNCONF=" ".join(kernconfs)) @@ -985,7 +1136,8 @@ def clean(self) -> ThreadJoiner: cleaning_kerneldir = True if kernel_dir.exists() and self.build_dir.exists(): assert not os.path.relpath(str(kernel_dir.resolve()), str(self.build_dir.resolve())).startswith( - ".."), builddir + ".." + ), builddir else: self.warning("Do not know the full path to the kernel build directory, will clean the whole tree!") if self.crossbuild: @@ -1001,20 +1153,28 @@ def _list_kernel_configs(self) -> None: blacklist = ["NOTES", "LINT", "DEFAULTS"] self.info("Valid kernel configuration files for --" + self.target + "/kernel-config:") for conf in configs: - if (conf.name in blacklist or conf.name.startswith("std.") or conf.name.endswith(".hints") or - conf.name.endswith("~")): + if ( + conf.name in blacklist + or conf.name.startswith("std.") + or conf.name.endswith(".hints") + or conf.name.endswith("~") + ): continue self.info(conf.name) - def _buildkernel(self, kernconfs: "list[str]", mfs_root_image: "Optional[Path]" = None, extra_make_args=None, - ignore_skip_kernel=False) -> None: + def _buildkernel( + self, + kernconfs: "list[str]", + mfs_root_image: "Optional[Path]" = None, + extra_make_args=None, + ignore_skip_kernel=False, + ) -> None: # Check that --skip-kernel is respected. However, we ignore it for the cheribsd-mfs-root-kernel targets # since those targets only build a kernel. assert not self.config.skip_kernel or ignore_skip_kernel, "--skip-kernel set but building kernel" kernel_make_args = self.kernel_make_args_for_config(kernconfs, extra_make_args) if not self.use_bootstrapped_toolchain and not self.CC.exists(): - self.fatal("Requested build of kernel with external toolchain, but", self.CC, - "doesn't exist!") + self.fatal("Requested build of kernel with external toolchain, but", self.CC, "doesn't exist!") if self.debug_kernel: if any(x.endswith(("_BENCHMARK", "-NODEBUG")) for x in kernconfs): if not self.query_yes_no("Trying to build BENCHMARK kernel without optimization. Continue?"): @@ -1035,11 +1195,15 @@ def _buildkernel(self, kernconfs: "list[str]", mfs_root_image: "Optional[Path]" self.run_make("kernel-toolchain", options=kernel_toolchain_opts) self.kernel_toolchain_exists = True self.info("Building kernels for configs:", " ".join(kernconfs)) - self.run_make("buildkernel", options=kernel_make_args, - compilation_db_name="compile_commands_" + " ".join(kernconfs).replace(" ", "_") + ".json") + self.run_make( + "buildkernel", + options=kernel_make_args, + compilation_db_name="compile_commands_" + " ".join(kernconfs).replace(" ", "_") + ".json", + ) - def _installkernel(self, kernconfs: "list[str]", *, install_dir: Path, extra_make_args=None, - ignore_skip_kernel=False) -> None: + def _installkernel( + self, kernconfs: "list[str]", *, install_dir: Path, extra_make_args=None, ignore_skip_kernel=False + ) -> None: # Check that --skip-kernel is respected. However, we ignore it for the cheribsd-mfs-root-kernel targets # since those targets only build a kernel. assert not self.config.skip_kernel or ignore_skip_kernel, "--skip-kernel set but building kernel" @@ -1057,8 +1221,15 @@ def _installkernel(self, kernconfs: "list[str]", *, install_dir: Path, extra_mak self.delete_file(install_dir / "METALOG.kernel") # Ensure that METALOG does not contain stale values. self.run_make("installkernel", options=install_kernel_args, parallel=False) - def _copykernel(self, kernconfs: "list[str]", rootfs_dir: Path, dest_dir: Path, *, - dest_kernel_suffix: str = "", dest_all_extra: bool = False) -> None: + def _copykernel( + self, + kernconfs: "list[str]", + rootfs_dir: Path, + dest_dir: Path, + *, + dest_kernel_suffix: str = "", + dest_all_extra: bool = False, + ) -> None: """ Copy kernels from the rootfs to the given directory (for MFS-ROOT kernels and Jenkins) """ @@ -1124,10 +1295,24 @@ def _remove_old_rootfs(self) -> None: if os.getuid() == 0: # if we installed as root remove the schg flag from files before cleaning (otherwise rm will fail) self._remove_schg_flag( - "lib/libc.so.7", "lib/libcrypt.so.5", "lib/libthr.so.3", - "libexec/ld-elf.so.1", "sbin/init", "usr/bin/chpass", "usr/bin/chsh", "usr/bin/ypchpass", - "usr/bin/ypchfn", "usr/bin/ypchsh", "usr/bin/login", "usr/bin/opieinfo", "usr/bin/opiepasswd", - "usr/bin/passwd", "usr/bin/yppasswd", "usr/bin/su", "usr/bin/crontab", "usr/lib/librt.so.1", + "lib/libc.so.7", + "lib/libcrypt.so.5", + "lib/libthr.so.3", + "libexec/ld-elf.so.1", + "sbin/init", + "usr/bin/chpass", + "usr/bin/chsh", + "usr/bin/ypchpass", + "usr/bin/ypchfn", + "usr/bin/ypchsh", + "usr/bin/login", + "usr/bin/opieinfo", + "usr/bin/opiepasswd", + "usr/bin/passwd", + "usr/bin/yppasswd", + "usr/bin/su", + "usr/bin/crontab", + "usr/lib/librt.so.1", "var/empty", ) # We keep 3rd-party programs (anything installed in /usr/local + /opt), but delete everything else prior @@ -1163,19 +1348,30 @@ def _query_make_var(self, args, var) -> Optional[Path]: return None query_args = args.copy() query_args.set_command(bmake_binary) - bw_flags = [*query_args.all_commandline_args(self.config), - "BUILD_WITH_STRICT_TMPPATH=0", - "-f", self.source_dir / "Makefile.inc1", - "-m", self.source_dir / "share/mk", - "showconfig", - "-D_NO_INCLUDE_COMPILERMK", # avoid calling ${CC} --version - "-V", var] + bw_flags = [ + *query_args.all_commandline_args(self.config), + "BUILD_WITH_STRICT_TMPPATH=0", + "-f", + self.source_dir / "Makefile.inc1", + "-m", + self.source_dir / "share/mk", + "showconfig", + "-D_NO_INCLUDE_COMPILERMK", # avoid calling ${CC} --version + "-V", + var, + ] if not self.source_dir.exists(): assert self.config.pretend, "This should only happen when running in a test environment" return None # https://github.com/freebsd/freebsd/commit/1edb3ba87657e28b017dffbdc3d0b3a32999d933 - cmd = self.run_cmd([bmake_binary, *bw_flags], env=args.env_vars, cwd=self.source_dir, - run_in_pretend_mode=True, capture_output=True, print_verbose_only=True) + cmd = self.run_cmd( + [bmake_binary, *bw_flags], + env=args.env_vars, + cwd=self.source_dir, + run_in_pretend_mode=True, + capture_output=True, + print_verbose_only=True, + ) lines = cmd.stdout.strip().split(b"\n") last_line = lines[-1].decode("utf-8").strip() if last_line.startswith("/") and cmd.returncode == 0: @@ -1193,8 +1389,9 @@ def objdir(self) -> Path: if result is None: result = Path() if self.realpath(result) == self.realpath(self.source_dir): - self.warning("bmake claims the build dir for", self.target, "is the source dir, assuming", - self.build_dir, "instead.") + self.warning( + "bmake claims the build dir for", self.target, "is the source dir, assuming", self.build_dir, "instead." + ) return self.build_dir if not result or result == Path(): # just clean the whole directory instead @@ -1239,8 +1436,10 @@ def install(self, *, kernconfs: "Optional[list[str]]" = None, sysroot_only=False # support building old versions of cheribsd before _compiler-metadata was renamed to _build-metadata self.run_make("_compiler-metadata", options=install_world_args) except subprocess.CalledProcessError: - self.warning("Failed to run either target _compiler-metadata or " - "_build_metadata, build system has changed!") + self.warning( + "Failed to run either target _compiler-metadata or " + "_build_metadata, build system has changed!" + ) # By default also create a sysroot when installing world installsysroot_args = install_world_args.copy() if self.has_installsysroot_target: @@ -1271,18 +1470,19 @@ def install(self, *, kernconfs: "Optional[list[str]]" = None, sysroot_only=False def rewrite_passwd(old): new = [] for line in old: - fields = line.split(':') + fields = line.split(":") if len(fields) == 10 and fields[0] == "toor" and fields[1] == "*" and not fields[9]: fields[1] = "" fields[9] = "/bin/sh" - line = ':'.join(fields) + line = ":".join(fields) new.append(line) return new pwd_mkdb_cmd = self.objdir / "tmp/legacy/usr/bin/pwd_mkdb" self.rewrite_file(self.destdir / "etc/master.passwd", rewrite_passwd) - self.run_cmd([pwd_mkdb_cmd, "-p", "-d", self.install_dir / "etc", - self.install_dir / "etc/master.passwd"]) + self.run_cmd( + [pwd_mkdb_cmd, "-p", "-d", self.install_dir / "etc", self.install_dir / "etc/master.passwd"] + ) assert not sysroot_only, "Should not end up here" if self.config.skip_kernel: @@ -1298,8 +1498,12 @@ def rewrite_passwd(old): self._copykernel(kernconfs=kernconfs, rootfs_dir=self.install_dir, dest_dir=self.config.output_root) def add_cross_build_options(self) -> None: - self.make_args.set_env(CC=self.host_CC, CXX=self.host_CXX, CPP=self.host_CPP, - STRIPBIN=shutil.which("strip") or shutil.which("llvm-strip") or "strip") + self.make_args.set_env( + CC=self.host_CC, + CXX=self.host_CXX, + CPP=self.host_CPP, + STRIPBIN=shutil.which("strip") or shutil.which("llvm-strip") or "strip", + ) if self.use_bootstrapped_toolchain: assert "XCC" not in self.make_args.env_vars # We have to provide the default X* values so that Makefile.inc1 does not disable MK_CLANG_BOOTSTRAP and @@ -1342,22 +1546,39 @@ def process(self) -> None: args = self.buildworld_args args.remove_flag("-s") # buildenv should not be silent if "bash" in os.getenv("SHELL", ""): - args.set(BUILDENV_SHELL="env -u PROMPT_COMMAND 'PS1=" + self.target + "-buildenv:\\w> ' " + - shutil.which("bash") + " --norc --noprofile") + args.set( + BUILDENV_SHELL="env -u PROMPT_COMMAND 'PS1=" + + self.target + + "-buildenv:\\w> ' " + + shutil.which("bash") + + " --norc --noprofile" + ) else: args.set(BUILDENV_SHELL="/bin/sh") buildenv_target = "buildenv" if self.config.libcompat_buildenv and self.libcompat_name(): buildenv_target = self.libcompat_name() + "buildenv" - self.run_cmd([self.make_args.command, *args.all_commandline_args(self.config), buildenv_target], - env=args.env_vars, cwd=self.source_dir) + self.run_cmd( + [self.make_args.command, *args.all_commandline_args(self.config), buildenv_target], + env=args.env_vars, + cwd=self.source_dir, + ) else: super().process() - def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_clean=None, skip_install=None, - install_to_internal_sysroot=True, libcompat_only=False, noncheri_only=False) -> None: + def build_and_install_subdir( + self, + make_args, + subdir, + skip_build=False, + skip_clean=None, + skip_install=None, + install_to_internal_sysroot=True, + libcompat_only=False, + noncheri_only=False, + ) -> None: is_lib = subdir.startswith("lib/") or "/lib/" in subdir or subdir.endswith("/lib") - make_in_subdir = "make -C \"" + subdir + "\" " + make_in_subdir = 'make -C "' + subdir + '" ' if skip_clean is None: skip_clean = not self.with_clean if skip_install is None: @@ -1367,10 +1588,10 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle install_to_sysroot_cmd = "" # We have to override INSTALL so that the sysroot installations don't end up in METALOG # This happens after https://github.com/freebsd/freebsd/commit/5496ab2ac950813edbd55d73c967184e033bea2f - install_nometalog_cmd = "INSTALL=\"install -N " + str(self.source_dir / "etc") + " -U\" METALOG=/dev/null" + install_nometalog_cmd = 'INSTALL="install -N ' + str(self.source_dir / "etc") + ' -U" METALOG=/dev/null' if is_lib and install_to_internal_sysroot: # Due to all the bmake + shell escaping I need 4 dollars here to get it to expand SYSROOT - sysroot_var = "\"$$$${SYSROOT}\"" + sysroot_var = '"$$$${SYSROOT}"' install_to_sysroot_cmd = ( f"if [ -n {sysroot_var} ]; then" f" {make_in_subdir} install {install_nometalog_cmd} MK_TESTS=no DESTDIR={sysroot_var}; " @@ -1380,7 +1601,7 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle if install_to_sysroot_cmd: install_cmd = install_to_sysroot_cmd else: - install_cmd = "echo \" Skipping make install\"" + install_cmd = 'echo " Skipping make install"' else: # if we are building a library also install to the sysroot so that other targets afterwards use the # updated static lib @@ -1391,11 +1612,14 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle # Note that installconfig doesn't fail if there is no configuration in a subdirectory. install_cmd = install_to_sysroot_cmd + make_in_subdir + "install installconfig " + install_nometalog_cmd colour_diags = "export CLANG_FORCE_COLOR_DIAGNOSTICS=always; " if self.config.clang_colour_diags else "" - build_cmd = "{colour_diags} {clean} && {build} && {install} && echo \" Done.\"".format( - build=make_in_subdir + "all " + self.commandline_to_str( - self.jflag) if not skip_build else "echo \" Skipping make all\"", - clean=make_in_subdir + "clean" if not skip_clean else "echo \" Skipping make clean\"", - install=install_cmd, colour_diags=colour_diags) + build_cmd = '{colour_diags} {clean} && {build} && {install} && echo " Done."'.format( + build=make_in_subdir + "all " + self.commandline_to_str(self.jflag) + if not skip_build + else 'echo " Skipping make all"', + clean=make_in_subdir + "clean" if not skip_clean else 'echo " Skipping make clean"', + install=install_cmd, + colour_diags=colour_diags, + ) make_args.set(BUILDENV_SHELL="sh -ex -c '" + build_cmd + "' || exit 1") # If --libcompat-buildenv was passed skip the MIPS lib has_libcompat = self.crosscompile_target.is_hybrid_or_purecap_cheri() and is_lib # TODO: handle lib32 @@ -1403,8 +1627,11 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle self.info("Skipping default ABI build of", subdir, "since --libcompat-buildenv was passed.") else: self.info("Building", subdir, "using buildenv target") - self.run_cmd([self.make_args.command, *make_args.all_commandline_args(self.config), "buildenv"], - env=make_args.env_vars, cwd=self.source_dir) + self.run_cmd( + [self.make_args.command, *make_args.all_commandline_args(self.config), "buildenv"], + env=make_args.env_vars, + cwd=self.source_dir, + ) # If we are building a library, we want to build both the CHERI and the mips version (unless the # user explicitly specified --libcompat-buildenv) if has_libcompat and not noncheri_only and self.libcompat_name(): @@ -1413,10 +1640,13 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle extra_flags = ["MK_TESTS=no"] # don't build tests since they will overwrite the non-compat ones self.run_cmd( [self.make_args.command, *make_args.all_commandline_args(self.config), *extra_flags, compat_target], - env=make_args.env_vars, cwd=self.source_dir) + env=make_args.env_vars, + cwd=self.source_dir, + ) - def _get_kernel_rootfs_install_path(self, kernconf: "Optional[str]", rootfs: Path, - default_kernconf: "Optional[str]") -> Path: + def _get_kernel_rootfs_install_path( + self, kernconf: "Optional[str]", rootfs: Path, default_kernconf: "Optional[str]" + ) -> Path: if kernconf is None or kernconf == default_kernconf: kerndir = "kernel" else: @@ -1473,17 +1703,20 @@ class BuildFreeBSDWithDefaultOptions(BuildFreeBSD): def clean(self) -> ThreadJoiner: # Bootstrapping LLVM takes forever with FreeBSD makefiles if self.use_bootstrapped_toolchain and not self.query_yes_no( - "You are about to do a clean FreeBSD build (without external toolchain). This will rebuild all of " - "LLVM and take a long time. Are you sure?", default_result=True): + "You are about to do a clean FreeBSD build (without external toolchain). This will rebuild all of " + "LLVM and take a long time. Are you sure?", + default_result=True, + ): return ThreadJoiner(None) return super().clean() @classmethod def setup_config_options(cls, install_directory_help=None, **kwargs) -> None: super().setup_config_options(bootstrap_toolchain=True) - cls.include_llvm = cls.add_bool_option("build-target-llvm", - help="Build LLVM for the target architecture. Note: this adds " - "significant time to the build") + cls.include_llvm = cls.add_bool_option( + "build-target-llvm", + help="Build LLVM for the target architecture. Note: this adds " "significant time to the build", + ) def add_cross_build_options(self) -> None: # Just try to build as much as possible (but using make.py) @@ -1516,20 +1749,27 @@ def setup_config_options(cls, **kwargs) -> None: super().setup_config_options(**kwargs) cls.tinderbox = cls.add_bool_option("tinderbox", help="Use `make tinderbox` instead of `make universe`") cls.worlds_only = cls.add_bool_option("worlds-only", help="Only build worlds (skip building kernels)") - cls.kernels_only = cls.add_bool_option("kernels-only", help="Only build kernels (skip building worlds)", - default=ComputedDefaultValue( - function=lambda conf, proj: conf.skip_world, - as_string="true if --skip-world is set")) + cls.kernels_only = cls.add_bool_option( + "kernels-only", + help="Only build kernels (skip building worlds)", + default=ComputedDefaultValue( + function=lambda conf, proj: conf.skip_world, as_string="true if --skip-world is set" + ), + ) - cls.jflag_in_subjobs = cls.add_config_option("jflag-in-subjobs", - help="Number of jobs in each world/kernel build", - kind=int, default=ComputedDefaultValue(jflag_in_subjobs, - "default -j flag / 2")) + cls.jflag_in_subjobs = cls.add_config_option( + "jflag-in-subjobs", + help="Number of jobs in each world/kernel build", + kind=int, + default=ComputedDefaultValue(jflag_in_subjobs, "default -j flag / 2"), + ) - cls.jflag_for_universe = cls.add_config_option("jflag-for-universe", - help="Number of parallel world/kernel builds", - kind=int, default=ComputedDefaultValue(jflag_for_universe, - "default -j flag / 4")) + cls.jflag_for_universe = cls.add_config_option( + "jflag-for-universe", + help="Number of parallel world/kernel builds", + kind=int, + default=ComputedDefaultValue(jflag_for_universe, "default -j flag / 4"), + ) def compile(self, **kwargs) -> None: # The build seems to behave differently when -j1 is passed (it still complains about parallel make failures) @@ -1574,8 +1814,9 @@ class BuildCHERIBSD(BuildFreeBSD): default_directory_basename: str = "cheribsd" target: str = "cheribsd" can_build_with_system_clang: bool = False # We need CHERI LLVM for most architectures - repository: GitRepository = GitRepository("https://github.com/CTSRD-CHERI/cheribsd.git", - old_branches={"master": "main"}) + repository: GitRepository = GitRepository( + "https://github.com/CTSRD-CHERI/cheribsd.git", old_branches={"master": "main"} + ) _default_install_dir_fn: ComputedDefaultValue[Path] = _arch_suffixed_custom_install_dir("rootfs") supported_architectures = CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID is_sdk_target: bool = True @@ -1595,46 +1836,70 @@ class BuildCHERIBSD(BuildFreeBSD): def setup_config_options(cls, kernel_only_target=False, install_directory_help=None, **kwargs) -> None: if install_directory_help is None: install_directory_help = "Install directory for CheriBSD root file system" - super().setup_config_options(install_directory_help=install_directory_help, use_upstream_llvm=False, - kernel_only_target=kernel_only_target) + super().setup_config_options( + install_directory_help=install_directory_help, + use_upstream_llvm=False, + kernel_only_target=kernel_only_target, + ) fpga_targets = CompilationTargets.ALL_CHERIBSD_RISCV_TARGETS - cls.build_fpga_kernels = cls.add_bool_option("build-fpga-kernels", show_help=True, _allow_unknown_targets=True, - only_add_for_targets=fpga_targets, - help="Also build kernels for the FPGA.") - cls.build_fett_kernels = cls.add_bool_option("build-fett-kernels", show_help=False, _allow_unknown_targets=True, - only_add_for_targets=fpga_targets, - help="Also build kernels for FETT.") + cls.build_fpga_kernels = cls.add_bool_option( + "build-fpga-kernels", + show_help=True, + _allow_unknown_targets=True, + only_add_for_targets=fpga_targets, + help="Also build kernels for the FPGA.", + ) + cls.build_fett_kernels = cls.add_bool_option( + "build-fett-kernels", + show_help=False, + _allow_unknown_targets=True, + only_add_for_targets=fpga_targets, + help="Also build kernels for FETT.", + ) cls.mfs_root_image = cls.add_optional_path_option( - "mfs-root-image", help="Path to an MFS root image to be embedded in the kernel for booting") + "mfs-root-image", help="Path to an MFS root image to be embedded in the kernel for booting" + ) cls.default_kernel_abi = cls.add_config_option( - "default-kernel-abi", show_help=True, _allow_unknown_targets=True, + "default-kernel-abi", + show_help=True, + _allow_unknown_targets=True, only_add_for_targets=cls.purecap_kernel_targets, - kind=KernelABI, default=KernelABI.HYBRID, + kind=KernelABI, + default=KernelABI.HYBRID, enum_choices=[KernelABI.HYBRID, KernelABI.PURECAP], - help="Select default kernel to build") + help="Select default kernel to build", + ) # We also want to add this config option to the fake "cheribsd" target (to keep the config file manageable) cls.build_alternate_abi_kernels = cls.add_bool_option( - "build-alternate-abi-kernels", show_help=True, + "build-alternate-abi-kernels", + show_help=True, _allow_unknown_targets=True, only_add_for_targets=cls.purecap_kernel_targets, default=True, - help="Also build kernels with non-default ABI (purecap or hybrid)") + help="Also build kernels with non-default ABI (purecap or hybrid)", + ) - cls.build_bench_kernels = cls.add_bool_option("build-bench-kernels", show_help=True, - _allow_unknown_targets=True, - help="Also build benchmark kernels") + cls.build_bench_kernels = cls.add_bool_option( + "build-bench-kernels", show_help=True, _allow_unknown_targets=True, help="Also build benchmark kernels" + ) cls.build_nocaprevoke_kernels = cls.add_bool_option( - "build-nocaprevoke-kernels", show_help=True, _allow_unknown_targets=True, + "build-nocaprevoke-kernels", + show_help=True, + _allow_unknown_targets=True, only_add_for_targets=CompilationTargets.ALL_CHERIBSD_CHERI_TARGETS_WITH_HYBRID, - help="Build kernels without caprevoke support") + help="Build kernels without caprevoke support", + ) if kernel_only_target: return # The remaining options only affect the userspace build - cls.sysroot_only = cls.add_bool_option("sysroot-only", show_help=False, - help="Only build a sysroot instead of the full system. This will only " - "build the libraries and skip all binaries") + cls.sysroot_only = cls.add_bool_option( + "sysroot-only", + show_help=False, + help="Only build a sysroot instead of the full system. This will only " + "build the libraries and skip all binaries", + ) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -1655,16 +1920,22 @@ def get_default_kernel_abi(self) -> KernelABI: kernel_abi = KernelABI.NOCHERI return kernel_abi - def _get_config_variants(self, platforms: "set[ConfigPlatform]", kernel_abis: "list[KernelABI]", - combine_flags: "list[str]", **filter_kwargs) -> "list[CheriBSDConfig]": + def _get_config_variants( + self, + platforms: "set[ConfigPlatform]", + kernel_abis: "list[KernelABI]", + combine_flags: "list[str]", + **filter_kwargs, + ) -> "list[CheriBSDConfig]": flag_values = itertools.product([False, True], repeat=len(combine_flags)) combine_tuples = list(itertools.product(platforms, kernel_abis, flag_values)) configs = [] for platform, kernel_abi, flag_tuple in combine_tuples: combined_filter = {flag: v for flag, v in zip(combine_flags, flag_tuple)} filter_kwargs.update(combined_filter) - configs += CheriBSDConfigTable.get_configs(self.crosscompile_target, platform=platform, - kernel_abi=kernel_abi, **filter_kwargs) + configs += CheriBSDConfigTable.get_configs( + self.crosscompile_target, platform=platform, kernel_abi=kernel_abi, **filter_kwargs + ) return configs def _get_kernel_abis_to_build(self) -> "list[KernelABI]": @@ -1692,8 +1963,9 @@ def _get_all_kernel_configs(self) -> "list[CheriBSDConfig]": combinations.append("fett") configs = self._get_config_variants({platform}, kernel_abis, combinations) if self.build_fpga_kernels: - configs += self._get_config_variants(ConfigPlatform.fpga_platforms(), kernel_abis, - [*combinations, "mfsroot"]) + configs += self._get_config_variants( + ConfigPlatform.fpga_platforms(), kernel_abis, [*combinations, "mfsroot"] + ) return configs def default_kernel_config(self, platform: "Optional[ConfigPlatform]" = None, **filter_kwargs) -> str: @@ -1750,8 +2022,9 @@ def compile(self, **kwargs) -> None: def install(self, **kwargs) -> None: # When building we build MFS and - super().install(kernconfs=self.kernconf_list() + self.extra_kernels_with_mfs, - sysroot_only=self.sysroot_only, **kwargs) + super().install( + kernconfs=self.kernconf_list() + self.extra_kernels_with_mfs, sysroot_only=self.sysroot_only, **kwargs + ) class BuildCheriBsdMfsKernel(BuildCHERIBSD): @@ -1763,9 +2036,9 @@ class BuildCheriBsdMfsKernel(BuildCHERIBSD): *CompilationTargets.ALL_CHERIBSD_MORELLO_TARGETS, *CompilationTargets.ALL_CHERIBSD_RISCV_TARGETS, ) - default_build_dir: ComputedDefaultValue[Path] = \ - ComputedDefaultValue(function=cheribsd_reuse_build_dir, - as_string=lambda cls: BuildCHERIBSD.project_build_dir_help()) + default_build_dir: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=cheribsd_reuse_build_dir, as_string=lambda cls: BuildCHERIBSD.project_build_dir_help() + ) # This exists specifically for this target has_default_buildkernel_kernel_config: bool = False # We want the CheriBSD config options as well, so that defaults (e.g. build-alternate-abi-kernels) are inherited. @@ -1780,6 +2053,7 @@ def __init__(self, *args, **kwargs) -> None: @cached_property def image(self) -> Path: from ..disk_image import BuildMfsRootCheriBSDDiskImage + return BuildMfsRootCheriBSDDiskImage.get_instance(self).disk_image_path @classmethod @@ -1806,16 +2080,23 @@ def _build_and_install_kernel_binaries(self, kernconfs: "list[str]", image: Path # Install to a temporary directory and then copy the kernel to OUTPUT_ROOT # Don't bother with modules for the MFS kernels: extra_make_args = dict(NO_MODULES="yes") - self._buildkernel(kernconfs=kernconfs, mfs_root_image=image, extra_make_args=extra_make_args, - ignore_skip_kernel=True) + self._buildkernel( + kernconfs=kernconfs, mfs_root_image=image, extra_make_args=extra_make_args, ignore_skip_kernel=True + ) with tempfile.TemporaryDirectory(prefix="cheribuild-" + self.target + "-") as td: temp_rootfs = Path(td) build_suffix = self.crosscompile_target.build_suffix(self.config, include_os=False) - self._installkernel(kernconfs=kernconfs, install_dir=temp_rootfs, extra_make_args=extra_make_args, - ignore_skip_kernel=True) - self._copykernel(kernconfs=kernconfs, rootfs_dir=temp_rootfs, dest_dir=self.config.cheribsd_image_root, - dest_kernel_suffix=build_suffix, dest_all_extra=True) + self._installkernel( + kernconfs=kernconfs, install_dir=temp_rootfs, extra_make_args=extra_make_args, ignore_skip_kernel=True + ) + self._copykernel( + kernconfs=kernconfs, + rootfs_dir=temp_rootfs, + dest_dir=self.config.cheribsd_image_root, + dest_kernel_suffix=build_suffix, + dest_all_extra=True, + ) def _get_all_kernel_configs(self) -> "list[CheriBSDConfig]": kernel_abis = self._get_kernel_abis_to_build() @@ -1827,8 +2108,9 @@ def _get_all_kernel_configs(self) -> "list[CheriBSDConfig]": combinations.append("nocaprevoke") configs = self._get_config_variants({platform}, kernel_abis, combinations, mfsroot=True) if self.build_fpga_kernels: - configs += self._get_config_variants(ConfigPlatform.fpga_platforms(), kernel_abis, - combinations, mfsroot=True) + configs += self._get_config_variants( + ConfigPlatform.fpga_platforms(), kernel_abis, combinations, mfsroot=True + ) return configs def default_kernel_config(self, platform: "Optional[ConfigPlatform]" = None, **filter_kwargs) -> str: @@ -1846,10 +2128,10 @@ def get_kernel_configs(self, platform: "Optional[ConfigPlatform]") -> "list[str] return [c.kernconf for c in filter_kernel_configs(configs, platform=platform, kernel_abi=None)] def get_kernel_install_path(self, kernconf: "Optional[str]" = None) -> Path: - """ Get the installed kernel path for an MFS kernel config that has been built. """ + """Get the installed kernel path for an MFS kernel config that has been built.""" path = self.config.cheribsd_image_root / ( - "kernel" + self.crosscompile_target.build_suffix(self.config, include_os=False) + - "." + kernconf) + "kernel" + self.crosscompile_target.build_suffix(self.config, include_os=False) + "." + kernconf + ) return path @@ -1946,8 +2228,9 @@ def process(self) -> None: self.objdir / "tmp/legacy/bin", self.objdir / "tmp/obj-tools/usr.sbin/makefs", ] - release_args.set_env(PATH=":".join(map(str, extra_path_entries)) + ":" + - release_args.env_vars.get("PATH", os.getenv("PATH"))) + release_args.set_env( + PATH=":".join(map(str, extra_path_entries)) + ":" + release_args.env_vars.get("PATH", os.getenv("PATH")) + ) # DESTDIR for install target is where to install the media, as you'd # expect, unlike the release target where it leaks into installworld @@ -1965,9 +2248,9 @@ class BuildFreeBSDRelease(BuildFreeBSDReleaseMixin, BuildFreeBSD): dependencies: "tuple[str, ...]" = ("freebsd",) repository: ReuseOtherProjectRepository = ReuseOtherProjectRepository(source_project=BuildFreeBSD) _always_add_suffixed_targets: bool = True - default_build_dir: ComputedDefaultValue[Path] = \ - ComputedDefaultValue(function=freebsd_reuse_build_dir, - as_string=lambda cls: BuildFreeBSD.project_build_dir_help()) + default_build_dir: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=freebsd_reuse_build_dir, as_string=lambda cls: BuildFreeBSD.project_build_dir_help() + ) _default_install_dir_fn: ComputedDefaultValue[Path] = _arch_suffixed_custom_install_dir("freebsd-release") # We want the FreeBSD config options as well so the release installworld, # distributeworld etc. calls match what was built. @@ -1979,9 +2262,9 @@ class BuildCheriBSDRelease(BuildFreeBSDReleaseMixin, BuildCHERIBSD): dependencies: "tuple[str, ...]" = ("cheribsd",) repository: ReuseOtherProjectRepository = ReuseOtherProjectRepository(source_project=BuildCHERIBSD) _always_add_suffixed_targets: bool = True - default_build_dir: ComputedDefaultValue[Path] = \ - ComputedDefaultValue(function=cheribsd_reuse_build_dir, - as_string=lambda cls: BuildCHERIBSD.project_build_dir_help()) + default_build_dir: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=cheribsd_reuse_build_dir, as_string=lambda cls: BuildCHERIBSD.project_build_dir_help() + ) _default_install_dir_fn: ComputedDefaultValue[Path] = _arch_suffixed_custom_install_dir("cheribsd-release") # We want the CheriBSD config options as well so the release installworld, # distributeworld etc. calls match what was built. @@ -2014,25 +2297,35 @@ def __init__(self, *args, **kwargs) -> None: def check_system_dependencies(self) -> None: super().check_system_dependencies() self.check_required_system_tool("bsdtar", cheribuild_target="bsdtar", apt="libarchive-tools") - if not OSInfo.IS_FREEBSD and not self.remote_path and not self.rootfs_source_class.get_instance( - self).crossbuild: + if ( + not OSInfo.IS_FREEBSD + and not self.remote_path + and not self.rootfs_source_class.get_instance(self).crossbuild + ): config_option = "'--" + self.get_config_option_name("remote_path") + "'" - self.fatal("Path to the remote SDK is not set, option", config_option, - "must be set to a path that scp understands (e.g. vica:~foo/cheri/output/sdk)") + self.fatal( + "Path to the remote SDK is not set, option", + config_option, + "must be set to a path that scp understands (e.g. vica:~foo/cheri/output/sdk)", + ) if not self.config.pretend: sys.exit("Cannot continue...") @classmethod def setup_config_options(cls, **kwargs) -> None: super().setup_config_options(**kwargs) - cls.copy_remote_sysroot = cls.add_bool_option("copy-remote-sysroot", - help="Copy sysroot from remote server instead of from local " - "machine") - cls.remote_path = cls.add_config_option("remote-sdk-path", show_help=True, metavar="PATH", - help="The path to the CHERI SDK on the remote FreeBSD machine (e.g. " - "vica:~foo/cheri/output/sdk)") + cls.copy_remote_sysroot = cls.add_bool_option( + "copy-remote-sysroot", help="Copy sysroot from remote server instead of from local " "machine" + ) + cls.remote_path = cls.add_config_option( + "remote-sdk-path", + show_help=True, + metavar="PATH", + help="The path to the CHERI SDK on the remote FreeBSD machine (e.g. " "vica:~foo/cheri/output/sdk)", + ) cls.install_dir_override = cls.add_optional_path_option( - "install-directory", help="Override for the sysroot install directory") + "install-directory", help="Override for the sysroot install directory" + ) @property def cross_sysroot_path(self) -> Path: @@ -2045,7 +2338,8 @@ def copy_sysroot_from_remote_machine(self) -> None: self.info("Copying sysroot from remote system.") if not self.remote_path: self.fatal( - "Missing remote SDK path: Please set --cheribsd-sysroot/remote-sdk-path (or --freebsd/crossbuild)") + "Missing remote SDK path: Please set --cheribsd-sysroot/remote-sdk-path (or --freebsd/crossbuild)" + ) if self.config.pretend: self.remote_path = "someuser@somehose:this/path/does/not/exist" # noinspection PyAttributeOutsideInit @@ -2073,18 +2367,32 @@ def create_sysroot(self) -> None: bsdtar_path = shutil.which(str(self.bsdtar_cmd)) if not bsdtar_path: bsdtar_path = str(self.bsdtar_cmd) - tar_cmd = [bsdtar_path, "cf", "-", "--include=./lib/", "--include=./usr/include/", - "--include=./usr/lib/", "--include=./usr/libdata/", - "--include=./usr/lib32", "--include=./usr/lib64", - "--include=./usr/lib64c", "--include=./usr/lib64cb", - "--include=./usr/lib128", "--include=./usr/lib128g", - # only pack those files that are mentioned in METALOG - "@METALOG.world"] + tar_cmd = [ + bsdtar_path, + "cf", + "-", + "--include=./lib/", + "--include=./usr/include/", + "--include=./usr/lib/", + "--include=./usr/libdata/", + "--include=./usr/lib32", + "--include=./usr/lib64", + "--include=./usr/lib64c", + "--include=./usr/lib64cb", + "--include=./usr/lib128", + "--include=./usr/lib128g", + # only pack those files that are mentioned in METALOG + "@METALOG.world", + ] rootfs_target = self.rootfs_source_class.get_instance(self) rootfs_dir = rootfs_target.real_install_root_dir if not (rootfs_dir / "lib/libc.so.7").is_file(): - self.fatal("Sysroot source directory", rootfs_dir, "does not contain libc.so.7", - fixit_hint="Run `cheribuild.py " + rootfs_target.target + "` first") + self.fatal( + "Sysroot source directory", + rootfs_dir, + "does not contain libc.so.7", + fixit_hint="Run `cheribuild.py " + rootfs_target.target + "` first", + ) print_command(tar_cmd, cwd=rootfs_dir, config=self.config) if not self.config.pretend: tar_cwd = str(rootfs_dir) @@ -2095,8 +2403,9 @@ def create_sysroot(self) -> None: # create an archive to make it easier to copy the sysroot to another machine self.delete_file(self.sysroot_archive, print_verbose_only=True) - self.run_cmd("tar", "-caf", self.sysroot_archive, self.cross_sysroot_path.name, - cwd=self.cross_sysroot_path.parent) + self.run_cmd( + "tar", "-caf", self.sysroot_archive, self.cross_sysroot_path.name, cwd=self.cross_sysroot_path.parent + ) self.info("Successfully populated sysroot") def process(self) -> None: diff --git a/pycheribuild/projects/cross/cheritest.py b/pycheribuild/projects/cross/cheritest.py index 13dde3dc7..d20b277d0 100644 --- a/pycheribuild/projects/cross/cheritest.py +++ b/pycheribuild/projects/cross/cheritest.py @@ -43,10 +43,12 @@ class _BuildCheriMipsTestBase(Project): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.single_test = cls.add_config_option("single-test", help="Run a single test instead of all of them") - cls.run_tests_with_build = cls.add_bool_option("run-tests-with-build", - help="Run tests as part of the --build step (option is useful " - "for jenkins)", - show_help=False, default=True) + cls.run_tests_with_build = cls.add_bool_option( + "run-tests-with-build", + help="Run tests as part of the --build step (option is useful " "for jenkins)", + show_help=False, + default=True, + ) def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/compiler_rt.py b/pycheribuild/projects/cross/compiler_rt.py index 4efac8370..757235c23 100644 --- a/pycheribuild/projects/cross/compiler_rt.py +++ b/pycheribuild/projects/cross/compiler_rt.py @@ -63,8 +63,9 @@ def setup(self): if is_jenkins_build(): llvm_tools_bindir = self.llvm_project.get_native_install_path(self.config) / "bin" else: - llvm_tools_bindir = self.llvm_project.get_build_dir( - self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin" + llvm_tools_bindir = ( + self.llvm_project.get_build_dir(self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin" + ) self.add_cmake_options( LLVM_CONFIG_PATH=llvm_tools_bindir / "llvm-config", LLVM_EXTERNAL_LIT=llvm_tools_bindir / "llvm-lit", @@ -80,7 +81,7 @@ def setup(self): COMPILER_RT_DEFAULT_TARGET_ONLY=not self.compiling_for_host(), # Per-target runtime directories don't add the purecap suffix so can't be used right now. LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=False, - ) + ) if self.should_include_debug_info: self.add_cmake_options(COMPILER_RT_DEBUG=True) @@ -94,7 +95,8 @@ def install(self, **kwargs): # HACK: we don't really need the ubsan runtime but the toolchain pulls it in automatically # TODO: is there an easier way to create an empty archive? ubsan_runtime_path = self.install_dir / ( - "lib/freebsd/libclang_rt.ubsan_standalone-mips64c" + self.config.mips_cheri_bits_str + ".a") + "lib/freebsd/libclang_rt.ubsan_standalone-mips64c" + self.config.mips_cheri_bits_str + ".a" + ) if not ubsan_runtime_path.exists(): self.warning("Did not install ubsan runtime", ubsan_runtime_path) if self.target_info.is_rtems(): @@ -103,8 +105,9 @@ def install(self, **kwargs): self.warning("Did not install compiler runtime", rt_runtime_path.exists) else: print(self.target_info.sysroot_dir) - self.create_symlink(rt_runtime_path, - self.target_info.sysroot_dir / "lib/libclang_rt.builtins-riscv64.a") + self.create_symlink( + rt_runtime_path, self.target_info.sysroot_dir / "lib/libclang_rt.builtins-riscv64.a" + ) def run_tests(self): self.run_make("check-compiler-rt") @@ -127,10 +130,12 @@ class BuildCompilerRtBuiltins(CrossCompileCMakeProject): is_sdk_target = True root_cmakelists_subdirectory = Path("lib/builtins") needs_sysroot = False # We don't need a complete sysroot - supported_architectures = (CompilationTargets.ALL_SUPPORTED_BAREMETAL_TARGETS + - CompilationTargets.ALL_SUPPORTED_RTEMS_TARGETS + - CompilationTargets.ALL_FREESTANDING_TARGETS + - CompilationTargets.ALL_NATIVE) + supported_architectures = ( + CompilationTargets.ALL_SUPPORTED_BAREMETAL_TARGETS + + CompilationTargets.ALL_SUPPORTED_RTEMS_TARGETS + + CompilationTargets.ALL_FREESTANDING_TARGETS + + CompilationTargets.ALL_NATIVE + ) # Note: needs to be @classproperty since it is called before __init__ @classproperty @@ -182,8 +187,9 @@ def install(self, **kwargs): self.move_file(self.install_dir / "lib/rtems5" / libname, self.install_dir / "lib" / libname) elif self.target_info.is_baremetal(): self.move_file(self.install_dir / "lib/baremetal" / libname, self.real_install_root_dir / "lib" / libname) - self.create_symlink(self.install_dir / "lib" / libname, self.install_dir / "lib/libgcc.a", - print_verbose_only=False) + self.create_symlink( + self.install_dir / "lib" / libname, self.install_dir / "lib/libgcc.a", print_verbose_only=False + ) class BuildUpstreamCompilerRtBuiltins(BuildCompilerRtBuiltins): diff --git a/pycheribuild/projects/cross/crosscompileproject.py b/pycheribuild/projects/cross/crosscompileproject.py index f1c17df8e..3fad44fc1 100644 --- a/pycheribuild/projects/cross/crosscompileproject.py +++ b/pycheribuild/projects/cross/crosscompileproject.py @@ -70,7 +70,8 @@ "Linkage", "MakeCommandKind", "SubversionRepository", - "commandline_to_str"] + "commandline_to_str", +] if typing.TYPE_CHECKING: @@ -180,8 +181,13 @@ def configure(self, **kwargs): env = {k: v for k, v in self.configure_environment.items() if v} self.configure_environment.clear() self.configure_environment.update(env) - self.print(coloured(AnsiColour.yellow, "Cross configure environment:\n\t", - "\n\t".join(k + "=" + str(v) for k, v in self.configure_environment.items()))) + self.print( + coloured( + AnsiColour.yellow, + "Cross configure environment:\n\t", + "\n\t".join(k + "=" + str(v) for k, v in self.configure_environment.items()), + ) + ) super().configure(**kwargs) def process(self): diff --git a/pycheribuild/projects/cross/current_directory.py b/pycheribuild/projects/cross/current_directory.py index 6f64b2f92..075cf3ac5 100644 --- a/pycheribuild/projects/cross/current_directory.py +++ b/pycheribuild/projects/cross/current_directory.py @@ -48,8 +48,7 @@ def _cwd_directory_basename(_, _1): class CurrentDirectoryMixin: do_not_add_to_targets = True - default_directory_basename = ComputedDefaultValue(function=_cwd_directory_basename, - as_string="$SOURCE_DIR_NAME") + default_directory_basename = ComputedDefaultValue(function=_cwd_directory_basename, as_string="$SOURCE_DIR_NAME") inherit_default_directory_basename = True repository = ExternallyManagedSourceRepository() source_dir = ComputedDefaultValue(function=_cwd_source_dir, as_string="$CWD") @@ -83,11 +82,11 @@ class BuildCurrent_Directory(CurrentDirectoryMixin, CrossCompileSimpleProject): @classmethod def dependencies(cls, _) -> "tuple[str, ...]": classes = [ - BuildCurrent_Directory_Autotools, - BuildCurrent_Directory_CMake, - BuildCurrent_Directory_Makefile, - BuildCurrent_Directory_Meson, - ] + BuildCurrent_Directory_Autotools, + BuildCurrent_Directory_CMake, + BuildCurrent_Directory_Makefile, + BuildCurrent_Directory_Meson, + ] for c in classes: for f in c.autodetect_files: if (_cwd_path / f).is_file(): diff --git a/pycheribuild/projects/cross/dbus.py b/pycheribuild/projects/cross/dbus.py index f889cd830..a3628517a 100644 --- a/pycheribuild/projects/cross/dbus.py +++ b/pycheribuild/projects/cross/dbus.py @@ -28,8 +28,10 @@ class BuildDBus(CrossCompileCMakeProject): target = "dbus" supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE - repository = GitRepository("https://gitlab.freedesktop.org/dbus/dbus.git", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/dbus.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/dbus/dbus.git", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/dbus.git"], + ) dependencies = ("libexpat",) ctest_script_extra_args = ["--test-timeout", str(120 * 60)] # Tests can take a long time to run @@ -52,11 +54,17 @@ def setup(self): def install(self, **kwargs): super().install() if not self.compiling_for_host() and self.target_info.is_freebsd(): - self.write_file(self.rootfs_dir / "etc/rc.conf.d/dbus", contents="dbus_enable=\"YES\"\n", - overwrite=True, print_verbose_only=False) + self.write_file( + self.rootfs_dir / "etc/rc.conf.d/dbus", + contents='dbus_enable="YES"\n', + overwrite=True, + print_verbose_only=False, + ) # Slightly modified version of https://cgit.freebsd.org/ports/plain/devel/dbus/files/dbus.in # to add the necessary users on-demand and chmod/chown the rsync'd files - self.write_file(self.rootfs_dir / self.target_info.localbase / "etc/rc.d/dbus", contents=f"""#!/bin/sh + self.write_file( + self.rootfs_dir / self.target_info.localbase / "etc/rc.d/dbus", + contents=f"""#!/bin/sh # PROVIDE: dbus # REQUIRE: DAEMON ldconfig @@ -103,4 +111,7 @@ def install(self, **kwargs): load_rc_config ${{name}} run_rc_command "$1" -""", overwrite=True, mode=0o755) +""", + overwrite=True, + mode=0o755, + ) diff --git a/pycheribuild/projects/cross/device_model.py b/pycheribuild/projects/cross/device_model.py index c5fc73b8a..146d72c6a 100644 --- a/pycheribuild/projects/cross/device_model.py +++ b/pycheribuild/projects/cross/device_model.py @@ -47,13 +47,12 @@ def compile(self, **kwargs): self.run_make("purecap") def install(self, **kwargs): - self.install_file(self.build_dir / "obj/device-model-riscv.bin", - self.real_install_root_dir / "device-model-riscv.bin") + self.install_file( + self.build_dir / "obj/device-model-riscv.bin", self.real_install_root_dir / "device-model-riscv.bin" + ) def setup(self): super().setup() cc = self.config.cheri_sdk_bindir / "clang" objcopy = self.config.cheri_sdk_bindir / "objcopy" - self.make_args.env_vars = {"CC": str(cc), - "AS": str(cc), - "OBJCOPY": str(objcopy)} + self.make_args.env_vars = {"CC": str(cc), "AS": str(cc), "OBJCOPY": str(objcopy)} diff --git a/pycheribuild/projects/cross/dlmalloc.py b/pycheribuild/projects/cross/dlmalloc.py index 59a68dbad..44d896483 100644 --- a/pycheribuild/projects/cross/dlmalloc.py +++ b/pycheribuild/projects/cross/dlmalloc.py @@ -47,19 +47,19 @@ def setup_config_options(cls, **kwargs): cls.cheri_set_bounds = cls.add_bool_option("cheri-bounds", default=True, help="Set bounds on allocations") - cls.qmabs = cls.add_config_option("qmabs", kind=int, - help="Quarantine memory absolute threshold") + cls.qmabs = cls.add_config_option("qmabs", kind=int, help="Quarantine memory absolute threshold") - cls.qmratio = cls.add_config_option("qmratio", kind=float, - help="Quarantine memory ratio threshold") + cls.qmratio = cls.add_config_option("qmratio", kind=float, help="Quarantine memory ratio threshold") - cls.qmmin = cls.add_config_option("qmmin", kind=int, - help="Minimum amount quarantined to trigger a revocation based on ratio") + cls.qmmin = cls.add_config_option( + "qmmin", kind=int, help="Minimum amount quarantined to trigger a revocation based on ratio" + ) cls.revoke = cls.add_bool_option("revoke", help="Revoke quarantine before reusing") - cls.consolidate_on_free = cls.add_bool_option("consolidate", default=True, - help="Consolidate memory when quarantining") + cls.consolidate_on_free = cls.add_bool_option( + "consolidate", default=True, help="Consolidate memory when quarantining" + ) cls.zero_memory = cls.add_bool_option("zero-memory", help="Zero allocated memory") @@ -67,11 +67,12 @@ def setup_config_options(cls, **kwargs): cls.unmap_support = cls.add_bool_option("unmap-support", default=True, help="support for unmapping") - cls.unmap_threshold = cls.add_config_option("unmap-threshold", kind=int, - help="Threshold (in pages) at which interior pages of quanantined " - "chunks are unmapped") - cls.quar_unsafe = cls.add_bool_option("unsafe-quarantine", - help="Don't isolate quarantine structures") + cls.unmap_threshold = cls.add_config_option( + "unmap-threshold", + kind=int, + help="Threshold (in pages) at which interior pages of quanantined " "chunks are unmapped", + ) + cls.quar_unsafe = cls.add_bool_option("unsafe-quarantine", help="Don't isolate quarantine structures") def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/doom.py b/pycheribuild/projects/cross/doom.py index 8c36f418a..7cca81fc9 100644 --- a/pycheribuild/projects/cross/doom.py +++ b/pycheribuild/projects/cross/doom.py @@ -32,9 +32,12 @@ class BuildChocolateDoom(CrossCompileAutotoolsProject): target = "chocolate-doom" - repository = GitRepository("https://github.com/chocolate-doom/chocolate-doom.git", - old_urls=[b"https://github.com/jrtc27/chocolate-doom.git"], - default_branch="master", force_branch=True) + repository = GitRepository( + "https://github.com/chocolate-doom/chocolate-doom.git", + old_urls=[b"https://github.com/jrtc27/chocolate-doom.git"], + default_branch="master", + force_branch=True, + ) dependencies = ("sdl", "sdl-mixer", "sdl-net", "libpng") _can_use_autogen_sh = False # Can't use autogen.sh since it will run configure in the wrong dir @@ -55,14 +58,14 @@ class BuildFreedoom(CrossCompileProject): version = "0.12.1" url_prefix: str = f"https://github.com/freedoom/freedoom/releases/download/v{version}/" packages: "dict[str, list[str]]" = { - 'freedoom': ['freedoom1', 'freedoom2'], - 'freedm': ['freedm'], + "freedoom": ["freedoom1", "freedoom2"], + "freedm": ["freedm"], } def compile(self, **kwargs): for pkgname, wads in self.packages.items(): filename = f"{pkgname}-{self.version}.zip" - wadfiles = ['*/' + wad + ".wad" for wad in wads] + wadfiles = ["*/" + wad + ".wad" for wad in wads] if not (self.build_dir / filename).is_file(): self.download_file(self.build_dir / filename, self.url_prefix + filename) self.run_cmd("unzip", "-jo", filename, *wadfiles, cwd=self.build_dir) @@ -73,6 +76,11 @@ def install(self, **kwargs): wadfile = wad + ".wad" wadpath = Path("share/doom") / wadfile self.install_file(self.build_dir / wadfile, self.install_dir / wadpath) - self.write_file(self.install_dir / "bin" / wad, overwrite=True, mode=0o755, - contents="#!/bin/sh\nexec {0}/bin/chocolate-doom -iwad {0}/{1} \"$@\"\n".format( - self.install_prefix, wadpath)) + self.write_file( + self.install_dir / "bin" / wad, + overwrite=True, + mode=0o755, + contents='#!/bin/sh\nexec {0}/bin/chocolate-doom -iwad {0}/{1} "$@"\n'.format( + self.install_prefix, wadpath + ), + ) diff --git a/pycheribuild/projects/cross/drm_kmod.py b/pycheribuild/projects/cross/drm_kmod.py index e41246878..d589016c7 100644 --- a/pycheribuild/projects/cross/drm_kmod.py +++ b/pycheribuild/projects/cross/drm_kmod.py @@ -29,7 +29,8 @@ def setup(self) -> None: MODULES_OVERRIDE="linuxkpi", ) self.kernel_make_args = self.freebsd_project.kernel_make_args_for_config( - self.freebsd_project.kernel_config, extra_make_args, + self.freebsd_project.kernel_config, + extra_make_args, ) assert self.kernel_make_args.kind == MakeCommandKind.BsdMake @@ -38,7 +39,11 @@ def clean(self, **kwargs) -> None: if self.use_buildenv: self.info("Cleaning drm-kmod modules for configs:", self.freebsd_project.kernel_config) self.freebsd_project.build_and_install_subdir( - self.kernel_make_args, str(self.source_dir), skip_build=True, skip_clean=False, skip_install=True, + self.kernel_make_args, + str(self.source_dir), + skip_build=True, + skip_clean=False, + skip_install=True, ) else: self.info("Clean not supported yet") @@ -48,11 +53,18 @@ def compile(self, **kwargs) -> None: self.info("Building drm-kmod modules for configs:", self.freebsd_project.kernel_config) if self.use_buildenv: self.freebsd_project.build_and_install_subdir( - self.kernel_make_args, str(self.source_dir), skip_build=False, skip_clean=True, skip_install=True, + self.kernel_make_args, + str(self.source_dir), + skip_build=False, + skip_clean=True, + skip_install=True, ) else: self.run_make( - "buildkernel", options=self.kernel_make_args, cwd=self.freebsd_project.source_dir, parallel=True, + "buildkernel", + options=self.kernel_make_args, + cwd=self.freebsd_project.source_dir, + parallel=True, ) def install(self, **kwargs) -> None: @@ -65,9 +77,16 @@ def install(self, **kwargs) -> None: make_args.set_env(METALOG=self.real_install_root_dir / "METALOG.drm-kmod") if self.use_buildenv: self.freebsd_project.build_and_install_subdir( - make_args, str(self.source_dir), skip_build=True, skip_clean=True, skip_install=False, + make_args, + str(self.source_dir), + skip_build=True, + skip_clean=True, + skip_install=False, ) else: self.run_make_install( - target="installkernel", options=make_args, cwd=self.freebsd_project.source_dir, parallel=False, + target="installkernel", + options=make_args, + cwd=self.freebsd_project.source_dir, + parallel=False, ) diff --git a/pycheribuild/projects/cross/ffmpeg.py b/pycheribuild/projects/cross/ffmpeg.py index 0110520a5..c90efd94c 100644 --- a/pycheribuild/projects/cross/ffmpeg.py +++ b/pycheribuild/projects/cross/ffmpeg.py @@ -28,9 +28,11 @@ class BuildFfmpeg(CrossCompileAutotoolsProject): target = "ffmpeg" - repository = GitRepository("https://github.com/FFmpeg/FFmpeg.git", - temporary_url_override="https://github.com/arichardson/FFmpeg.git", - url_override_reason="Needs --disable-neon workarounds") + repository = GitRepository( + "https://github.com/FFmpeg/FFmpeg.git", + temporary_url_override="https://github.com/arichardson/FFmpeg.git", + url_override_reason="Needs --disable-neon workarounds", + ) ctest_script_extra_args = ["--test-timeout", str(180 * 60)] # Tests take a long time to run add_host_target_build_config_options = False # doesn't understand --host _configure_supports_variables_on_cmdline = False # not really an autotools project @@ -39,31 +41,35 @@ class BuildFfmpeg(CrossCompileAutotoolsProject): def setup(self): super().setup() cflags = self.default_compiler_flags - self.configure_args.extend([ - f"--ar={self.target_info.ar}", - f"--as={self.CC}", - f"--cc={self.CC}", - f"--cxx={self.CXX}", - f"--ld={self.CC}", - f"--nm={self.target_info.nm}", - f"--ranlib={self.target_info.ranlib}", - f"--strip={self.target_info.strip_tool}", - f"--extra-cflags={self.commandline_to_str(cflags + self.CFLAGS)}", - f"--extra-cxxflags={self.commandline_to_str(cflags + self.CXXFLAGS)}", - f"--extra-ldflags={self.commandline_to_str(self.default_ldflags + self.LDFLAGS)}", - "--enable-pic", - "--disable-doc", - "--disable-ffplay", - "--disable-ffprobe", - ]) + self.configure_args.extend( + [ + f"--ar={self.target_info.ar}", + f"--as={self.CC}", + f"--cc={self.CC}", + f"--cxx={self.CXX}", + f"--ld={self.CC}", + f"--nm={self.target_info.nm}", + f"--ranlib={self.target_info.ranlib}", + f"--strip={self.target_info.strip_tool}", + f"--extra-cflags={self.commandline_to_str(cflags + self.CFLAGS)}", + f"--extra-cxxflags={self.commandline_to_str(cflags + self.CXXFLAGS)}", + f"--extra-ldflags={self.commandline_to_str(self.default_ldflags + self.LDFLAGS)}", + "--enable-pic", + "--disable-doc", + "--disable-ffplay", + "--disable-ffprobe", + ] + ) if not self.compiling_for_host(): - self.configure_args.extend([ - "--enable-cross-compile", - f"--host-cc={self.host_CC}", - f"--host-ld={self.host_CC}", - f"--arch={self.crosscompile_target.cpu_architecture.value}", - f"--target-os={self.target_info.cmake_system_name.lower()}", - ]) + self.configure_args.extend( + [ + "--enable-cross-compile", + f"--host-cc={self.host_CC}", + f"--host-ld={self.host_CC}", + f"--arch={self.crosscompile_target.cpu_architecture.value}", + f"--target-os={self.target_info.cmake_system_name.lower()}", + ] + ) if self.compiling_for_cheri(): self.configure_args.append("--disable-neon") # NEON asm needs some adjustments diff --git a/pycheribuild/projects/cross/freertos.py b/pycheribuild/projects/cross/freertos.py index 951d7e88e..7c88d115d 100644 --- a/pycheribuild/projects/cross/freertos.py +++ b/pycheribuild/projects/cross/freertos.py @@ -40,22 +40,25 @@ class BuildFreeRTOS(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/FreeRTOS-mirror", - force_branch=True, default_branch="cheri") + repository = GitRepository( + "https://github.com/CTSRD-CHERI/FreeRTOS-mirror", force_branch=True, default_branch="cheri" + ) target = "freertos" dependencies = ("newlib", "compiler-rt-builtins") is_sdk_target = True needs_sysroot = False # We don't need a complete sysroot supported_architectures = ( CompilationTargets.BAREMETAL_NEWLIB_RISCV64_PURECAP, - CompilationTargets.BAREMETAL_NEWLIB_RISCV64) + CompilationTargets.BAREMETAL_NEWLIB_RISCV64, + ) default_install_dir = DefaultInstallDir.ROOTFS_LOCALBASE # FreeRTOS Demos to build supported_freertos_demos = [ # Generic/simple (CHERI-)RISC-V Demo that runs main_blinky on simulators # and simple SoCs - "RISC-V-Generic"] + "RISC-V-Generic" + ] # Map Demos and the FreeRTOS apps we support building/running for supported_demo_apps = {"RISC-V-Generic": ["main_blinky"]} @@ -68,9 +71,11 @@ class BuildFreeRTOS(CrossCompileAutotoolsProject): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.default_demo_app = "qemu_virt-" + self.target_info.get_riscv_arch_string(self.crosscompile_target, - softfloat=True) + \ - self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + self.default_demo_app = ( + "qemu_virt-" + + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + + self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + ) def setup(self): super().setup() @@ -100,26 +105,47 @@ def setup(self): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.demo: str = typing.cast(str, cls.add_config_option( - "demo", metavar="DEMO", show_help=True, default=cls.default_demo, - help="The FreeRTOS Demo build.")) - - cls.demo_app: str = typing.cast(str, cls.add_config_option( - "prog", metavar="PROG", show_help=True, default=cls.default_demo_app, - help="The FreeRTOS program to build.")) - - cls.demo_bsp: str = typing.cast(str, cls.add_config_option( - "bsp", metavar="BSP", show_help=True, - default=ComputedDefaultValue(function=lambda _, p: p.default_demo_bsp(), - as_string="target-dependent default"), - help="The FreeRTOS BSP to build. This is only valid for the " - "paramterized RISC-V-Generic. The BSP option chooses " - "platform, RISC-V arch and RISC-V abi in the " - "$platform-$arch-$abi format. See RISC-V-Generic/README for more details")) + cls.demo: str = typing.cast( + str, + cls.add_config_option( + "demo", metavar="DEMO", show_help=True, default=cls.default_demo, help="The FreeRTOS Demo build." + ), + ) + + cls.demo_app: str = typing.cast( + str, + cls.add_config_option( + "prog", + metavar="PROG", + show_help=True, + default=cls.default_demo_app, + help="The FreeRTOS program to build.", + ), + ) + + cls.demo_bsp: str = typing.cast( + str, + cls.add_config_option( + "bsp", + metavar="BSP", + show_help=True, + default=ComputedDefaultValue( + function=lambda _, p: p.default_demo_bsp(), as_string="target-dependent default" + ), + help="The FreeRTOS BSP to build. This is only valid for the " + "paramterized RISC-V-Generic. The BSP option chooses " + "platform, RISC-V arch and RISC-V abi in the " + "$platform-$arch-$abi format. See RISC-V-Generic/README for more details", + ), + ) def default_demo_bsp(self): - return "qemu_virt-" + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + "-" + \ - self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + return ( + "qemu_virt-" + + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + + "-" + + self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + ) def compile(self, **kwargs): self.make_args.set(BSP=self.demo_bsp) @@ -129,8 +155,10 @@ def compile(self, **kwargs): self.make_args.set(PROG=self.demo_app) self.run_make("clean", cwd=self.source_dir / str("FreeRTOS/Demo/" + self.demo)) self.run_make(cwd=self.source_dir / str("FreeRTOS/Demo/" + self.demo)) - self.move_file(self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo_app + ".elf"), - self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo + self.demo_app + ".elf")) + self.move_file( + self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo_app + ".elf"), + self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo + self.demo_app + ".elf"), + ) def configure(self): pass @@ -141,7 +169,8 @@ def needs_configure(self): def install(self, **kwargs): self.install_file( self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo + self.demo_app + ".elf"), - self.real_install_root_dir / str("FreeRTOS/Demo/" + self.demo + "_" + self.demo_app + ".elf")) + self.real_install_root_dir / str("FreeRTOS/Demo/" + self.demo + "_" + self.demo_app + ".elf"), + ) def process(self): if self.demo not in self.supported_freertos_demos: @@ -150,17 +179,21 @@ def process(self): if self.demo_app not in self.supported_demo_apps[self.demo]: self.fatal(self.demo + " Demo doesn't support/have " + self.demo_app) - with self.set_env(PATH=str(self.sdk_bindir) + ":" + os.getenv("PATH", ""), - # Add compiler-rt location to the search path - LDFLAGS="-L" + str(BuildCompilerRtBuiltins.get_install_dir(self) / "lib")): + with self.set_env( + PATH=str(self.sdk_bindir) + ":" + os.getenv("PATH", ""), + # Add compiler-rt location to the search path + LDFLAGS="-L" + str(BuildCompilerRtBuiltins.get_install_dir(self) / "lib"), + ): super().process() class LaunchFreeRTOSQEMU(LaunchQEMUBase): target = "run-freertos" dependencies = ("freertos",) - supported_architectures = (CompilationTargets.BAREMETAL_NEWLIB_RISCV64_PURECAP, - CompilationTargets.BAREMETAL_NEWLIB_RISCV64) + supported_architectures = ( + CompilationTargets.BAREMETAL_NEWLIB_RISCV64_PURECAP, + CompilationTargets.BAREMETAL_NEWLIB_RISCV64, + ) forward_ssh_port = False qemu_user_networking = False _enable_smbfs_support = False @@ -179,26 +212,47 @@ def setup(self): def setup_config_options(cls, **kwargs): super().setup_config_options(defaultSshPort=None, **kwargs) - cls.demo: str = typing.cast(str, cls.add_config_option( - "demo", metavar="DEMO", show_help=True, default=cls.default_demo, - help="The FreeRTOS Demo to run.")) - - cls.demo_app: str = typing.cast(str, cls.add_config_option( - "prog", metavar="PROG", show_help=True, default=cls.default_demo_app, - help="The FreeRTOS program to run.")) - - cls.demo_bsp = typing.cast(str, cls.add_config_option( - "bsp", metavar="BSP", show_help=True, - default=ComputedDefaultValue(function=lambda _, p: p.default_demo_bsp(), - as_string="target-dependent default"), - help="The FreeRTOS BSP to run. This is only valid for the " - "paramterized RISC-V-Generic. The BSP option chooses " - "platform, RISC-V arch and RISC-V abi in the " - "$platform-$arch-$abi format. See RISC-V-Generic/README for more details")) + cls.demo: str = typing.cast( + str, + cls.add_config_option( + "demo", metavar="DEMO", show_help=True, default=cls.default_demo, help="The FreeRTOS Demo to run." + ), + ) + + cls.demo_app: str = typing.cast( + str, + cls.add_config_option( + "prog", + metavar="PROG", + show_help=True, + default=cls.default_demo_app, + help="The FreeRTOS program to run.", + ), + ) + + cls.demo_bsp = typing.cast( + str, + cls.add_config_option( + "bsp", + metavar="BSP", + show_help=True, + default=ComputedDefaultValue( + function=lambda _, p: p.default_demo_bsp(), as_string="target-dependent default" + ), + help="The FreeRTOS BSP to run. This is only valid for the " + "paramterized RISC-V-Generic. The BSP option chooses " + "platform, RISC-V arch and RISC-V abi in the " + "$platform-$arch-$abi format. See RISC-V-Generic/README for more details", + ), + ) def default_demo_bsp(self): - return "qemu_virt-" + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + "-" + \ - self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + return ( + "qemu_virt-" + + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + + "-" + + self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + ) def get_riscv_bios_args(self) -> "list[str]": # Use -bios none to ensure the FreeRTOS demo application runs in machine mode. diff --git a/pycheribuild/projects/cross/freetype.py b/pycheribuild/projects/cross/freetype.py index 0341c0a94..1d2208e63 100644 --- a/pycheribuild/projects/cross/freetype.py +++ b/pycheribuild/projects/cross/freetype.py @@ -33,8 +33,9 @@ class BuildFreeType2(CrossCompileMesonProject): target = "freetype2" supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE - repository = GitRepository("https://gitlab.freedesktop.org/freetype/freetype", - old_urls=[b"https://github.com/freetype/freetype2.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/freetype/freetype", old_urls=[b"https://github.com/freetype/freetype2.git"] + ) dependencies = ("libpng",) def setup(self): @@ -53,7 +54,8 @@ class BuildFontConfig(CrossCompileMesonProject): repository = GitRepository( "https://gitlab.freedesktop.org/fontconfig/fontconfig", temporary_url_override="https://gitlab.freedesktop.org/arichardson/fontconfig", - url_override_reason="Needs pointer provenance fixes (no PR posted yet)") + url_override_reason="Needs pointer provenance fixes (no PR posted yet)", + ) def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/gdb.py b/pycheribuild/projects/cross/gdb.py index a8491136b..ebb5a7070 100644 --- a/pycheribuild/projects/cross/gdb.py +++ b/pycheribuild/projects/cross/gdb.py @@ -106,22 +106,24 @@ def setup(self) -> None: install_root = self.install_dir if self.compiling_for_host() else self.install_prefix # See https://github.com/bsdjhb/kdbg/blob/master/gdb/build # ./configure flags - self.configure_args.extend([ - "--disable-nls", - "--enable-tui", - "--disable-ld", - "--disable-gold", - "--disable-sim", - "--enable-64-bit-bfd", - "--without-gnu-as", - "--mandir=" + str(install_root / "man"), - "--infodir=" + str(install_root / "info"), - "--disable-werror", - "MAKEINFO=" + str(shutil.which("false")), - "--with-gdb-datadir=" + str(install_root / "share/gdb"), - "--disable-libstdcxx", - "--with-guile=no", - ]) + self.configure_args.extend( + [ + "--disable-nls", + "--enable-tui", + "--disable-ld", + "--disable-gold", + "--disable-sim", + "--enable-64-bit-bfd", + "--without-gnu-as", + "--mandir=" + str(install_root / "man"), + "--infodir=" + str(install_root / "info"), + "--disable-werror", + "MAKEINFO=" + str(shutil.which("false")), + "--with-gdb-datadir=" + str(install_root / "share/gdb"), + "--disable-libstdcxx", + "--with-guile=no", + ] + ) if self.target_info.is_freebsd(): self.configure_args.append("--with-separate-debug-dir=/usr/lib/debug") @@ -162,14 +164,15 @@ def setup(self) -> None: self.configure_args.append("--with-expat") else: self.configure_args.extend(["--without-python", "--without-expat", "--without-libunwind-ia64"]) - self.configure_environment.update(gl_cv_func_gettimeofday_clobber="no", - lt_cv_sys_max_cmd_len="262144", - # The build system run CC without any flags to detect dependency style... - # (ZW_PROG_COMPILER_DEPENDENCIES([CC])) -> for gcc3 mode which seems - # correct - am_cv_CC_dependencies_compiler_type="gcc3", - MAKEINFO="/bin/false", - ) + self.configure_environment.update( + gl_cv_func_gettimeofday_clobber="no", + lt_cv_sys_max_cmd_len="262144", + # The build system run CC without any flags to detect dependency style... + # (ZW_PROG_COMPILER_DEPENDENCIES([CC])) -> for gcc3 mode which seems + # correct + am_cv_CC_dependencies_compiler_type="gcc3", + MAKEINFO="/bin/false", + ) self.configure_args.append("--with-gmp=" + str(BuildGmp.get_install_dir(self))) self.configure_args.append("--with-mpfr=" + str(BuildMpfr.get_install_dir(self))) # GDB > 12 only uses --with-gmp @@ -241,8 +244,14 @@ def install(self, **kwargs): self.install_file(self.build_dir / "binutils/strip-new", bindir / "gstrip") if self.native_target_prefix is not None: - gdbfiles = ("man/man5/gdbinit.5", "man/man1/gdbserver.1", "man/man1/gdb.1", "man/man1/gdb-add-index.1", - "bin/gdb", "bin/gdb-add-index") + gdbfiles = ( + "man/man5/gdbinit.5", + "man/man1/gdbserver.1", + "man/man1/gdb.1", + "man/man1/gdb-add-index.1", + "bin/gdb", + "bin/gdb-add-index", + ) for file in gdbfiles: dst = self.install_dir / file src = dst.parent / (self.native_target_prefix + dst.name) @@ -253,12 +262,13 @@ class BuildUpstreamGDB(BuildGDBBase): repository = GitRepository("git://sourceware.org/git/binutils-gdb.git") target = "upstream-gdb" _default_install_dir_fn = ComputedDefaultValue( - function=lambda config, proj: - config.output_root / "upstream-gdb" if proj._xtarget.is_native() else None, + function=lambda config, proj: config.output_root / "upstream-gdb" if proj._xtarget.is_native() else None, # NB: We set default_architecture so the unsuffixed target is native - as_string=lambda cls: - "$INSTALL_ROOT/upstream-gdb" if cls._xtarget is None or cls._xtarget.is_native() else None, - inherit=BuildGDBBase._default_install_dir_fn) + as_string=lambda cls: "$INSTALL_ROOT/upstream-gdb" + if cls._xtarget is None or cls._xtarget.is_native() + else None, + inherit=BuildGDBBase._default_install_dir_fn, + ) class BuildGDB(BuildGDBBase): @@ -269,12 +279,15 @@ class BuildGDB(BuildGDBBase): "https://github.com/CTSRD-CHERI/gdb.git", # Branch name is changed for every major GDB release: default_branch=default_branch, - old_branches={"mips_cheri_7.12": default_branch, - "mips_cheri-8.0.1": default_branch, - "mips_cheri-8.2": default_branch, - "mips_cheri-8.3": default_branch, - "morello-8.3": default_branch}, - old_urls=[b'https://github.com/bsdjhb/gdb.git']) + old_branches={ + "mips_cheri_7.12": default_branch, + "mips_cheri-8.0.1": default_branch, + "mips_cheri-8.2": default_branch, + "mips_cheri-8.3": default_branch, + "morello-8.3": default_branch, + }, + old_urls=[b"https://github.com/bsdjhb/gdb.git"], + ) class BuildKGDB(BuildGDB): @@ -284,4 +297,5 @@ class BuildKGDB(BuildGDB): # Branch name is changed for every major GDB release: default_branch=default_branch, old_branches={"mips_cheri-8.3-kgdb": default_branch}, - old_urls=[b'https://github.com/bsdjhb/gdb.git']) + old_urls=[b"https://github.com/bsdjhb/gdb.git"], + ) diff --git a/pycheribuild/projects/cross/git.py b/pycheribuild/projects/cross/git.py index d053da7cb..a03ae1c90 100644 --- a/pycheribuild/projects/cross/git.py +++ b/pycheribuild/projects/cross/git.py @@ -60,10 +60,8 @@ def setup(self): ac_cv_snprintf_returns_bogus="no", ) # Doesn't use pkg-config - self.configure_args.extend([ - "--with-curl=" + str(BuildCurl.get_install_dir(self)), - "--with-expat=" + str(BuildExpat.get_install_dir(self)), - ]) + self.configure_args.append(f"--with-curl={BuildCurl.get_install_dir(self)}") + self.configure_args.append(f"--with-expat={BuildExpat.get_install_dir(self)}") # Build-time detection of uname to determine more properties # Only S and R seem to be used currently, but provide sensible # values or, for V, a dummy kernconf (and format it like a release diff --git a/pycheribuild/projects/cross/gkermit.py b/pycheribuild/projects/cross/gkermit.py index 93d62d6ef..ee9f63dfd 100644 --- a/pycheribuild/projects/cross/gkermit.py +++ b/pycheribuild/projects/cross/gkermit.py @@ -40,8 +40,7 @@ def setup(self): super().setup() self.common_warning_flags.append("-Wno-unused-value") self.common_warning_flags.append("-Wno-non-literal-null-conversion") - self.make_args.set_env(KFLAGS=self.commandline_to_str( - [*self.default_compiler_flags, "-include", "unistd.h"])) + self.make_args.set_env(KFLAGS=self.commandline_to_str([*self.default_compiler_flags, "-include", "unistd.h"])) def update(self): filename = "gku201.tar.gz" diff --git a/pycheribuild/projects/cross/glib.py b/pycheribuild/projects/cross/glib.py index 4bd714c62..2df90c749 100644 --- a/pycheribuild/projects/cross/glib.py +++ b/pycheribuild/projects/cross/glib.py @@ -29,11 +29,14 @@ class BuildGlib(CrossCompileMesonProject): target = "glib" dependencies = ("pcre2", "libffi", "dbus") - repository = GitRepository("https://gitlab.gnome.org/GNOME/glib.git", - temporary_url_override="https://gitlab.gnome.org/arichardson/glib.git", - old_urls=[b"https://github.com/CTSRD-CHERI/glib.git"], - url_override_reason="Lots of CHERI incompatibilities", - default_branch="main-with-cheri-fixes", force_branch=True) + repository = GitRepository( + "https://gitlab.gnome.org/GNOME/glib.git", + temporary_url_override="https://gitlab.gnome.org/arichardson/glib.git", + old_urls=[b"https://github.com/CTSRD-CHERI/glib.git"], + url_override_reason="Lots of CHERI incompatibilities", + default_branch="main-with-cheri-fixes", + force_branch=True, + ) def setup(self) -> None: super().setup() diff --git a/pycheribuild/projects/cross/gmp.py b/pycheribuild/projects/cross/gmp.py index 39b87cf3e..8c4763395 100644 --- a/pycheribuild/projects/cross/gmp.py +++ b/pycheribuild/projects/cross/gmp.py @@ -30,9 +30,12 @@ class BuildGmp(CrossCompileAutotoolsProject): repository = MercurialRepository("https://gmplib.org/repo/gmp") - supported_architectures = (CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID + - CompilationTargets.ALL_CHERIBSD_HYBRID_FOR_PURECAP_ROOTFS_TARGETS + - CompilationTargets.ALL_SUPPORTED_FREEBSD_TARGETS + CompilationTargets.ALL_NATIVE) + supported_architectures = ( + CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID + + CompilationTargets.ALL_CHERIBSD_HYBRID_FOR_PURECAP_ROOTFS_TARGETS + + CompilationTargets.ALL_SUPPORTED_FREEBSD_TARGETS + + CompilationTargets.ALL_NATIVE + ) native_install_dir = DefaultInstallDir.CHERI_SDK def check_system_dependencies(self) -> None: diff --git a/pycheribuild/projects/cross/juliet_test_suite.py b/pycheribuild/projects/cross/juliet_test_suite.py index 55e5f3ce0..29cc004f6 100644 --- a/pycheribuild/projects/cross/juliet_test_suite.py +++ b/pycheribuild/projects/cross/juliet_test_suite.py @@ -52,8 +52,10 @@ def setup_config_options(cls, **kwargs): def process(self): if self.build_type != BuildType.DEBUG: - self.warning("The Juliet test suite contains undefined behaviour that might be optimized away unless " - "you compile at -O0.") + self.warning( + "The Juliet test suite contains undefined behaviour that might be optimized away unless " + "you compile at -O0." + ) self.ask_for_confirmation("Are you sure you want to continue?") super().process() @@ -112,78 +114,80 @@ def run_tests(self): # For stdin redirection args.append("--test-setup-command=touch /tmp/in.txt") - self.target_info.run_cheribsd_test_script("run_juliet_tests.py", *args, mount_sourcedir=True, - mount_sysroot=True, mount_builddir=True) + self.target_info.run_cheribsd_test_script( + "run_juliet_tests.py", *args, mount_sourcedir=True, mount_sysroot=True, mount_builddir=True + ) class BuildJulietCWE121(BuildJulietCWESubdir): target = "juliet-cwe-121" cwe_number = 121 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE121_Stack_Based_Buffer_Overflow") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE121_Stack_Based_Buffer_Overflow" + ) cwe_setup_commands = [ - "echo 500 > /tmp/in.txt", - ] + "echo 500 > /tmp/in.txt", + ] class BuildJulietCWE122(BuildJulietCWESubdir): target = "juliet-cwe-122" cwe_number = 122 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE122_Heap_Based_Buffer_Overflow") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE122_Heap_Based_Buffer_Overflow" + ) cwe_setup_commands = [ - "echo 500 > /tmp/in.txt", - ] + "echo 500 > /tmp/in.txt", + ] class BuildJulietCWE124(BuildJulietCWESubdir): target = "juliet-cwe-124" cwe_number = 124 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE124_Buffer_Underwrite") + repository = ReuseOtherProjectRepository(BuildJulietTestSuite, subdirectory="testcases/CWE124_Buffer_Underwrite") cwe_setup_commands = [ - "echo -500 > /tmp/in.txt", - ] + "echo -500 > /tmp/in.txt", + ] class BuildJulietCWE126(BuildJulietCWESubdir): target = "juliet-cwe-126" cwe_number = 126 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE126_Buffer_Overread") + repository = ReuseOtherProjectRepository(BuildJulietTestSuite, subdirectory="testcases/CWE126_Buffer_Overread") cwe_setup_commands = [ - "echo 500 > /tmp/in.txt", - ] + "echo 500 > /tmp/in.txt", + ] class BuildJulietCWE127(BuildJulietCWESubdir): target = "juliet-cwe-127" cwe_number = 127 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE127_Buffer_Underread") + repository = ReuseOtherProjectRepository(BuildJulietTestSuite, subdirectory="testcases/CWE127_Buffer_Underread") cwe_setup_commands = [ - "echo -500 > /tmp/in.txt", - ] + "echo -500 > /tmp/in.txt", + ] class BuildJulietCWE134(BuildJulietCWESubdir): target = "juliet-cwe-134" cwe_number = 134 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE134_Uncontrolled_Format_String") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE134_Uncontrolled_Format_String" + ) cwe_warning_flags = ["-Wno-error=format-security"] cwe_setup_commands = [ - "export ADD=%s%d%s", - "echo Format string: %s %d %s > /tmp/file.txt", - "echo Format string: %s %d %s > /tmp/in.txt", - ] + "export ADD=%s%d%s", + "echo Format string: %s %d %s > /tmp/file.txt", + "echo Format string: %s %d %s > /tmp/in.txt", + ] class BuildJulietCWE188(BuildJulietCWESubdir): target = "juliet-cwe-188" cwe_number = 188 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE188_Reliance_on_Data_Memory_Layout") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE188_Reliance_on_Data_Memory_Layout" + ) def setup(self): super().setup() @@ -206,31 +210,33 @@ class BuildJulietCWE416(BuildJulietCWESubdir): class BuildJulietCWE587(BuildJulietCWESubdir): target = "juliet-cwe-587" cwe_number = 587 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE587_Assignment_of_Fixed_Address_to_Pointer") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE587_Assignment_of_Fixed_Address_to_Pointer" + ) class BuildJulietCWE588(BuildJulietCWESubdir): target = "juliet-cwe-588" cwe_number = 588 repository = ReuseOtherProjectRepository( - BuildJulietTestSuite, - subdirectory="testcases/CWE588_Attempt_to_Access_Child_of_Non_Structure_Pointer") + BuildJulietTestSuite, subdirectory="testcases/CWE588_Attempt_to_Access_Child_of_Non_Structure_Pointer" + ) class BuildJulietCWE680(BuildJulietCWESubdir): target = "juliet-cwe-680" cwe_number = 680 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE680_Integer_Overflow_to_Buffer_Overflow") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE680_Integer_Overflow_to_Buffer_Overflow" + ) class BuildJulietCWE685(BuildJulietCWESubdir): target = "juliet-cwe-685" cwe_number = 685 repository = ReuseOtherProjectRepository( - BuildJulietTestSuite, - subdirectory="testcases/CWE685_Function_Call_With_Incorrect_Number_of_Arguments") + BuildJulietTestSuite, subdirectory="testcases/CWE685_Function_Call_With_Incorrect_Number_of_Arguments" + ) cwe_warning_flags = ["-Wno-error=format-insufficient-args"] diff --git a/pycheribuild/projects/cross/kde.py b/pycheribuild/projects/cross/kde.py index a0e022581..71e9bb835 100644 --- a/pycheribuild/projects/cross/kde.py +++ b/pycheribuild/projects/cross/kde.py @@ -90,16 +90,19 @@ def ctest_script_extra_args(self): kde_prefix = self.install_prefix if self.tests_need_full_disk_image: return ["--test-setup-command", ". /build/prefix.sh && env | sort"] - return ["--extra-library-path", "/build/bin", "--extra-library-path", "/build/lib", - # Add the libraries from other frameworks - "--extra-library-path", "/sysroot" + str(self.install_prefix) + "/lib:/sysroot/usr/lib:/sysroot/lib", - # Also need the X11 libraries for most tests - "--extra-library-path", "/sysroot" + str(BuildLibXCB.get_instance(self).install_prefix) + "/lib", - # And of course QtCore/QtTest - "--extra-library-path", "/sysroot" + str(BuildQtBase.get_instance(self).install_prefix) + "/lib", - "--test-setup-command", - f"mkdir -p {kde_prefix.parent} && ln -sn /sysroot{kde_prefix} {kde_prefix}", - "--test-setup-command", ". /build/prefix.sh && env | sort"] + # fmt: off + return [ + "--extra-library-path", "/build/bin", "--extra-library-path", "/build/lib", + # Add the libraries from other frameworks + "--extra-library-path", "/sysroot" + str(self.install_prefix) + "/lib:/sysroot/usr/lib:/sysroot/lib", + # Also need the X11 libraries for most tests + "--extra-library-path", "/sysroot" + str(BuildLibXCB.get_instance(self).install_prefix) + "/lib", + # And of course QtCore/QtTest + "--extra-library-path", "/sysroot" + str(BuildQtBase.get_instance(self).install_prefix) + "/lib", + "--test-setup-command", + f"mkdir -p {kde_prefix.parent} && ln -sn /sysroot{kde_prefix} {kde_prefix}", + "--test-setup-command", ". /build/prefix.sh && env | sort" + ] # fmt: on def setup(self): super().setup() @@ -117,10 +120,17 @@ def setup(self): # We need native tools (e.g. desktoptojson/kconfig_compiler) for some projects native_project = BuildKCoreAddons.get_instance(self, cross_target=CompilationTargets.NATIVE) self.add_cmake_options( - KF5_HOST_TOOLING=native_project.install_dir / native_project.target_info.default_libdir / "cmake") - dep_names = " ".join(x.name for x in self._direct_dependencies(self.config, include_sdk_dependencies=False, - include_toolchain_dependencies=False, - explicit_dependencies_only=True)) + KF5_HOST_TOOLING=native_project.install_dir / native_project.target_info.default_libdir / "cmake" + ) + dep_names = " ".join( + x.name + for x in self._direct_dependencies( + self.config, + include_sdk_dependencies=False, + include_toolchain_dependencies=False, + explicit_dependencies_only=True, + ) + ) if "qtx11extras" in dep_names: self.warning("Adding include path as workaround for broken QtX11Extras") self.COMMON_FLAGS.append("-I" + str(BuildLibXCB.get_install_dir(self) / "include")) @@ -138,8 +148,10 @@ def setup(self): self.add_cmake_options(CMAKE_DISABLE_FIND_PACKAGE_KF5DocTools=True) def run_tests(self): - self.info("To debug failing tests, you can increase verbosity by setting", - coloured(AnsiColour.yellow, 'export QT_LOGGING_RULES="*.debug=true"')) + self.info( + "To debug failing tests, you can increase verbosity by setting", + coloured(AnsiColour.yellow, 'export QT_LOGGING_RULES="*.debug=true"'), + ) super().run_tests() @@ -178,16 +190,18 @@ class BuildGettext(CrossCompileAutotoolsProject): def setup(self): super().setup() - self.configure_args.extend([ - "--enable-relocatable", - "--disable-csharp", - "--disable-java", - "--disable-libasprintf", - "--disable-openmp", - "--without-emacs", - "--with-included-gettext", - "ac_cv_lib_rt_sched_yield=no", - ]) + self.configure_args.extend( + [ + "--enable-relocatable", + "--disable-csharp", + "--disable-java", + "--disable-libasprintf", + "--disable-openmp", + "--without-emacs", + "--with-included-gettext", + "ac_cv_lib_rt_sched_yield=no", + ] + ) def clean(self): if not (self.source_dir / "Makefile").exists(): @@ -204,9 +218,13 @@ def process(self): new_env = dict() if OSInfo.IS_MAC: # /usr/bin/bison and /usr/bin/sed on macOS are not compatible with this build system - new_env["PATH"] = ":".join([str(self.get_homebrew_prefix("gnu-sed") / "libexec/gnubin"), - str(self.get_homebrew_prefix("bison") / "bin"), - os.getenv("PATH")]) + new_env["PATH"] = ":".join( + [ + str(self.get_homebrew_prefix("gnu-sed") / "libexec/gnubin"), + str(self.get_homebrew_prefix("bison") / "bin"), + os.getenv("PATH"), + ] + ) with self.set_env(**new_env): super().process() @@ -226,8 +244,13 @@ class BuildKWayland(KDECMakeProject): @classmethod def dependencies(cls, config) -> "tuple[str, ...]": - result = (*super().dependencies(config), "libglvnd", "wayland-protocols", "qtwayland", - "plasma-wayland-protocols") + result = ( + *super().dependencies(config), + "libglvnd", + "wayland-protocols", + "qtwayland", + "plasma-wayland-protocols", + ) if cls.get_crosscompile_target().target_info_cls.is_freebsd(): result += ("linux-input-h",) return result @@ -278,10 +301,14 @@ def install(self, **kwargs): if not self.compiling_for_host(): # TODO: should probably just install Qt and KDE files in the same directory install_prefix = self.install_prefix - all_prefixes = " ".join(shlex.quote("/" + str(s.relative_to(self.rootfs_dir))) for s in - self.dependency_install_prefixes) - self.write_file(self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-common", - overwrite=True, mode=0o755, contents=f"""#!/bin/sh + all_prefixes = " ".join( + shlex.quote("/" + str(s.relative_to(self.rootfs_dir))) for s in self.dependency_install_prefixes + ) + self.write_file( + self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-common", + overwrite=True, + mode=0o755, + contents=f"""#!/bin/sh set -xe # QML disk caching is currently broken export QML_DISABLE_DISK_CACHE=1 @@ -342,21 +369,33 @@ def install(self, **kwargs): else exec sh fi; -""") - self.write_file(self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-x11", - overwrite=True, mode=0o755, contents=f"""#!/bin/sh +""", + ) + self.write_file( + self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-x11", + overwrite=True, + mode=0o755, + contents=f"""#!/bin/sh export DISPLAY=:0 export QT_QPA_PLATFORM=xcb exec "/{self.target_info.localbase / "bin/kde-shell-common"}" -""") - self.write_file(self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-wayland", - overwrite=True, mode=0o755, contents=f"""#!/bin/sh +""", + ) + self.write_file( + self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-wayland", + overwrite=True, + mode=0o755, + contents=f"""#!/bin/sh export QT_QPA_PLATFORM=wayland exec "/{self.target_info.localbase / "bin/kde-shell-common"}" -""") - - self.write_file(self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-x11-smbfs", - overwrite=True, mode=0o755, contents=f"""#!/bin/sh +""", + ) + + self.write_file( + self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-x11-smbfs", + overwrite=True, + mode=0o755, + contents=f"""#!/bin/sh set -xe if df -t smbfs,nfs "{install_prefix}" >/dev/null 2>/dev/null; then echo "{install_prefix} is already mounted from the host, skipping" @@ -367,7 +406,8 @@ def install(self, **kwargs): fi set +xe exec "/{self.target_info.localbase / "bin/kde-shell-x11"}" -""") +""", + ) class BuildKConfig(KDECMakeProject): @@ -523,8 +563,15 @@ class BuildKNotifications(KDECMakeProject): @classmethod def dependencies(cls, config) -> "tuple[str, ...]": - result = ("qtdeclarative", "kwindowsystem", "kconfig", "kconfig-native", "kcoreaddons", "kcoreaddons-native", - "phonon") + result = ( + "qtdeclarative", + "kwindowsystem", + "kconfig", + "kconfig-native", + "kcoreaddons", + "kcoreaddons-native", + "phonon", + ) if cls.get_crosscompile_target().target_info_cls.is_macos(): return (*result, "qtmacextras") return (*result, "qtx11extras") @@ -533,7 +580,8 @@ def dependencies(cls, config) -> "tuple[str, ...]": class BuildKPackage(KDECMakeProject): dependencies = ("karchive", "ki18n", "kcoreaddons", "kcoreaddons-native") repository = KF5GitRepository( - "https://invent.kde.org/frameworks/kpackage.git", old_urls=[b"https://invent.kde.org/arichardson/kpackage.git"], + "https://invent.kde.org/frameworks/kpackage.git", + old_urls=[b"https://invent.kde.org/arichardson/kpackage.git"], ) @@ -563,8 +611,15 @@ class BuildKBookmarks(KDECMakeProject): class BuildKCMUtils(KDECMakeProject): - dependencies = ("kitemviews", "kconfigwidgets", "kservice", "kxmlgui", "kdeclarative", "kauth", - "kcmutils-tools-native") + dependencies = ( + "kitemviews", + "kconfigwidgets", + "kservice", + "kxmlgui", + "kdeclarative", + "kauth", + "kcmutils-tools-native", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kcmutils.git") @@ -581,8 +636,16 @@ def setup(self): class BuildKConfigWidgets(KDECMakeProject): - dependencies = ("kauth", "kcoreaddons", "kcodecs", "kconfig", "kguiaddons", "ki18n", "kwidgetsaddons", - "kconfig-native") + dependencies = ( + "kauth", + "kcoreaddons", + "kcodecs", + "kconfig", + "kguiaddons", + "ki18n", + "kwidgetsaddons", + "kconfig-native", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kconfigwidgets.git") _has_qt_designer_plugin = True @@ -595,17 +658,34 @@ class BuildKConfigWidgets(KDECMakeProject): # frameworks/kemoticons: frameworks/kservice # frameworks/kjs: frameworks/kdoctools class BuildKNewStuff(KDECMakeProject): - dependencies = ("attica", "kitemviews", "kiconthemes", "ktextwidgets", "kxmlgui", - "solid", "kio", "kbookmarks", "kpackage", "kpackage-native", "ksyndication", "kirigami") + dependencies = ( + "attica", + "kitemviews", + "kiconthemes", + "ktextwidgets", + "kxmlgui", + "solid", + "kio", + "kbookmarks", + "kpackage", + "kpackage-native", + "ksyndication", + "kirigami", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/knewstuff.git") _needs_newer_bison = True class BuildKService(KDECMakeProject): - dependencies = ("kconfig", "kcoreaddons", "kcrash", "kdbusaddons", "ki18n", - "kcoreaddons-native", # desktoptojson - "kconfig-native", # kconfig_compiler - ) + dependencies = ( + "kconfig", + "kcoreaddons", + "kcrash", + "kdbusaddons", + "ki18n", + "kcoreaddons-native", # desktoptojson + "kconfig-native", # kconfig_compiler + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kservice.git") _needs_newer_bison = True @@ -640,24 +720,50 @@ def dependencies(cls, config) -> "tuple[str, ...]": class BuildKXMLGUI(KDECMakeProject): - dependencies = ("kitemviews", "kconfig", "kconfig-native", "kglobalaccel", - "kconfigwidgets", "ki18n", "kiconthemes", - "ktextwidgets", "kwidgetsaddons", "kwindowsystem") + dependencies = ( + "kitemviews", + "kconfig", + "kconfig-native", + "kglobalaccel", + "kconfigwidgets", + "ki18n", + "kiconthemes", + "ktextwidgets", + "kwidgetsaddons", + "kwindowsystem", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kxmlgui.git") _has_qt_designer_plugin = True class BuildKDeclarative(KDECMakeProject): repository = KF5GitRepository("https://invent.kde.org/frameworks/kdeclarative.git") - dependencies = ("kpackage", "kpackage-native", "kio", "kiconthemes", "knotifications", "qtdeclarative", "kio", - "libepoxy") + dependencies = ( + "kpackage", + "kpackage-native", + "kio", + "kiconthemes", + "knotifications", + "qtdeclarative", + "kio", + "libepoxy", + ) _has_qt_designer_plugin = True class BuildKInit(KDECMakeProject): target = "kinit" - dependencies = ("kio", "kservice", "kcrash", "kjobwidgets", "solid", "kdbusaddons", "kwindowsystem", "libx11", - "libxcb") + dependencies = ( + "kio", + "kservice", + "kcrash", + "kjobwidgets", + "solid", + "kdbusaddons", + "kwindowsystem", + "libx11", + "libxcb", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kinit.git") @@ -675,12 +781,33 @@ class BuildKDED(KDECMakeProject): class BuildKIO(KDECMakeProject): target = "kio" - dependencies = ("kauth", "kdbusaddons", "ki18n", "kguiaddons", "kconfigwidgets", "kitemviews", "kcoreaddons", - "kwidgetsaddons", "kservice", "karchive", "qtx11extras", "solid", - "kjobwidgets", "kiconthemes", "kwindowsystem", "kcrash", "kcompletion", "ktextwidgets", - "kxmlgui", "kbookmarks", "kconfig", "kconfig-native", "knotifications", "kded", - # optional: "kwallet" - ) + dependencies = ( + "kauth", + "kdbusaddons", + "ki18n", + "kguiaddons", + "kconfigwidgets", + "kitemviews", + "kcoreaddons", + "kwidgetsaddons", + "kservice", + "karchive", + "qtx11extras", + "solid", + "kjobwidgets", + "kiconthemes", + "kwindowsystem", + "kcrash", + "kcompletion", + "ktextwidgets", + "kxmlgui", + "kbookmarks", + "kconfig", + "kconfig-native", + "knotifications", + "kded", + # optional: "kwallet" + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kio.git") _has_qt_designer_plugin = True @@ -759,9 +886,11 @@ class BuildKioExtras(KDECMakeProject): # This includes e.g. the thumbnail provider for dolphin target = "kio-extras" dependencies = ("kio", "ksyntaxhighlighting") - repository = GitRepository("https://invent.kde.org/network/kio-extras.git", - temporary_url_override="https://invent.kde.org/arichardson/kio-extras.git", - url_override_reason="https://invent.kde.org/network/kio-extras/-/merge_requests/110") + repository = GitRepository( + "https://invent.kde.org/network/kio-extras.git", + temporary_url_override="https://invent.kde.org/arichardson/kio-extras.git", + url_override_reason="https://invent.kde.org/network/kio-extras/-/merge_requests/110", + ) def setup(self): super().setup() @@ -802,15 +931,29 @@ class BuildKirigami(KDECMakeProject): target = "kirigami" dependencies = ("qtquickcontrols2", "extra-cmake-modules", "qtgraphicaleffects") repository = KF5GitRepository( - "https://invent.kde.org/frameworks/kirigami.git", old_urls=[b"https://invent.kde.org/arichardson/kirigami.git"], + "https://invent.kde.org/frameworks/kirigami.git", + old_urls=[b"https://invent.kde.org/arichardson/kirigami.git"], ) class BuildPlasmaFramework(KDECMakeProject): target = "plasma-framework" - dependencies = ("kio", "kconfigwidgets", "kactivities", "kdbusaddons", "kglobalaccel", "kpackage", "kdeclarative", - "qtquickcontrols", "qtquickcontrols2", "kxmlgui", "threadweaver", "kirigami", "kwayland", - "libglvnd") + dependencies = ( + "kio", + "kconfigwidgets", + "kactivities", + "kdbusaddons", + "kglobalaccel", + "kpackage", + "kdeclarative", + "qtquickcontrols", + "qtquickcontrols2", + "kxmlgui", + "threadweaver", + "kirigami", + "kwayland", + "libglvnd", + ) repository = KF5GitRepository( "https://invent.kde.org/frameworks/plasma-framework.git", old_urls=[b"https://invent.kde.org/arichardson/plasma-framework.git"], @@ -838,8 +981,17 @@ class BuildKFrameworkIntegration(KDECMakeProject): class BuildBreezeStyle(KDECMakeProject): target = "breeze" repository = KDEPlasmaGitRepository("https://invent.kde.org/plasma/breeze.git") - dependencies = ("kdecoration", "kconfig", "kcoreaddons", "kguiaddons", "kiconthemes", "kconfigwidgets", - "kwindowsystem", "kcmutils", "kframeworkintegration") + dependencies = ( + "kdecoration", + "kconfig", + "kcoreaddons", + "kguiaddons", + "kiconthemes", + "kconfigwidgets", + "kwindowsystem", + "kcmutils", + "kframeworkintegration", + ) class BuildKIdleTime(KDECMakeProject): @@ -885,8 +1037,16 @@ class BuildKScreenLocker(KDECMakeProject): class BuildKDECliTools(KDECMakeProject): target = "kde-cli-tools" repository = KDEPlasmaGitRepository("https://invent.kde.org/plasma/kde-cli-tools.git") - dependencies = ("kconfig", "kiconthemes", "ki18n", "kcmutils", "kio", "kservice", "kwindowsystem", - "kactivities") # optional: "kdesu" + dependencies = ( + "kconfig", + "kiconthemes", + "ki18n", + "kcmutils", + "kio", + "kservice", + "kwindowsystem", + "kactivities", + ) # optional: "kdesu" class BuildKWin(KDECMakeProject): @@ -905,8 +1065,21 @@ class BuildKWin(KDECMakeProject): @classmethod def dependencies(cls, config) -> "tuple[str, ...]": - result = (*super().dependencies(config), "kdecoration", "qtx11extras", "breeze", "kcmutils", "kscreenlocker", - "plasma-framework", "libinput", "qttools", "libepoxy", "libxcvt", "lcms2", "krunner") + result = ( + *super().dependencies(config), + "kdecoration", + "qtx11extras", + "breeze", + "kcmutils", + "kscreenlocker", + "plasma-framework", + "libinput", + "qttools", + "libepoxy", + "libxcvt", + "lcms2", + "krunner", + ) if cls.use_mesa: result += ("mesa", "libdrm") if cls.get_crosscompile_target().target_info_cls.is_freebsd(): @@ -920,10 +1093,14 @@ def setup(self): self.add_cmake_options(KWIN_BUILD_DRM_BACKEND=self.use_mesa, KWIN_BUILD_WAYLAND_EGL=self.use_mesa) if not self.compiling_for_host(): # We need to find the host Qt libraries for qwaylandscanner_kde. - self.add_cmake_options(NATIVE_PREFIX=";".join([ - str(BuildQtBase.get_install_dir(self, cross_target=CompilationTargets.NATIVE)), - str(BuildKCoreAddons.get_install_dir(self, cross_target=CompilationTargets.NATIVE)), - ])) + self.add_cmake_options( + NATIVE_PREFIX=";".join( + [ + str(BuildQtBase.get_install_dir(self, cross_target=CompilationTargets.NATIVE)), + str(BuildKCoreAddons.get_install_dir(self, cross_target=CompilationTargets.NATIVE)), + ] + ) + ) class BuildLibKScreen(KDECMakeProject): @@ -973,11 +1150,38 @@ class BuildPlasmaWorkspace(KDECMakeProject): _uses_wayland_scanner = True repository = KDEPlasmaGitRepository( "https://invent.kde.org/plasma/plasma-workspace.git", - old_urls=[b"https://invent.kde.org/arichardson/plasma-workspace.git"]) - dependencies = ("xprop", "xsetroot", "plasma-framework", "kwin", "breeze", "kidletime", "kitemmodels", "kcmutils", - "knotifyconfig", "kded", "kinit", "kscreenlocker", "libkscreen", "libxft", "libxtst", "kpeople", - "kparts", "prison", "krunner", "kactivities-stats", "libksysguard", "kunitconversion", "kwallet", - "ktexteditor", "kwayland", "layer-shell-qt", "kquickcharts", "kactivitymanagerd") + old_urls=[b"https://invent.kde.org/arichardson/plasma-workspace.git"], + ) + dependencies = ( + "xprop", + "xsetroot", + "plasma-framework", + "kwin", + "breeze", + "kidletime", + "kitemmodels", + "kcmutils", + "knotifyconfig", + "kded", + "kinit", + "kscreenlocker", + "libkscreen", + "libxft", + "libxtst", + "kpeople", + "kparts", + "prison", + "krunner", + "kactivities-stats", + "libksysguard", + "kunitconversion", + "kwallet", + "ktexteditor", + "kwayland", + "layer-shell-qt", + "kquickcharts", + "kactivitymanagerd", + ) def setup(self): super().setup() @@ -1005,9 +1209,17 @@ class BuildPlasmaDesktop(KDECMakeProject): repository = KDEPlasmaGitRepository( "https://invent.kde.org/plasma/plasma-desktop.git", temporary_url_override="https://invent.kde.org/arichardson/plasma-desktop.git", - url_override_reason="https://invent.kde.org/plasma/plasma-desktop/-/merge_requests/944") - dependencies = ("plasma-workspace", "kirigami", "krunner", "kwallet", "qqc2-desktop-style", - "libxkbfile", "xkeyboard-config") + url_override_reason="https://invent.kde.org/plasma/plasma-desktop/-/merge_requests/944", + ) + dependencies = ( + "plasma-workspace", + "kirigami", + "krunner", + "kwallet", + "qqc2-desktop-style", + "libxkbfile", + "xkeyboard-config", + ) def setup(self): super().setup() @@ -1031,8 +1243,12 @@ class BuildDoplhin(KDECMakeProject): class BuildLibPng(CrossCompileCMakeProject): supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE - repository = GitRepository("https://github.com/glennrp/libpng", old_urls=[b"https://github.com/CTSRD-CHERI/libpng"], - default_branch="libpng16", force_branch=True) + repository = GitRepository( + "https://github.com/glennrp/libpng", + old_urls=[b"https://github.com/CTSRD-CHERI/libpng"], + default_branch="libpng16", + force_branch=True, + ) target = "libpng" # The tests take a really long time to run (~2.5 hours on purecap RISC-V) ctest_script_extra_args = ("--test-timeout", 5 * 60 * 60) @@ -1085,8 +1301,17 @@ class BuildKImageAnnotator(KDECMakeProject): class BuildGwenview(KDECMakeProject): target = "gwenview" - dependencies = ("qtsvg", "kitemmodels", "kimageformats", "kio", "kparts", "lcms2", "libpng", "exiv2", - "kimageannotator") + dependencies = ( + "qtsvg", + "kitemmodels", + "kimageformats", + "kio", + "kparts", + "lcms2", + "libpng", + "exiv2", + "kimageannotator", + ) repository = GitRepository("https://invent.kde.org/graphics/gwenview.git") @@ -1105,8 +1330,10 @@ def setup(self): class BuildPoppler(CrossCompileCMakeProject): target = "poppler" dependencies = ("freetype2", "fontconfig", "qtbase", "libtiff", "openjpeg") - repository = GitRepository("https://gitlab.freedesktop.org/poppler/poppler.git", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/poppler.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/poppler/poppler.git", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/poppler.git"], + ) def setup(self): super().setup() @@ -1123,10 +1350,16 @@ def update(self): @property def ctest_script_extra_args(self): - return ["--extra-library-path", "/build/bin", - "--extra-library-path", "/build/lib", - "--extra-library-path", "/sysroot" + str(self.install_prefix) + "/lib:/sysroot/usr/lib:/sysroot/lib", - "--extra-library-path", "/sysroot" + str(BuildQtBase.get_instance(self).install_prefix) + "/lib"] + return [ + "--extra-library-path", + "/build/bin", + "--extra-library-path", + "/build/lib", + "--extra-library-path", + "/sysroot" + str(self.install_prefix) + "/lib:/sysroot/usr/lib:/sysroot/lib", + "--extra-library-path", + "/sysroot" + str(BuildQtBase.get_instance(self).install_prefix) + "/lib", + ] class BuildThreadWeaver(KDECMakeProject): @@ -1137,7 +1370,8 @@ class BuildThreadWeaver(KDECMakeProject): class BuildKPty(KDECMakeProject): target = "kpty" repository = KF5GitRepository( - "https://invent.kde.org/frameworks/kpty.git", old_urls=[b"https://invent.kde.org/arichardson/kpty"], + "https://invent.kde.org/frameworks/kpty.git", + old_urls=[b"https://invent.kde.org/arichardson/kpty"], ) def setup(self): @@ -1149,10 +1383,32 @@ def setup(self): class BuildKonsole(KDECMakeProject): target = "konsole" repository = GitRepository("https://invent.kde.org/utilities/konsole.git") - dependencies = ("extra-cmake-modules", "kbookmarks", "kconfig", "kconfigwidgets", "kcoreaddons", "kcrash", - "kglobalaccel", "kguiaddons", "kdbusaddons", "ki18n", "kiconthemes", "kio", "knewstuff", - "knotifications", "knotifyconfig", "kparts", "kpty", "kservice", "ktextwidgets", "kwidgetsaddons", - "kwindowsystem", "kxmlgui", "qtbase", "qtmultimedia") + dependencies = ( + "extra-cmake-modules", + "kbookmarks", + "kconfig", + "kconfigwidgets", + "kcoreaddons", + "kcrash", + "kglobalaccel", + "kguiaddons", + "kdbusaddons", + "ki18n", + "kiconthemes", + "kio", + "knewstuff", + "knotifications", + "knotifyconfig", + "kparts", + "kpty", + "kservice", + "ktextwidgets", + "kwidgetsaddons", + "kwindowsystem", + "kxmlgui", + "qtbase", + "qtmultimedia", + ) # TODO: fails to build due to exiv2 usage of auto_ptr @@ -1169,8 +1425,9 @@ class BuildOkular(KDECMakeProject): target = "okular" dependencies = ("poppler", "threadweaver", "kparts", "kio", "kiconthemes", "kpty", "kwallet", "libtiff") # TODO: after the next exiv2 release add "libkexiv2" (currently fails to build due to auto_ptr). - repository = GitRepository("https://invent.kde.org/graphics/okular.git", - old_urls=[b"https://invent.kde.org/arichardson/okular.git"]) + repository = GitRepository( + "https://invent.kde.org/graphics/okular.git", old_urls=[b"https://invent.kde.org/arichardson/okular.git"] + ) def setup(self): super().setup() @@ -1181,7 +1438,8 @@ def setup(self): "KF5Purpose", # Could add this framework (only required for enabling the share menu). "Qt5TextToSpeech", # We don't need speech features. "LibSpectre", # We don't need postscript support. - "CHM", "LibZip", # We don't need CHM support. + "CHM", + "LibZip", # We don't need CHM support. "DjVuLibre", # We don't need DjVu support. "Discount", # We don't need markdown support. "EPub", # We don't need EPub support. @@ -1195,22 +1453,64 @@ def setup(self): class BuildKTextEditor(KDECMakeProject): target = "ktexteditor" repository = KF5GitRepository("https://invent.kde.org/frameworks/ktexteditor.git") - dependencies = ("editorconfig-core-c", "extra-cmake-modules", "karchive", "kauth", "kconfig", "kguiaddons", "ki18n", - "kio", "kparts", "ksyntaxhighlighting", "ktextwidgets", "sonnet", "qtbase", "qtdeclarative") + dependencies = ( + "editorconfig-core-c", + "extra-cmake-modules", + "karchive", + "kauth", + "kconfig", + "kguiaddons", + "ki18n", + "kio", + "kparts", + "ksyntaxhighlighting", + "ktextwidgets", + "sonnet", + "qtbase", + "qtdeclarative", + ) class BuildKate(KDECMakeProject): target = "kate" repository = GitRepository("https://invent.kde.org/utilities/kate.git") - dependencies = ("extra-cmake-modules", "kactivities", "kconfig", "kcoreaddons", "kcrash", "kdbusaddons", - "kguiaddons", "ki18n", "kiconthemes", "kitemmodels", "kitemviews", "knewstuff", "kwallet", - "ksyntaxhighlighting", "ktexteditor", "ktextwidgets", "kwindowsystem", "qtbase") + dependencies = ( + "extra-cmake-modules", + "kactivities", + "kconfig", + "kcoreaddons", + "kcrash", + "kdbusaddons", + "kguiaddons", + "ki18n", + "kiconthemes", + "kitemmodels", + "kitemviews", + "knewstuff", + "kwallet", + "ksyntaxhighlighting", + "ktexteditor", + "ktextwidgets", + "kwindowsystem", + "qtbase", + ) class BuildKDEX11Desktop(TargetAliasWithDependencies): target = "kde-x11-desktop" supported_architectures = CompilationTargets.ALL_SUPPORTED_CHERIBSD_AND_HOST_TARGETS # TODO: "systemsettings" - now needs a newer plasma-workspace - dependencies = ("plasma-desktop", "dolphin", "konsole", "okular", "gwenview", "kate", - "xvnc-server", "systemsettings", - "xeyes", "twm", "xev", "xauth") # Add some basic X11 things as a fallback + dependencies = ( + "plasma-desktop", + "dolphin", + "konsole", + "okular", + "gwenview", + "kate", + "xvnc-server", + "systemsettings", + "xeyes", + "twm", + "xev", + "xauth", + ) # Add some basic X11 things as a fallback diff --git a/pycheribuild/projects/cross/littlekernel.py b/pycheribuild/projects/cross/littlekernel.py index 47da0a42a..35ec64563 100644 --- a/pycheribuild/projects/cross/littlekernel.py +++ b/pycheribuild/projects/cross/littlekernel.py @@ -44,9 +44,11 @@ class BuildLittleKernel(CrossCompileMakefileProject): CompilationTargets.FREESTANDING_RISCV64, CompilationTargets.FREESTANDING_RISCV64_PURECAP, ) - repository = GitRepository("https://github.com/littlekernel/lk", - temporary_url_override="https://github.com/arichardson/lk.git", - url_override_reason="Fixes to allow building with Clang") + repository = GitRepository( + "https://github.com/littlekernel/lk", + temporary_url_override="https://github.com/arichardson/lk.git", + url_override_reason="Fixes to allow building with Clang", + ) default_install_dir = DefaultInstallDir.DO_NOT_INSTALL set_pkg_config_path = False needs_sysroot = False @@ -73,15 +75,19 @@ def build_dir_suffix(self): def compiler_rt_builtins_path(self) -> Path: path = BuildCompilerRtBuiltins.get_build_dir(self) / f"lib/baremetal/libclang_rt.builtins-{self.triple_arch}.a" if not path.exists(): - self.dependency_error("Compiler builtins library", path, "does not exist", - cheribuild_target="compiler-rt-builtins") + self.dependency_error( + "Compiler builtins library", path, "does not exist", cheribuild_target="compiler-rt-builtins" + ) return path @property def essential_compiler_and_linker_flags(self) -> "list[str]": if self.compiling_for_cheri(): - return [*self.target_info.get_essential_compiler_and_linker_flags(softfloat=False), - "-Werror=cheri-capability-misuse", "-Werror=shorten-cap-to-int"] + return [ + *self.target_info.get_essential_compiler_and_linker_flags(softfloat=False), + "-Werror=cheri-capability-misuse", + "-Werror=shorten-cap-to-int", + ] return ["--target=" + self.target_info.target_triple] def setup(self): @@ -103,8 +109,11 @@ def setup(self): self.set_make_cmd_with_args("LD", self.target_info.linker, ["--unresolved-symbols=report-all"]) if self.crosscompile_target.is_riscv(include_purecap=True) and self.use_mmu: self.make_args.set(RISCV_MMU="sv39", RISCV_MODE="supervisor") - self.make_args.set(TOOLCHAIN_PREFIX=toolchain_prefix, ARCH_arm64_TOOLCHAIN_PREFIX=toolchain_prefix, - ARCH_riscv64_TOOLCHAIN_PREFIX=toolchain_prefix) + self.make_args.set( + TOOLCHAIN_PREFIX=toolchain_prefix, + ARCH_arm64_TOOLCHAIN_PREFIX=toolchain_prefix, + ARCH_riscv64_TOOLCHAIN_PREFIX=toolchain_prefix, + ) def setup_late(self) -> None: super().setup_late() @@ -121,16 +130,43 @@ def kernel_path(self) -> Path: def run_tests(self): if self.compiling_for_aarch64(include_purecap=True): - cmd = [self.config.qemu_bindir / "qemu-system-aarch64", "-cpu", - "cortex-a53", "-m", "512", "-smp", "1", "-machine", "virt", - "-net", "none", "-nographic", - "-kernel", self.kernel_path] + cmd = [ + self.config.qemu_bindir / "qemu-system-aarch64", + "-cpu", + "cortex-a53", + "-m", + "512", + "-smp", + "1", + "-machine", + "virt", + "-net", + "none", + "-nographic", + "-kernel", + self.kernel_path, + ] elif self.compiling_for_riscv(include_purecap=True): bios_args = ["-bios", "none"] if self.use_mmu: bios_args = riscv_bios_arguments(self.crosscompile_target, self) - cmd = [self.config.qemu_bindir / "qemu-system-riscv64cheri", "-cpu", "rv64", "-m", "512", "-smp", "1", - "-machine", "virt", "-net", "none", "-nographic", "-kernel", self.kernel_path, *bios_args] + cmd = [ + self.config.qemu_bindir / "qemu-system-riscv64cheri", + "-cpu", + "rv64", + "-m", + "512", + "-smp", + "1", + "-machine", + "virt", + "-net", + "none", + "-nographic", + "-kernel", + self.kernel_path, + *bios_args, + ] else: return self.fatal("Unsupported arch") self.run_cmd(cmd, cwd=self.build_dir, give_tty_control=True) diff --git a/pycheribuild/projects/cross/llvm_test_suite.py b/pycheribuild/projects/cross/llvm_test_suite.py index a537082cf..2aa9262e0 100644 --- a/pycheribuild/projects/cross/llvm_test_suite.py +++ b/pycheribuild/projects/cross/llvm_test_suite.py @@ -93,7 +93,8 @@ def setup(self): TEST_SUITE_LIT=self.llvm_lit, TEST_SUITE_COLLECT_CODE_SIZE=self.collect_stats, TEST_SUITE_COLLECT_COMPILE_TIME=self.collect_stats, - TEST_SUITE_COLLECT_STATS=self.collect_stats) + TEST_SUITE_COLLECT_STATS=self.collect_stats, + ) if self.compiling_for_host() and self.target_info.is_linux() and shutil.which("perf") is not None: self.add_cmake_options(TEST_SUITE_USE_PERF=True) @@ -103,8 +104,9 @@ def setup(self): self.add_cmake_options(TEST_SUITE_CXX_LIBRARY="-lc++;-lgcc_eh") self.add_cmake_options(BENCHMARK_USE_LIBCXX=True) if self.can_run_binaries_on_remote_morello_board(): - self.add_cmake_options(TEST_SUITE_RUN_BENCHMARKS=True, - TEST_SUITE_REMOTE_HOST=self.config.remote_morello_board) + self.add_cmake_options( + TEST_SUITE_RUN_BENCHMARKS=True, TEST_SUITE_REMOTE_HOST=self.config.remote_morello_board + ) else: self.add_cmake_options(TEST_SUITE_RUN_BENCHMARKS=False) # Would need to set up custom executor if self.crosscompile_target.is_any_x86(): @@ -148,8 +150,10 @@ class BuildLLVMTestSuiteCheriBSDUpstreamLLVM(BuildLLVMTestSuite): @property def custom_c_preprocessor(self): - return self.llvm_project.get_install_dir( - self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin/clang-cpp" + return ( + self.llvm_project.get_install_dir(self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) + / "bin/clang-cpp" + ) @property def custom_c_compiler(self): @@ -157,5 +161,6 @@ def custom_c_compiler(self): @property def custom_cxx_compiler(self): - return self.llvm_project.get_install_dir( - self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin/clang++" + return ( + self.llvm_project.get_install_dir(self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin/clang++" + ) diff --git a/pycheribuild/projects/cross/mesa.py b/pycheribuild/projects/cross/mesa.py index 11835cdf1..8abf68ac7 100644 --- a/pycheribuild/projects/cross/mesa.py +++ b/pycheribuild/projects/cross/mesa.py @@ -35,9 +35,11 @@ class BuildLibDrm(CrossCompileMesonProject): target = "libdrm" dependencies = ("libpciaccess", "xorg-pthread-stubs") - repository = GitRepository("https://gitlab.freedesktop.org/mesa/drm.git", - temporary_url_override="https://gitlab.freedesktop.org/arichardson/drm.git", - url_override_reason="Lots of uinptr_t != u64 fun") + repository = GitRepository( + "https://gitlab.freedesktop.org/mesa/drm.git", + temporary_url_override="https://gitlab.freedesktop.org/arichardson/drm.git", + url_override_reason="Lots of uinptr_t != u64 fun", + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def setup(self): @@ -55,8 +57,10 @@ def setup(self): class BuildLibGlvnd(CrossCompileMesonProject): target = "libglvnd" dependencies = ("libx11", "libxext") - repository = GitRepository("https://gitlab.freedesktop.org/glvnd/libglvnd.git", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/libglvnd.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/glvnd/libglvnd.git", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/libglvnd.git"], + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def setup(self): @@ -73,11 +77,14 @@ def setup(self): class BuildMesa(CrossCompileMesonProject): target = "mesa" - repository = GitRepository("https://gitlab.freedesktop.org/mesa/mesa.git", - temporary_url_override="https://github.com/CTSRD-CHERI/mesa", - url_override_reason="Various changes to allow purecap compilation", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/mesa.git"], - force_branch=True, default_branch="mesa-22.3.7-cheriabi") + repository = GitRepository( + "https://gitlab.freedesktop.org/mesa/mesa.git", + temporary_url_override="https://github.com/CTSRD-CHERI/mesa", + url_override_reason="Various changes to allow purecap compilation", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/mesa.git"], + force_branch=True, + default_branch="mesa-22.3.7-cheriabi", + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE include_x11 = True include_wayland = True @@ -96,8 +103,9 @@ def check_system_dependencies(self): try: self.run_cmd(sys.executable, "-c", "import mako") except subprocess.CalledProcessError: - self.dependency_error("Missing python module mako", - install_instructions=InstallInstructions("pip3 install --user mako")) + self.dependency_error( + "Missing python module mako", install_instructions=InstallInstructions("pip3 install --user mako") + ) def setup(self): super().setup() @@ -116,8 +124,16 @@ def setup(self): meson_args["gallium-drivers"].append("panfrost") # Does not compile yet: meson_args["vulkan-drivers"].append("panfrost") meson_args = {k: str(v) for k, v in meson_args.items()} - self.add_meson_options(gbm="enabled", egl="enabled", glvnd=True, llvm="disabled", osmesa=False, - platforms=str(platforms), _include_empty_vars=True, **meson_args) + self.add_meson_options( + gbm="enabled", + egl="enabled", + glvnd=True, + llvm="disabled", + osmesa=False, + platforms=str(platforms), + _include_empty_vars=True, + **meson_args, + ) # threads_posix.h:274:13: error: releasing mutex 'mtx' that was not held [-Werror,-Wthread-safety-analysis] self.cross_warning_flags.append("-Wno-thread-safety-analysis") # There are quite a lot of -Wcheri-capability-misuse warnings, but for now we just want the library to exist @@ -131,8 +147,9 @@ def setup(self): class BuildLibEpoxy(CrossCompileMesonProject): target = "libepoxy" dependencies = ("libglvnd",) - repository = GitRepository("https://github.com/anholt/libepoxy", - old_urls=[b"https://github.com/arichardson/libepoxy"]) + repository = GitRepository( + "https://github.com/anholt/libepoxy", old_urls=[b"https://github.com/arichardson/libepoxy"] + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def setup(self): diff --git a/pycheribuild/projects/cross/mpfr.py b/pycheribuild/projects/cross/mpfr.py index 7b52740c9..dbf00be0b 100644 --- a/pycheribuild/projects/cross/mpfr.py +++ b/pycheribuild/projects/cross/mpfr.py @@ -34,9 +34,12 @@ class BuildMpfr(CrossCompileAutotoolsProject): repository = GitRepository("https://gitlab.inria.fr/mpfr/mpfr.git") - supported_architectures = (CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID + - CompilationTargets.ALL_CHERIBSD_HYBRID_FOR_PURECAP_ROOTFS_TARGETS + - CompilationTargets.ALL_SUPPORTED_FREEBSD_TARGETS + CompilationTargets.ALL_NATIVE) + supported_architectures = ( + CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID + + CompilationTargets.ALL_CHERIBSD_HYBRID_FOR_PURECAP_ROOTFS_TARGETS + + CompilationTargets.ALL_SUPPORTED_FREEBSD_TARGETS + + CompilationTargets.ALL_NATIVE + ) native_install_dir = DefaultInstallDir.CHERI_SDK def check_system_dependencies(self) -> None: diff --git a/pycheribuild/projects/cross/mrs.py b/pycheribuild/projects/cross/mrs.py index 04926b8e5..9c2f42b11 100644 --- a/pycheribuild/projects/cross/mrs.py +++ b/pycheribuild/projects/cross/mrs.py @@ -46,32 +46,40 @@ def setup_config_options(cls, **kwargs): cls.build_target = cls.add_config_option("build-target", kind=str, help="specify a target to build, or all") cls.debug = cls.add_bool_option("debug", help="enable debug output") - cls.offload_quarantine = cls.add_bool_option("offload-quarantine", - help="process the quarantine in a separate worker thread") + cls.offload_quarantine = cls.add_bool_option( + "offload-quarantine", help="process the quarantine in a separate worker thread" + ) cls.bypass_quarantine = cls.add_bool_option("bypass-quarantine", help="MADV_FREE freed page-size allocations") cls.clear_on_alloc = cls.add_bool_option("clear-on-alloc", help="zero regions during allocation") cls.clear_on_free = cls.add_bool_option("clear-on-free", help="zero regions as they come out of quarantine") cls.print_stats = cls.add_bool_option("print-stats", help="print heap statistics on exit") cls.print_caprevoke = cls.add_bool_option("print-caprevoke", help="print per-revocation statistics") - cls.concurrent_revocation_passes = cls.add_config_option("concurrent-revocation-passes", kind=int, - help="enable N concurrent revocation passes before " - "the stop-the-world pass") - cls.revoke_on_free = cls.add_bool_option("revoke-on-free", - help="perform revocation on free rather than during allocation " - "routines") + cls.concurrent_revocation_passes = cls.add_config_option( + "concurrent-revocation-passes", + kind=int, + help="enable N concurrent revocation passes before " "the stop-the-world pass", + ) + cls.revoke_on_free = cls.add_bool_option( + "revoke-on-free", help="perform revocation on free rather than during allocation " "routines" + ) cls.just_interpose = cls.add_bool_option("just-interpose", help="just call the real functions") cls.just_bookkeeping = cls.add_bool_option("just-bookkeeping", help="just update data structures") cls.just_quarantine = cls.add_bool_option("just-quarantine", help="do bookkeeping and quarantining") - cls.just_paint_bitmap = cls.add_bool_option("just-paint-bitmap", - help="do bookkeeping, quarantining, and bitmap painting") + cls.just_paint_bitmap = cls.add_bool_option( + "just-paint-bitmap", help="do bookkeeping, quarantining, and bitmap painting" + ) - cls.quarantine_ratio = cls.add_config_option("quarantine-ratio", kind=int, - help="limit the quarantine size to 1/QUARANTINE_RATIO times the " - "size of the heap") - cls.quarantine_highwater = cls.add_config_option("quarantine-highwater", kind=int, - help="limit the quarantine size to QUARANTINE_HIGHWATER " - "bytes (supersedes QUARANTINE_RATIO)") + cls.quarantine_ratio = cls.add_config_option( + "quarantine-ratio", + kind=int, + help="limit the quarantine size to 1/QUARANTINE_RATIO times the " "size of the heap", + ) + cls.quarantine_highwater = cls.add_config_option( + "quarantine-highwater", + kind=int, + help="limit the quarantine size to QUARANTINE_HIGHWATER " "bytes (supersedes QUARANTINE_RATIO)", + ) def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/neomutt.py b/pycheribuild/projects/cross/neomutt.py index 9cbc49771..362ad2c3f 100644 --- a/pycheribuild/projects/cross/neomutt.py +++ b/pycheribuild/projects/cross/neomutt.py @@ -29,15 +29,14 @@ class BuildNeomutt(CrossCompileAutotoolsProject): repository = GitRepository("https://github.com/neomutt/neomutt.git") - dependencies = ('libxml2',) + dependencies = ("libxml2",) def setup(self): super().setup() # neomutt's build system doesn't use autotools, it justs pretends to look the same # - but it doesn't implement the --target option, so we strip it - self.configure_args[:] = [arg for arg in self.configure_args if not arg.startswith('--target=')] + self.configure_args[:] = [arg for arg in self.configure_args if not arg.startswith("--target=")] # enable OpenSSL (in base system), disable internationalisation libs we don't have - self.configure_args.extend(['--disable-nls', '--disable-idn', - '--disable-doc', '--ssl']) + self.configure_args.extend(["--disable-nls", "--disable-idn", "--disable-doc", "--ssl"]) diff --git a/pycheribuild/projects/cross/newlib.py b/pycheribuild/projects/cross/newlib.py index 6a442d1b5..4fa5446a3 100644 --- a/pycheribuild/projects/cross/newlib.py +++ b/pycheribuild/projects/cross/newlib.py @@ -84,7 +84,7 @@ def add_configure_vars(self, **kwargs): self.add_configure_env_arg(k, v) # self.make_args.env_vars[k] = str(v) if k.endswith("_FOR_BUILD"): - k2 = k[0:-len("_FOR_BUILD")] + k2 = k[0 : -len("_FOR_BUILD")] self.add_configure_env_arg(k2, v) # self.make_args.env_vars[k2] = str(v) @@ -101,10 +101,13 @@ def setup(self): AS_FOR_TARGET=str(self.CC), # + target_cflags, CC_FOR_TARGET=str(self.CC), # + target_cflags, CXX_FOR_TARGET=str(self.CXX), # + target_cflags, - AR_FOR_TARGET=self.target_info.ar, STRIP_FOR_TARGET=self.target_info.strip_tool, - OBJCOPY_FOR_TARGET=bindir / "objcopy", RANLIB_FOR_TARGET=self.target_info.ranlib, + AR_FOR_TARGET=self.target_info.ar, + STRIP_FOR_TARGET=self.target_info.strip_tool, + OBJCOPY_FOR_TARGET=bindir / "objcopy", + RANLIB_FOR_TARGET=self.target_info.ranlib, OBJDUMP_FOR_TARGET=bindir / "llvm-objdump", - READELF_FOR_TARGET=bindir / "readelf", NM_FOR_TARGET=self.target_info.nm, + READELF_FOR_TARGET=bindir / "readelf", + NM_FOR_TARGET=self.target_info.nm, # Set all the flags: CFLAGS_FOR_TARGET=target_cflags, CCASFLAGS_FOR_TARGET=target_cflags, @@ -115,7 +118,7 @@ def setup(self): CPP_FOR_BUILD=self.host_CPP, LD_FOR_TARGET=str(self.target_info.linker), LDFLAGS_FOR_TARGET=commandline_to_str(self.default_ldflags), - ) + ) if self.compiling_for_mips(include_purecap=True): # long double is the same as double for MIPS @@ -127,36 +130,36 @@ def setup(self): self.configure_args.append("--disable-libgloss") if self.target_info.is_baremetal(): - self.configure_args.extend([ - "--enable-malloc-debugging", - "--enable-newlib-long-time_t", # we want time_t to be long and not int! - "--enable-newlib-io-c99-formats", - "--enable-newlib-io-long-long", - # --enable-newlib-io-pos-args (probably not needed) - "--disable-newlib-io-long-double", # we don't need this, MIPS long double == double - "--enable-newlib-io-float", - # "--disable-newlib-supplied-syscalls" - "--disable-libstdcxx", # not sure if this is needed - - # we don't have any multithreading support on baremetal - "--disable-newlib-multithread", - - "--enable-newlib-global-atexit", # TODO: is this needed? - # --enable-newlib-nano-malloc (should we do this?) - "--disable-multilib", - - # TODO: smaller lib? "--enable-target-optspace" - - # FIXME: these don't seem to work - "--enable-serial-build-configure", - "--enable-serial-target-configure", - "--enable-serial-host-configure", - ]) + self.configure_args.extend( + [ + "--enable-malloc-debugging", + "--enable-newlib-long-time_t", # we want time_t to be long and not int! + "--enable-newlib-io-c99-formats", + "--enable-newlib-io-long-long", + # --enable-newlib-io-pos-args (probably not needed) + "--disable-newlib-io-long-double", # we don't need this, MIPS long double == double + "--enable-newlib-io-float", + # "--disable-newlib-supplied-syscalls" + "--disable-libstdcxx", # not sure if this is needed + # we don't have any multithreading support on baremetal + "--disable-newlib-multithread", + "--enable-newlib-global-atexit", # TODO: is this needed? + # --enable-newlib-nano-malloc (should we do this?) + "--disable-multilib", + # TODO: smaller lib? "--enable-target-optspace" + # FIXME: these don't seem to work + "--enable-serial-build-configure", + "--enable-serial-target-configure", + "--enable-serial-host-configure", + ] + ) elif self.target_info.is_rtems(): - self.configure_args.extend([ - "--enable-newlib-io-c99-formats", - "--disable-libstdcxx", # not sure if this is needed - ]) + self.configure_args.extend( + [ + "--enable-newlib-io-c99-formats", + "--disable-libstdcxx", # not sure if this is needed + ] + ) if self.locale_support: # needed for locale support @@ -182,21 +185,37 @@ def install(self, **kwargs): def run_tests(self): with tempfile.TemporaryDirectory(prefix="cheribuild-" + self.target + "-") as td: - self.write_file(Path(td, "main.c"), contents=""" + self.write_file( + Path(td, "main.c"), + contents=""" #include int main(int argc, char** argv) { for (int i = 0; i < argc; i++) { printf("argv[%d] = '%s'\\n", i, argv[i]); } } -""", overwrite=True) +""", + overwrite=True, + ) test_exe = Path(td, "test.exe") # FIXME: CHERI helloworld - compiler_flags = self.essential_compiler_and_linker_flags + self.COMMON_FLAGS + [ - "-Wl,-T,qemu-malta.ld", "-Wl,-verbose", "--sysroot=" + str(self.sdk_sysroot)] + compiler_flags = ( + self.essential_compiler_and_linker_flags + + self.COMMON_FLAGS + + ["-Wl,-T,qemu-malta.ld", "-Wl,-verbose", "--sysroot=" + str(self.sdk_sysroot)] + ) self.run_cmd([self.sdk_bindir / "clang", "main.c", "-o", test_exe, *compiler_flags, "-###"], cwd=td) self.run_cmd([self.sdk_bindir / "clang", "main.c", "-o", test_exe, *compiler_flags], cwd=td) self.run_cmd(self.sdk_bindir / "llvm-readobj", "-h", test_exe) from ..build_qemu import BuildQEMU - self.run_cmd(self.sdk_sysroot / "bin/run_with_qemu.py", "--qemu", BuildQEMU.qemu_binary(self), - "--timeout", "20", test_exe, "HELLO", "WORLD") + + self.run_cmd( + self.sdk_sysroot / "bin/run_with_qemu.py", + "--qemu", + BuildQEMU.qemu_binary(self), + "--timeout", + "20", + test_exe, + "HELLO", + "WORLD", + ) diff --git a/pycheribuild/projects/cross/nginx.py b/pycheribuild/projects/cross/nginx.py index 8baed06ea..cf02f8a22 100644 --- a/pycheribuild/projects/cross/nginx.py +++ b/pycheribuild/projects/cross/nginx.py @@ -48,12 +48,15 @@ def setup(self): super().setup() self.configure_command = self.source_dir / "auto/configure" if not self.compiling_for_host(): - self.COMMON_FLAGS.extend(["-pedantic", - "-Wno-gnu-statement-expression", - "-Wno-flexible-array-extensions", # TODO: could this cause errors? - # "-Wno-extended-offsetof", - "-Wno-format-pedantic", - ]) + self.COMMON_FLAGS.extend( + [ + "-pedantic", + "-Wno-gnu-statement-expression", + "-Wno-flexible-array-extensions", # TODO: could this cause errors? + # "-Wno-extended-offsetof", + "-Wno-format-pedantic", + ] + ) self.configure_environment["AR"] = str(self.sdk_bindir / "cheri-unknown-freebsd-ar") # The makefile expects the current working directory to be the source dir. Therefore we add -f $build/Makefile # This is also in the makefile generated in the source dir but it doesn't work with multiple build dirs @@ -67,9 +70,10 @@ def install(self, **kwargs): # install the benchmark script benchmark = self.read_file(self.source_dir / "nginx-benchmark.sh") if not self.compiling_for_host(): - benchmark = re.sub(r'NGINX=.*', "NGINX=\"" + str(self.install_prefix / "sbin/nginx") + "\"", benchmark) - benchmark = re.sub(r'FETCHBENCH=.*', "FETCHBENCH=\"" + str(self.install_prefix / "sbin/fetchbench") + "\"", - benchmark) + benchmark = re.sub(r"NGINX=.*", 'NGINX="' + str(self.install_prefix / "sbin/nginx") + '"', benchmark) + benchmark = re.sub( + r"FETCHBENCH=.*", 'FETCHBENCH="' + str(self.install_prefix / "sbin/fetchbench") + '"', benchmark + ) self.write_file(self.real_install_root_dir / "nginx-benchmark.sh", benchmark, overwrite=True, mode=0o755) def needs_configure(self): @@ -78,21 +82,28 @@ def needs_configure(self): def configure(self): if self.should_include_debug_info: self.configure_args.append("--with-debug") - self.configure_args.extend(["--without-pcre", - "--without-http_rewrite_module", - "--with-http_v2_module", - "--with-http_ssl_module", - "--without-http_gzip_module", - "--without-http_rewrite_module", - "--without-pcre", - "--builddir=" + str(self.build_dir)]) + self.configure_args.extend( + [ + "--without-pcre", + "--without-http_rewrite_module", + "--with-http_v2_module", + "--with-http_ssl_module", + "--without-http_gzip_module", + "--without-http_rewrite_module", + "--without-pcre", + "--builddir=" + str(self.build_dir), + ] + ) if not self.compiling_for_host(): self.LDFLAGS.append("-v") - self.configure_args.extend(["--crossbuild=FreeBSD:12.0-CURRENT:mips", - "--with-cc-opt=" + self.commandline_to_str(self.default_compiler_flags), - "--with-ld-opt=" + self.commandline_to_str(self.default_ldflags), - "--sysroot=" + str(self.sdk_sysroot), - ]) + self.configure_args.extend( + [ + "--crossbuild=FreeBSD:12.0-CURRENT:mips", + "--with-cc-opt=" + self.commandline_to_str(self.default_compiler_flags), + "--with-ld-opt=" + self.commandline_to_str(self.default_ldflags), + "--sysroot=" + str(self.sdk_sysroot), + ] + ) self.configure_environment["CC_TEST_FLAGS"] = self.commandline_to_str(self.default_compiler_flags) self.configure_environment["NGX_TEST_LD_OPT"] = self.commandline_to_str(self.default_ldflags) self.configure_environment["NGX_SIZEOF_int"] = "4" diff --git a/pycheribuild/projects/cross/opensbi.py b/pycheribuild/projects/cross/opensbi.py index d3998add9..02b05739f 100644 --- a/pycheribuild/projects/cross/opensbi.py +++ b/pycheribuild/projects/cross/opensbi.py @@ -61,11 +61,12 @@ class BuildOpenSBI(Project): CompilationTargets.FREESTANDING_RISCV64_HYBRID, CompilationTargets.FREESTANDING_RISCV64, # Won't compile yet: CompilationTargets.FREESTANDING_RISCV64_PURECAP - ) + ) make_kind = MakeCommandKind.GnuMake _always_add_suffixed_targets = True - _default_install_dir_fn = ComputedDefaultValue(function=opensbi_install_dir, - as_string="$SDK_ROOT/opensbi/riscv{32,64}{-hybrid,-purecap,}") + _default_install_dir_fn = ComputedDefaultValue( + function=opensbi_install_dir, as_string="$SDK_ROOT/opensbi/riscv{32,64}{-hybrid,-purecap,}" + ) @classproperty def needs_sysroot(self): @@ -99,7 +100,7 @@ def setup(self): PLATFORM_RISCV_ABI=self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True), PLATFORM_RISCV_ISA=self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True), PLATFORM_RISCV_XLEN=64, - ) + ) if self.config.verbose: self.make_args.set(V=True) @@ -135,14 +136,21 @@ def install(self, **kwargs): # Install into the QEMU firware directory so that `-bios default` works qemu_fw_dir = BuildQEMU.get_install_dir(self, cross_target=CompilationTargets.NATIVE) / "share/qemu/" self.makedirs(qemu_fw_dir) - self.run_cmd(self.sdk_bindir / "llvm-objcopy", "-S", "-O", "binary", - self._fw_jump_path(), qemu_fw_dir / "opensbi-riscv64cheri-virt-fw_jump.bin", - print_verbose_only=False) + self.run_cmd( + self.sdk_bindir / "llvm-objcopy", + "-S", + "-O", + "binary", + self._fw_jump_path(), + qemu_fw_dir / "opensbi-riscv64cheri-virt-fw_jump.bin", + print_verbose_only=False, + ) def _fw_jump_path(self) -> Path: # share/opensbi/lp64/generic/firmware//fw_payload.bin return self.install_dir / "share/opensbi/{abi}/generic/firmware/fw_jump.elf".format( - abi=self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True)) + abi=self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + ) @classmethod def get_nocap_instance(cls, caller, cpu_arch=CPUArchitecture.RISCV64) -> "BuildOpenSBI": @@ -170,20 +178,27 @@ class BuildOpenSBIGFE(BuildOpenSBI): def setup(self): super().setup() - self.make_args.set(FW_TEXT_START=0xc0000000) + self.make_args.set(FW_TEXT_START=0xC0000000) class BuildUpstreamOpenSBI(BuildOpenSBI): target = "upstream-opensbi" _default_install_dir_fn = ComputedDefaultValue( function=lambda config, p: config.cheri_sdk_dir / "upstream-opensbi/riscv64", - as_string="$SDK_ROOT/upstream-opensbi/riscv64") + as_string="$SDK_ROOT/upstream-opensbi/riscv64", + ) repository = GitRepository("https://github.com/riscv-software-src/opensbi.git") supported_architectures = (CompilationTargets.FREESTANDING_RISCV64,) def run_tests(self): options = QemuOptions(self.crosscompile_target) - self.run_cmd(options.get_commandline( - qemu_command=BuildQEMU.qemu_binary(self), add_network_device=False, bios_args=["-bios", "none"], - kernel_file=self.install_dir / "share/opensbi/lp64/generic/firmware//fw_payload.elf"), - give_tty_control=True, cwd="/") + self.run_cmd( + options.get_commandline( + qemu_command=BuildQEMU.qemu_binary(self), + add_network_device=False, + bios_args=["-bios", "none"], + kernel_file=self.install_dir / "share/opensbi/lp64/generic/firmware//fw_payload.elf", + ), + give_tty_control=True, + cwd="/", + ) diff --git a/pycheribuild/projects/cross/openssl.py b/pycheribuild/projects/cross/openssl.py index b7df43bfa..b452fb112 100644 --- a/pycheribuild/projects/cross/openssl.py +++ b/pycheribuild/projects/cross/openssl.py @@ -38,8 +38,9 @@ class BuildOpenSSL(CrossCompileProject): # nginx looks in /usr/local path_in_rootfs = "/usr/local" - repository = GitRepository("https://github.com/CTSRD-CHERI/openssl.git", - default_branch="OpenSSL_1_1_1-stable-cheri") + repository = GitRepository( + "https://github.com/CTSRD-CHERI/openssl.git", default_branch="OpenSSL_1_1_1-stable-cheri" + ) native_install_dir = DefaultInstallDir.DO_NOT_INSTALL cross_install_dir = DefaultInstallDir.ROOTFS_OPTBASE diff --git a/pycheribuild/projects/cross/picolibc.py b/pycheribuild/projects/cross/picolibc.py index 635087300..93f2aa8dd 100644 --- a/pycheribuild/projects/cross/picolibc.py +++ b/pycheribuild/projects/cross/picolibc.py @@ -120,7 +120,9 @@ def compile(self, **kwargs): # Symlink libgcc.a to the build dir to allow linking against it without adding all of /lib. self.makedirs(self.build_dir / "local-libgcc") self.create_symlink( - self.sdk_sysroot / "lib/libgcc.a", self.build_dir / "local-libgcc/libgcc.a", print_verbose_only=False, + self.sdk_sysroot / "lib/libgcc.a", + self.build_dir / "local-libgcc/libgcc.a", + print_verbose_only=False, ) super().compile(**kwargs) diff --git a/pycheribuild/projects/cross/pkg.py b/pycheribuild/projects/cross/pkg.py index 9c06b49d0..0d21b11ba 100644 --- a/pycheribuild/projects/cross/pkg.py +++ b/pycheribuild/projects/cross/pkg.py @@ -57,8 +57,11 @@ def setup(self): if self.build_type.should_include_debug_info: self.COMMON_FLAGS.append("-g") - self.make_args.set_env(CPPFLAGS=self.commandline_to_str( - self.COMMON_FLAGS + self.compiler_warning_flags + self.optimization_flags + self.COMMON_FLAGS)) + self.make_args.set_env( + CPPFLAGS=self.commandline_to_str( + self.COMMON_FLAGS + self.compiler_warning_flags + self.optimization_flags + self.COMMON_FLAGS + ) + ) self.make_args.set_env(LDFLAGS=self.commandline_to_str(self.default_ldflags)) def compile(self, **kwargs): diff --git a/pycheribuild/projects/cross/postgres.py b/pycheribuild/projects/cross/postgres.py index 854868f46..42115cc15 100644 --- a/pycheribuild/projects/cross/postgres.py +++ b/pycheribuild/projects/cross/postgres.py @@ -35,8 +35,9 @@ class BuildPostgres(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/postgres.git", - default_branch="96-cheri", force_branch=True) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/postgres.git", default_branch="96-cheri", force_branch=True + ) # we have to build in the source directory, out-of-source is broken # build_in_source_dir = True make_kind = MakeCommandKind.GnuMake @@ -50,9 +51,14 @@ def setup(self): # self.COMMON_FLAGS.append("-DLOCK_DEBUG=1") self.configure_args.append("--enable-cassert") - self.common_warning_flags.extend(["-pedantic", "-Wno-gnu-statement-expression", - "-Wno-flexible-array-extensions", # TODO: could this cause errors? - "-Wno-format-pedantic"]) + self.common_warning_flags.extend( + [ + "-pedantic", + "-Wno-gnu-statement-expression", + "-Wno-flexible-array-extensions", # TODO: could this cause errors? + "-Wno-format-pedantic", + ] + ) self.LDFLAGS.append("-pthread") if OSInfo.IS_FREEBSD and self.compiling_for_host(): # Something werid is happending with the locale code (somehow not being built -FPIC?): @@ -99,10 +105,11 @@ def install(self, **kwargs): pg_root = str(self.install_prefix) else: pg_root = str(self.install_dir) - benchmark = re.sub(r'POSTGRES_ROOT=".*"', "POSTGRES_ROOT=\"" + pg_root + "\"", benchmark) + benchmark = re.sub(r'POSTGRES_ROOT=".*"', 'POSTGRES_ROOT="' + pg_root + '"', benchmark) self.write_file(self.real_install_root_dir / benchname, benchmark, overwrite=True, mode=0o755) - self.install_file(self.source_dir / "run-postgres-tests.sh", - self.real_install_root_dir / "run-postgres-tests.sh") + self.install_file( + self.source_dir / "run-postgres-tests.sh", self.real_install_root_dir / "run-postgres-tests.sh" + ) @property def default_ldflags(self): @@ -124,9 +131,14 @@ def run_tests(self): # self.run_make("check", cwd=self.build_dir / "src/interfaces/ecpg/test", stdout_filter=None) else: locale_dir = self.rootfs_dir / "usr/share/locale" - self.target_info.run_cheribsd_test_script("run_postgres_tests.py", "--smb-mount-directory", - str(self.install_dir) + ":" + str(self.install_prefix), - "--locale-files-dir", locale_dir, mount_builddir=False, - # long running test -> speed up by using a kernel without - # invariants - use_benchmark_kernel_by_default=True) + self.target_info.run_cheribsd_test_script( + "run_postgres_tests.py", + "--smb-mount-directory", + str(self.install_dir) + ":" + str(self.install_prefix), + "--locale-files-dir", + locale_dir, + mount_builddir=False, + # long running test -> speed up by using a kernel without + # invariants + use_benchmark_kernel_by_default=True, + ) diff --git a/pycheribuild/projects/cross/python.py b/pycheribuild/projects/cross/python.py index 3848bb74e..ee644b6d3 100644 --- a/pycheribuild/projects/cross/python.py +++ b/pycheribuild/projects/cross/python.py @@ -72,12 +72,16 @@ def configure(self, **kwargs): if not self.compiling_for_host(): self.configure_args.append("--without-doc-strings") # should reduce size - native_python = self.get_instance_for_cross_target(CompilationTargets.NATIVE_NON_PURECAP, - self.config).install_dir / "bin/python3" + native_python = ( + self.get_instance_for_cross_target(CompilationTargets.NATIVE_NON_PURECAP, self.config).install_dir + / "bin/python3" + ) if not native_python.exists(): - self.dependency_error("Native python3 doesn't exist, you must build the `python-native` target first.", - cheribuild_target="python", - cheribuild_xtarget=CompilationTargets.NATIVE_NON_PURECAP) + self.dependency_error( + "Native python3 doesn't exist, you must build the `python-native` target first.", + cheribuild_target="python", + cheribuild_xtarget=CompilationTargets.NATIVE_NON_PURECAP, + ) self.add_configure_vars( ac_cv_buggy_getaddrinfo="no", # Doesn't work since that remove all flags, need to set PATH instead @@ -98,10 +102,21 @@ def run_tests(self): # python build system adds .exe for case-insensitive dirs suffix = "" if is_case_sensitive_dir(self.build_dir) else ".exe" if self.compiling_for_host(): - self.run_cmd(self.build_dir / ("python" + suffix), "-m", "test", "-w", "--junit-xml=python-tests.xml", - self.config.make_j_flag, cwd=self.build_dir) + self.run_cmd( + self.build_dir / ("python" + suffix), + "-m", + "test", + "-w", + "--junit-xml=python-tests.xml", + self.config.make_j_flag, + cwd=self.build_dir, + ) else: # Python executes tons of system calls, hopefully using the benchmark kernel helps - self.target_info.run_cheribsd_test_script("run_python_tests.py", "--buildexe-suffix=" + suffix, - mount_installdir=True, - mount_sourcedir=True, use_benchmark_kernel_by_default=True) + self.target_info.run_cheribsd_test_script( + "run_python_tests.py", + "--buildexe-suffix=" + suffix, + mount_installdir=True, + mount_sourcedir=True, + use_benchmark_kernel_by_default=True, + ) diff --git a/pycheribuild/projects/cross/qt5.py b/pycheribuild/projects/cross/qt5.py index b3c4f114b..793725fcf 100644 --- a/pycheribuild/projects/cross/qt5.py +++ b/pycheribuild/projects/cross/qt5.py @@ -129,16 +129,21 @@ def check_system_dependencies(self) -> None: super().check_system_dependencies() self.check_required_system_tool("xmlto", homebrew="xmlto", apt="xmlto") self.check_required_system_tool( - "xmllint", homebrew="libxml2", apt="libxml2-utils", cheribuild_target="libxml2-native", + "xmllint", + homebrew="libxml2", + apt="libxml2-utils", + cheribuild_target="libxml2-native", ) self.check_required_system_tool("msgfmt", freebsd="gettext-tools") # no way to disable translations def setup(self): super().setup() - self.add_meson_options(**{ - "update-mimedb": True, - "build-tools": self._can_build_tools, - }) + self.add_meson_options( + **{ + "update-mimedb": True, + "build-tools": self._can_build_tools, + } + ) if not self._can_build_tools: # Ensure that we have update-mime-database available as it will be used in a post-install action. self.get_update_mime_database_path(self) @@ -184,21 +189,23 @@ def dependencies(cls, config: CheriConfig) -> "tuple[str, ...]": return tuple(deps) if cls.use_x11: # The system X11 libraries might be too old, so add the cheribuild-provided ones as a dependency - deps.extend([ - "libx11", - "libxcb", - "libxkbcommon", - "libxcb-cursor", - "libxcb-util", - "libxcb-image", - "libice", - "libsm", - "libxext", - "libxtst", - "libxcb-render-util", - "libxcb-wm", - "libxcb-keysyms", - ]) + deps.extend( + [ + "libx11", + "libxcb", + "libxkbcommon", + "libxcb-cursor", + "libxcb-util", + "libxcb-image", + "libice", + "libsm", + "libxext", + "libxtst", + "libxcb-render-util", + "libxcb-wm", + "libxcb-keysyms", + ] + ) # Always use our patched image/sql libraries instead of the host ones: deps.extend(["libpng", "libjpeg-turbo"]) if not cls.get_crosscompile_target().is_native(): @@ -243,10 +250,16 @@ def setup_config_options(cls, **kwargs): # Link against X11 libs by default if we aren't compiling for macOS native_is_macos = cls._xtarget is not None and cls._xtarget.target_info_cls.is_macos() cls.use_x11 = cls.add_bool_option( - "use-x11", default=not native_is_macos, show_help=False, help="Build Qt with the XCB backend.", + "use-x11", + default=not native_is_macos, + show_help=False, + help="Build Qt with the XCB backend.", ) cls.use_opengl = cls.add_bool_option( - "use-opengl", default=True, show_help=False, help="Build Qt with OpenGL support", + "use-opengl", + default=True, + show_help=False, + help="Build Qt with OpenGL support", ) def configure(self, **kwargs): @@ -285,23 +298,25 @@ def configure(self, **kwargs): linker_flags = filter(lambda s: not s.startswith("--sysroot"), linker_flags) compiler_flags = filter(lambda s: not s.startswith("--sysroot"), compiler_flags) cross_tools_prefix = self.target_info.get_target_triple(include_version=False) - self.configure_args.extend([ - "-device", - "freebsd-generic-clang", - "-device-option", - f"CROSS_COMPILE={self.sdk_bindir}/{cross_tools_prefix}-", - "-device-option", - "COMPILER_FLAGS=" + self.commandline_to_str(compiler_flags), - "-device-option", - "LINKER_FLAGS=" + self.commandline_to_str(linker_flags), - "-sysroot", - self.cross_sysroot_path, - "-prefix", - self.install_prefix, - # The prefix for host tools such as qmake - "-hostprefix", - str(self.qt_host_tools_path), - ]) + self.configure_args.extend( + [ + "-device", + "freebsd-generic-clang", + "-device-option", + f"CROSS_COMPILE={self.sdk_bindir}/{cross_tools_prefix}-", + "-device-option", + "COMPILER_FLAGS=" + self.commandline_to_str(compiler_flags), + "-device-option", + "LINKER_FLAGS=" + self.commandline_to_str(linker_flags), + "-sysroot", + self.cross_sysroot_path, + "-prefix", + self.install_prefix, + # The prefix for host tools such as qmake + "-hostprefix", + str(self.qt_host_tools_path), + ] + ) xcb = BuildLibXCB.get_instance(self) if xcb.install_prefix != self.install_prefix: self.configure_args.append( @@ -316,25 +331,25 @@ def configure(self, **kwargs): if self.use_asan: self.configure_args.extend(["-sanitize", "address", "-sanitize", "undefined"]) - self.configure_args.extend([ - ( - # To ensure the host and cross-compiled version is the same also disable opengl - "-opengl" - if self.use_opengl - else "-no-opengl" - ), - # Since the cross-compiled version doesn't have glib, also disable it for the native on - "-no-glib", - # Needed for webkit: - # "-icu", - # "-no-Werror", - "-no-use-gold-linker", - "-no-iconv", - "-no-headersclean", - # Don't embed the mimetype DB in libQt5Core.so. It's huge and results in lots XML parsing. Instead, we just - # ensure that the binary cache exists in the disk image. - "-no-mimetype-database", - ]) + self.configure_args.extend( + [ + ( + # To ensure the host and cross-compiled version is the same also disable opengl + "-opengl" if self.use_opengl else "-no-opengl" + ), + # Since the cross-compiled version doesn't have glib, also disable it for the native on + "-no-glib", + # Needed for webkit: + # "-icu", + # "-no-Werror", + "-no-use-gold-linker", + "-no-iconv", + "-no-headersclean", + # Don't embed the mimetype DB in libQt5Core.so. It's huge and results in lots XML parsing. + # Instead, we just ensure that the binary cache exists in the disk image. + "-no-mimetype-database", + ] + ) if self.build_tests: self.configure_args.append("-developer-build") if self.target_info.is_macos(): @@ -387,16 +402,18 @@ def configure(self, **kwargs): # self.configure_args.append("-reduce-relocations") if self.minimal: - self.configure_args.extend([ - "-no-widgets", - "-no-glib", - "-no-gtk", - "-no-opengl", - "-no-cups", - "-no-syslog", - "-no-gui", - "-no-iconv", - ]) + self.configure_args.extend( + [ + "-no-widgets", + "-no-glib", + "-no-gtk", + "-no-opengl", + "-no-cups", + "-no-syslog", + "-no-gui", + "-no-iconv", + ] + ) else: self.configure_args.append("-dbus") # we want to build QtDBus if not self.target_info.is_macos(): @@ -436,7 +453,10 @@ class BuildQtBaseDev(CrossCompileCMakeProject): gui: bool = BoolConfigOption("gui", show_help=True, default=True, help="Include QtGui") use_opengl: bool = BoolConfigOption("opengl", show_help=True, default=False, help="Include QtOpenGl") minimal: bool = BoolConfigOption( - "minimal", show_help=True, default=True, help="Don't build QtWidgets or QtSql, etc", + "minimal", + show_help=True, + default=True, + help="Don't build QtWidgets or QtSql, etc", ) @classmethod @@ -446,22 +466,24 @@ def dependencies(cls, config: CheriConfig) -> "tuple[str, ...]": deps.append(BuildSharedMimeInfo.get_class_for_target(rootfs_target).target) if cls.gui: # The system X11 libraries might be too old, so add the cheribuild-provided ones as a dependency - deps.extend([ - "libx11", - "libxkbcommon", - "libinput", - "libxcb", - "libxcb-cursor", - "libxcb-util", - "libxcb-image", - "libice", - "libsm", - "libxext", - "libxtst", - "libxcb-render-util", - "libxcb-wm", - "libxcb-keysyms", - ]) + deps.extend( + [ + "libx11", + "libxkbcommon", + "libinput", + "libxcb", + "libxcb-cursor", + "libxcb-util", + "libxcb-image", + "libice", + "libsm", + "libxext", + "libxtst", + "libxcb-render-util", + "libxcb-wm", + "libxcb-keysyms", + ] + ) # Always use our patched image/sql libraries instead of the host ones: deps.extend(["libpng", "libjpeg-turbo"]) if not cls.get_crosscompile_target().is_native(): @@ -573,7 +595,10 @@ def run_tests(self): else: # TODO: run `ctest --show-only=json-v1` to get list of tests self.target_info.run_cheribsd_test_script( - "run_qtbase_tests.py", use_benchmark_kernel_by_default=False, mount_sysroot=True, mount_sourcedir=True, + "run_qtbase_tests.py", + use_benchmark_kernel_by_default=False, + mount_sysroot=True, + mount_sourcedir=True, ) @@ -585,7 +610,9 @@ class BuildQt5(BuildQtWithConfigureScript): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.all_modules = cls.add_bool_option( - "all-modules", show_help=True, help="Build all modules (even those that don't make sense for CHERI)", + "all-modules", + show_help=True, + help="Build all modules (even those that don't make sense for CHERI)", ) def configure(self, **kwargs): @@ -670,7 +697,8 @@ def install(self, **kwargs): self.clean_directory(qt_fonts_dir, ensure_dir_exists=False) self.makedirs(qt_fonts_dir.parent) dejavu_fonts = InstallDejaVuFonts.get_instance( - self, cross_target=self.crosscompile_target.get_rootfs_target(), + self, + cross_target=self.crosscompile_target.get_rootfs_target(), ) self.create_symlink(dejavu_fonts.fonts_dir, qt_fonts_dir, print_verbose_only=False) @@ -700,7 +728,9 @@ def run_tests(self): print_verbose_only=False, ) self.create_symlink( - mimedb_tests_dir / "shared-mime-info-2.1", mimedb_tests_dir / "s-m-i", print_verbose_only=False, + mimedb_tests_dir / "shared-mime-info-2.1", + mimedb_tests_dir / "s-m-i", + print_verbose_only=False, ) self._compile_relevant_tests() if self.compiling_for_host(): @@ -800,15 +830,19 @@ class BuildQtMacExtras(BuildQtModuleWithQMake): class BuildQtDeclarative(BuildQtModuleWithQMake): target = "qtdeclarative" repository = GitRepository( - "https://github.com/CTSRD-CHERI/qtdeclarative.git", default_branch="5.15", force_branch=True, + "https://github.com/CTSRD-CHERI/qtdeclarative.git", + default_branch="5.15", + force_branch=True, ) def setup(self): super().setup() - self.configure_args.extend([ - "-no-qml-debug", # debugger not compatibale with CHERI purecap - "-quick-designer", # needed for quickcontrols2 - ]) + self.configure_args.extend( + [ + "-no-qml-debug", # debugger not compatibale with CHERI purecap + "-quick-designer", # needed for quickcontrols2 + ] + ) class BuildQtTools(BuildQtModuleWithQMake): @@ -825,14 +859,16 @@ def setup(self): super().setup() # No need to build all the developer GUI tools, we only want programs that are # useful inside the disk image. - self.configure_args.extend([ - "-no-feature-assistant", - # Some KDE programs/libraries install designer plugins, so for now we install it by default. - # This avoids having to patch those projects to make the feature optional. - # "-no-feature-designer", - # kColorPicker (which is used by Gwenview needs linguist, so we build this too). - # "-no-feature-linguist", - ]) + self.configure_args.extend( + [ + "-no-feature-assistant", + # Some KDE programs/libraries install designer plugins, so for now we install it by default. + # This avoids having to patch those projects to make the feature optional. + # "-no-feature-designer", + # kColorPicker (which is used by Gwenview needs linguist, so we build this too). + # "-no-feature-linguist", + ] + ) class BuildQtWayland(BuildQtModuleWithQMake): @@ -848,10 +884,12 @@ class BuildQtWayland(BuildQtModuleWithQMake): def setup(self): super().setup() - self.configure_args.extend([ - "-feature-wayland-client", - "-feature-wayland-server", - ]) + self.configure_args.extend( + [ + "-feature-wayland-client", + "-feature-wayland-server", + ] + ) # def compile(self, **kwargs): # self.run_make() @@ -1002,7 +1040,9 @@ def setup(self): class BuildQtWebkit(CrossCompileCMakeProject): repository = GitRepository( - "https://github.com/CTSRD-CHERI/qtwebkit", default_branch="qtwebkit-5.212-cheri", force_branch=True, + "https://github.com/CTSRD-CHERI/qtwebkit", + default_branch="qtwebkit-5.212-cheri", + force_branch=True, ) is_large_source_repository = True dependencies = ("qtbase", "icu4c", "libxml2", "sqlite") @@ -1087,7 +1127,9 @@ def setup(self): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.build_jsc_only = cls.add_bool_option( - "build-jsc-only", show_help=True, help="only build the JavaScript interpreter executable", + "build-jsc-only", + show_help=True, + help="only build the JavaScript interpreter executable", ) def compile(self, **kwargs): @@ -1096,7 +1138,10 @@ def compile(self, **kwargs): update_mime = BuildSharedMimeInfo.get_update_mime_database_path(self) mime_info_src = BuildQtBase.get_source_dir(self) / "src/corelib/mimetypes/mime/packages/freedesktop.org.xml" self.install_file( - mime_info_src, Path(td, "mime/packages/freedesktop.org.xml"), force=True, print_verbose_only=False, + mime_info_src, + Path(td, "mime/packages/freedesktop.org.xml"), + force=True, + print_verbose_only=False, ) self.run_cmd(update_mime, "-V", Path(td, "mime"), cwd="/") @@ -1104,10 +1149,16 @@ def compile(self, **kwargs): self.fatal("Could not generated shared-mime-info cache!") # install mime.cache and freedesktop.org.xml into the build dir for tests self.install_file( - mime_info_src, self.build_dir / "freedesktop.org.xml", force=True, print_verbose_only=False, + mime_info_src, + self.build_dir / "freedesktop.org.xml", + force=True, + print_verbose_only=False, ) self.install_file( - Path(td, "mime/mime.cache"), self.build_dir / "mime.cache", force=True, print_verbose_only=False, + Path(td, "mime/mime.cache"), + self.build_dir / "mime.cache", + force=True, + print_verbose_only=False, ) # TODO: get https://github.com/annulen/webkit-test-fonts to run the full testsuite if self.build_jsc_only: @@ -1120,7 +1171,9 @@ def install(self, **kwargs): if not self.build_jsc_only: dump_render_tree = self.build_dir / "bin/DumpRenderTree" self.maybe_strip_elf_file( - dump_render_tree, output_path=dump_render_tree.with_suffix(".stripped"), print_verbose_only=False, + dump_render_tree, + output_path=dump_render_tree.with_suffix(".stripped"), + print_verbose_only=False, ) jsc = self.build_dir / "bin/jsc" self.maybe_strip_elf_file(jsc, output_path=jsc.with_suffix(".stripped"), print_verbose_only=False) diff --git a/pycheribuild/projects/cross/rlbox.py b/pycheribuild/projects/cross/rlbox.py index 3442dd349..ad787f545 100644 --- a/pycheribuild/projects/cross/rlbox.py +++ b/pycheribuild/projects/cross/rlbox.py @@ -46,8 +46,9 @@ def run_tests(self): self.run_make("test") else: args = ["--verbose"] if self.config.verbose else [] - self.target_info.run_cheribsd_test_script("run_rlbox_tests.py", *args, mount_builddir=True, - mount_sourcedir=True, mount_sysroot=True) + self.target_info.run_cheribsd_test_script( + "run_rlbox_tests.py", *args, mount_builddir=True, mount_sourcedir=True, mount_sysroot=True + ) class BuildCatch2(CrossCompileCMakeProject): diff --git a/pycheribuild/projects/cross/ros2.py b/pycheribuild/projects/cross/ros2.py index 816aec3d6..02b6d7496 100644 --- a/pycheribuild/projects/cross/ros2.py +++ b/pycheribuild/projects/cross/ros2.py @@ -32,8 +32,9 @@ class BuildRos2(CrossCompileCMakeProject): target = "ros2" - repository = GitRepository("https://github.com/dodsonmg/ros2_dashing_minimal.git", default_branch="master", - force_branch=True) + repository = GitRepository( + "https://github.com/dodsonmg/ros2_dashing_minimal.git", default_branch="master", force_branch=True + ) # it may eventually be useful to install to rootfs or sysroot depending on whether we want to use ROS2 # as a library for building other applications using cheribuild # therefore, the _install_dir doesn't do anything, but cheribuild requires them @@ -49,21 +50,28 @@ def _ignore_packages(self): def _run_vcs(self): # this is the meta version control system used by ros for downloading and unpacking repos if not shutil.which("vcs"): - self.dependency_error("Missing vcs command", - install_instructions=InstallInstructions("pip3 install --user vcstool")) + self.dependency_error( + "Missing vcs command", install_instructions=InstallInstructions("pip3 install --user vcstool") + ) cmdline = ["vcs", "import", "--input", "ros2_minimal.repos", "src"] self.run_cmd(cmdline, cwd=self.source_dir) def _run_colcon(self, **kwargs): # colcon is the meta build system (on top of cmake) used by ros if not shutil.which("colcon"): - self.dependency_error("Missing colcon command", - install_instructions=InstallInstructions( - "pip3 install --user colcon-common-extensions")) - colcon_cmd = ["colcon", "build", - "--install-base", self.install_dir, - "--build-base", self.build_dir, - "--merge-install"] + self.dependency_error( + "Missing colcon command", + install_instructions=InstallInstructions("pip3 install --user colcon-common-extensions"), + ) + colcon_cmd = [ + "colcon", + "build", + "--install-base", + self.install_dir, + "--build-base", + self.build_dir, + "--merge-install", + ] colcon_args = ["--no-warn-unused-cli", "--packages-skip-build-finished"] cmake_args = ["--cmake-args", "-DBUILD_TESTING=NO"] if not self.compiling_for_host(): @@ -134,7 +142,7 @@ def _set_env(self): endif setenv LD_LIBRARY_PATH {ld_library_path}:${{LD_LIBRARY_PATH}} """ - self.write_file(self.install_dir / 'cheri_setup.csh', csh_script, overwrite=True) + self.write_file(self.install_dir / "cheri_setup.csh", csh_script, overwrite=True) # write LD_64C_LIBRARY_PATH to a text file to source from sh in CheriBSD posix_sh_script = f"""#!/bin/sh rootdir=`pwd` @@ -142,7 +150,7 @@ def _set_env(self): export LD_CHERI_LIBRARY_PATH={ld_library_path}:${{LD_LIBRARY_PATH}} export LD_LIBRARY_PATH={ld_library_path}:${{LD_LIBRARY_PATH}} """ - self.write_file(self.install_dir / 'cheri_setup.sh', posix_sh_script, overwrite=True) + self.write_file(self.install_dir / "cheri_setup.sh", posix_sh_script, overwrite=True) def update(self): super().update() @@ -171,7 +179,6 @@ def install(self, **kwargs): def run_tests(self): # only test when not compiling for host if not self.compiling_for_host(): - self.target_info.run_cheribsd_test_script("run_ros2_tests.py", - mount_sourcedir=True, - mount_installdir=True, - mount_sysroot=True) + self.target_info.run_cheribsd_test_script( + "run_ros2_tests.py", mount_sourcedir=True, mount_installdir=True, mount_sysroot=True + ) diff --git a/pycheribuild/projects/cross/rtems.py b/pycheribuild/projects/cross/rtems.py index cfeb34036..d76ac745a 100644 --- a/pycheribuild/projects/cross/rtems.py +++ b/pycheribuild/projects/cross/rtems.py @@ -36,8 +36,7 @@ class BuildRtems(CrossCompileProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/rtems", - force_branch=True, default_branch="cheri_waf1") + repository = GitRepository("https://github.com/CTSRD-CHERI/rtems", force_branch=True, default_branch="cheri_waf1") target = "rtems" include_os_in_target_suffix = False dependencies = ("newlib", "compiler-rt-builtins") @@ -64,11 +63,12 @@ def _run_waf(self, *args, **kwargs): return self.run_cmd(cmdline, cwd=self.source_dir, **kwargs) def configure(self, **kwargs): - waf_run = self._run_waf("bsp_defaults", "--rtems-bsps=" + ",".join(self.rtems_bsps), "--rtems-compiler=clang", - capture_output=True) + waf_run = self._run_waf( + "bsp_defaults", "--rtems-bsps=" + ",".join(self.rtems_bsps), "--rtems-compiler=clang", capture_output=True + ) # waf configure reads config.ini by default to read RTEMS flags from - self.write_file(self.source_dir / "config.ini", str(waf_run.stdout, 'utf-8'), overwrite=True) + self.write_file(self.source_dir / "config.ini", str(waf_run.stdout, "utf-8"), overwrite=True) self._run_waf("configure", "--prefix", self.destdir) def compile(self, **kwargs): @@ -78,9 +78,11 @@ def install(self, **kwargs): self._run_waf("install") def process(self): - with self.set_env(PATH=str(self.sdk_bindir) + ":" + os.getenv("PATH", ""), - CFLAGS="--sysroot=" + str(self.sdk_sysroot), - LDFLAGS="--sysroot=" + str(self.sdk_sysroot)): + with self.set_env( + PATH=str(self.sdk_bindir) + ":" + os.getenv("PATH", ""), + CFLAGS="--sysroot=" + str(self.sdk_sysroot), + LDFLAGS="--sysroot=" + str(self.sdk_sysroot), + ): super().process() diff --git a/pycheribuild/projects/cross/simple_benchmark.py b/pycheribuild/projects/cross/simple_benchmark.py index cdc65297f..f62d58bee 100644 --- a/pycheribuild/projects/cross/simple_benchmark.py +++ b/pycheribuild/projects/cross/simple_benchmark.py @@ -42,9 +42,17 @@ class BuildSimpleCheriBenchmarks(BenchmarkMixin, CrossCompileCMakeProject): def create_test_dir(self, outdir: Path): self.clean_directory(outdir) - for f in ("run_jenkins-bluehive.sh", "libqsort_default.so", "test_qsort_default", "test_qsort_static", - "benchmark_qsort", "malloc_bench_shared", "malloc_bench_static", "malloc_benchmark.sh", - "run_cheribsd.sh"): + for f in ( + "run_jenkins-bluehive.sh", + "libqsort_default.so", + "test_qsort_default", + "test_qsort_static", + "benchmark_qsort", + "malloc_bench_shared", + "malloc_bench_static", + "malloc_benchmark.sh", + "run_cheribsd.sh", + ): self.install_file(self.build_dir / f, outdir / f, force=True, print_verbose_only=False) return outdir @@ -59,9 +67,11 @@ def run_tests(self): self.create_test_dir(self.build_dir / "test-dir") # testing, not benchmarking -> run only once: (-s small / -s large?) test_command = "cd /build/test-dir && ./run_jenkins-bluehive.sh -d0 -r1 -o {output} -a {tgt}".format( - tgt=self.archname_column, output=self.default_statcounters_csv_name) - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), mount_builddir=True) + tgt=self.archname_column, output=self.default_statcounters_csv_name + ) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", "--test-command", test_command, "--test-timeout", str(120 * 60), mount_builddir=True + ) def run_benchmarks(self): if not self.compiling_for_mips(include_purecap=True): @@ -69,6 +79,15 @@ def run_benchmarks(self): return with tempfile.TemporaryDirectory() as td: benchmarks_dir = self.create_test_dir(Path(td)) - self.run_fpga_benchmark(benchmarks_dir, output_file=self.default_statcounters_csv_name, - benchmark_script_args=["-d1", "-r10", "-o", self.default_statcounters_csv_name, - "-a", self.archname_column]) + self.run_fpga_benchmark( + benchmarks_dir, + output_file=self.default_statcounters_csv_name, + benchmark_script_args=[ + "-d1", + "-r10", + "-o", + self.default_statcounters_csv_name, + "-a", + self.archname_column, + ], + ) diff --git a/pycheribuild/projects/cross/snmalloc.py b/pycheribuild/projects/cross/snmalloc.py index 54c077c05..c718190fd 100644 --- a/pycheribuild/projects/cross/snmalloc.py +++ b/pycheribuild/projects/cross/snmalloc.py @@ -48,26 +48,27 @@ def setup_config_options(cls, **kwargs): cls.check_client = cls.add_bool_option("check-client", help="Don't accept malformed input to free") - cls.pagemap_pointers = cls.add_bool_option("pagemap-pointers", - help="Change pagemap data structure to store pointers") - cls.pagemap_rederive = cls.add_bool_option("pagemap-rederive", - help="Rederive internal pointers using the pagemap") + cls.pagemap_pointers = cls.add_bool_option( + "pagemap-pointers", help="Change pagemap data structure to store pointers" + ) + cls.pagemap_rederive = cls.add_bool_option( + "pagemap-rederive", help="Rederive internal pointers using the pagemap" + ) cls.cheri_align = cls.add_bool_option("cheri-align", help="Align sizes for CHERI bounds setting") cheri_bounds_default = cls._xtarget is not None and cls._xtarget.is_cheri_purecap() - cls.cheri_bounds = cls.add_bool_option("cheri-bounds", default=cheri_bounds_default, - help="Set bounds on returned allocations") + cls.cheri_bounds = cls.add_bool_option( + "cheri-bounds", default=cheri_bounds_default, help="Set bounds on returned allocations" + ) cls.quarantine = cls.add_bool_option("quarantine", help="Quarantine deallocations") - cls.qpathresh = cls.add_config_option("qpathresh", kind=int, - help="Quarantine physical memory per allocator threshold") - cls.qpacthresh = cls.add_config_option("qpacthresh", kind=int, - help="Quarantine chunk per allocator threshold") - cls.qcsc = cls.add_config_option("qcsc", kind=int, - help="Quarantine chunk size class") + cls.qpathresh = cls.add_config_option( + "qpathresh", kind=int, help="Quarantine physical memory per allocator threshold" + ) + cls.qpacthresh = cls.add_config_option("qpacthresh", kind=int, help="Quarantine chunk per allocator threshold") + cls.qcsc = cls.add_config_option("qcsc", kind=int, help="Quarantine chunk size class") - cls.decommit = cls.add_config_option("decommit", kind=str, - help="Specify memory decommit policy") + cls.decommit = cls.add_config_option("decommit", kind=str, help="Specify memory decommit policy") cls.zero = cls.add_bool_option("zero", help="Specify memory decommit policy") @@ -106,8 +107,9 @@ def __init__(self, *args, **kwargs): self.COMMON_FLAGS.append("-DSNMALLOC_REVOKE_PARANOIA=%d" % self.revoke_paranoia) self.COMMON_FLAGS.append("-DSNMALLOC_REVOKE_THROUGHPUT=%d" % self.revoke_tput) self.COMMON_FLAGS.append("-DSNMALLOC_QUARANTINE_CHATTY=%d" % self.revoke_verbose) - self.COMMON_FLAGS.append("-DSNMALLOC_DEFAULT_ZERO=%s" % - ("ZeroMem::YesZero" if self.zero else "ZeroMem::NoZero")) + self.COMMON_FLAGS.append( + "-DSNMALLOC_DEFAULT_ZERO=%s" % ("ZeroMem::YesZero" if self.zero else "ZeroMem::NoZero") + ) if self.decommit is not None: self.COMMON_FLAGS.append("-DUSE_DECOMMIT_STRATEGY=%s" % self.decommit) diff --git a/pycheribuild/projects/cross/sqlite.py b/pycheribuild/projects/cross/sqlite.py index e33454ed5..60c855def 100644 --- a/pycheribuild/projects/cross/sqlite.py +++ b/pycheribuild/projects/cross/sqlite.py @@ -31,8 +31,9 @@ class BuildSQLite(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/sqlite.git", - default_branch="3.22.0-cheri", force_branch=True) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/sqlite.git", default_branch="3.22.0-cheri", force_branch=True + ) def check_system_dependencies(self) -> None: super().check_system_dependencies() diff --git a/pycheribuild/projects/cross/u_boot.py b/pycheribuild/projects/cross/u_boot.py index a2c2616de..11fecb13c 100644 --- a/pycheribuild/projects/cross/u_boot.py +++ b/pycheribuild/projects/cross/u_boot.py @@ -51,8 +51,7 @@ def uboot_install_dir(config: CheriConfig, project: "BuildUBoot") -> Path: class BuildUBoot(Project): target = "u-boot" - repository = GitRepository("https://github.com/CTSRD-CHERI/u-boot", - default_branch="cheri") + repository = GitRepository("https://github.com/CTSRD-CHERI/u-boot", default_branch="cheri") dependencies = ("compiler-rt-builtins",) needs_sysroot = False # We don't need a complete sysroot default_install_dir = DefaultInstallDir.CUSTOM_INSTALL_DIR @@ -64,9 +63,9 @@ class BuildUBoot(Project): ) make_kind = MakeCommandKind.GnuMake _always_add_suffixed_targets = True - _default_install_dir_fn: ComputedDefaultValue[Path] = \ - ComputedDefaultValue(function=uboot_install_dir, - as_string="$SDK_ROOT/u-boot/riscv{32,64}{,-hybrid,-purecap}") + _default_install_dir_fn: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=uboot_install_dir, as_string="$SDK_ROOT/u-boot/riscv{32,64}{,-hybrid,-purecap}" + ) def check_system_dependencies(self) -> None: super().check_system_dependencies() @@ -95,7 +94,7 @@ def setup(self) -> None: # any relocations (and BFD lets it do this). OBJCOPY=str(self.sdk_bindir / "llvm-objcopy") + " --allow-broken-links", OBJDUMP=self.sdk_bindir / "llvm-objdump", - ) + ) self.kconfig_overrides = { "CONFIG_SREC": False, @@ -124,8 +123,9 @@ def firmware_path(self) -> Path: return self.install_dir / "u-boot" @classmethod - def get_firmware_path(cls, caller, config: "Optional[CheriConfig]" = None, - cross_target: "Optional[CrossCompileTarget]" = None): + def get_firmware_path( + cls, caller, config: "Optional[CheriConfig]" = None, cross_target: "Optional[CrossCompileTarget]" = None + ): return cls.get_instance(caller, config=config, cross_target=cross_target).firmware_path def configure(self, **kwargs): @@ -156,7 +156,7 @@ def install(self, **kwargs): self.install_file(self.build_dir / "u-boot", qemu_fw_path) def run_make(self, *args, **kwargs): - if 'cwd' in kwargs: - assert kwargs['cwd'] == self.build_dir - del kwargs['cwd'] + if "cwd" in kwargs: + assert kwargs["cwd"] == self.build_dir + del kwargs["cwd"] super().run_make(*args, **kwargs, cwd=self.source_dir) diff --git a/pycheribuild/projects/cross/wayland.py b/pycheribuild/projects/cross/wayland.py index 401492b68..477baf3b8 100644 --- a/pycheribuild/projects/cross/wayland.py +++ b/pycheribuild/projects/cross/wayland.py @@ -40,9 +40,11 @@ class BuildEPollShim(CrossCompileCMakeProject): target = "epoll-shim" - repository = GitRepository("https://github.com/jiixyj/epoll-shim", - temporary_url_override="https://github.com/arichardson/epoll-shim", - url_override_reason="https://github.com/jiixyj/epoll-shim/pull/36") + repository = GitRepository( + "https://github.com/jiixyj/epoll-shim", + temporary_url_override="https://github.com/arichardson/epoll-shim", + url_override_reason="https://github.com/jiixyj/epoll-shim/pull/36", + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def configure(self, **kwargs): @@ -70,8 +72,9 @@ def run_tests(self): class BuildLibUdevDevd(CrossCompileMesonProject): target = "libudev-devd" - repository = GitRepository("https://github.com/wulf7/libudev-devd", - old_urls=[b"https://github.com/FreeBSDDesktop/libudev-devd"]) + repository = GitRepository( + "https://github.com/wulf7/libudev-devd", old_urls=[b"https://github.com/FreeBSDDesktop/libudev-devd"] + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.NATIVE_IF_FREEBSD dependencies = ("linux-input-h",) @@ -96,8 +99,12 @@ def process(self): dev_evdev_h = src_headers / "dev/evdev" / header if not dev_evdev_h.is_file(): self.fatal("Missing evdev header:", dev_evdev_h) - self.write_file(self.include_install_dir / "linux" / header, contents=f"#include \n", - overwrite=True, print_verbose_only=False) + self.write_file( + self.include_install_dir / "linux" / header, + contents=f"#include \n", + overwrite=True, + print_verbose_only=False, + ) @property def include_install_dir(self) -> Path: @@ -168,17 +175,22 @@ def setup(self): class BuildDejaGNU(AutotoolsProject): - repository = GitRepository("https://git.savannah.gnu.org/git/dejagnu.git", - temporary_url_override="https://github.com/arichardson/dejagnu.git", - url_override_reason="Remote test execution is broken(-ish) upstream") + repository = GitRepository( + "https://git.savannah.gnu.org/git/dejagnu.git", + temporary_url_override="https://github.com/arichardson/dejagnu.git", + url_override_reason="Remote test execution is broken(-ish) upstream", + ) native_install_dir = DefaultInstallDir.BOOTSTRAP_TOOLS class BuildLibFFI(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/libffi/libffi.git", - temporary_url_override="https://github.com/CTSRD-CHERI/libffi.git", - default_branch="v3.4.4-cheriabi", force_branch=True, - url_override_reason="Needs lots of CHERI fixes") + repository = GitRepository( + "https://github.com/libffi/libffi.git", + temporary_url_override="https://github.com/CTSRD-CHERI/libffi.git", + default_branch="v3.4.4-cheriabi", + force_branch=True, + url_override_reason="Needs lots of CHERI fixes", + ) target = "libffi" supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE @@ -193,29 +205,46 @@ def setup(self): def run_tests(self): runtest_cmd = shutil.which("runtest") if not runtest_cmd: - self.dependency_error("DejaGNU is not installed.", - install_instructions=OSInfo.install_instructions("runtest", False, default="dejagnu", - apt="dejagnu", homebrew="deja-gnu"), - cheribuild_target="dejagnu", cheribuild_xtarget=CompilationTargets.NATIVE) + self.dependency_error( + "DejaGNU is not installed.", + install_instructions=OSInfo.install_instructions( + "runtest", False, default="dejagnu", apt="dejagnu", homebrew="deja-gnu" + ), + cheribuild_target="dejagnu", + cheribuild_xtarget=CompilationTargets.NATIVE, + ) runtest_flags = "-a" if self.config.debug_output: runtest_flags += " -v -v -v" elif self.config.verbose: runtest_flags += " -v" if self.compiling_for_host(): - self.run_cmd("make", "check", f"RUNTESTFLAGS={runtest_flags}", cwd=self.build_dir, - env=dict(DEJAGNU=self.source_dir / ".ci/site.exp", BOARDSDIR=self.source_dir / ".ci")) + self.run_cmd( + "make", + "check", + f"RUNTESTFLAGS={runtest_flags}", + cwd=self.build_dir, + env=dict(DEJAGNU=self.source_dir / ".ci/site.exp", BOARDSDIR=self.source_dir / ".ci"), + ) elif self.target_info.is_cheribsd(): # We need two minor fixes for SSH execution: - runtest_ver = get_program_version(Path(runtest_cmd or "runtest"), program_name=b"DejaGnu", - config=self.config) + runtest_ver = get_program_version( + Path(runtest_cmd or "runtest"), program_name=b"DejaGnu", config=self.config + ) if runtest_ver < (1, 6, 4): - self.dependency_error("DejaGnu version", runtest_ver, "cannot be used to run tests remotely,", - "please install a newer version with cheribuild", - cheribuild_target="dejagnu", cheribuild_xtarget=CompilationTargets.NATIVE) + self.dependency_error( + "DejaGnu version", + runtest_ver, + "cannot be used to run tests remotely,", + "please install a newer version with cheribuild", + cheribuild_target="dejagnu", + cheribuild_xtarget=CompilationTargets.NATIVE, + ) if self.can_run_binaries_on_remote_morello_board(): - self.write_file(self.build_dir / "site.exp", contents=f""" + self.write_file( + self.build_dir / "site.exp", + contents=f""" if ![info exists boards_dir] {{ set boards_dir {{}} }} @@ -223,11 +252,15 @@ def run_tests(self): verbose "Global Config File: target_triplet is $target_triplet" 2 global target_list set target_list "remote-cheribsd" -""", overwrite=True) +""", + overwrite=True, + ) ssh_options = "-o NoHostAuthenticationForLocalhost=yes" ssh_port = ssh_config_parameters(self.config.remote_morello_board, self.config).get("port", "22") ssh_user = ssh_config_parameters(self.config.remote_morello_board, self.config).get("user", "root") - self.write_file(self.build_dir / "remote-cheribsd.exp", contents=f""" + self.write_file( + self.build_dir / "remote-cheribsd.exp", + contents=f""" load_generic_config "unix" set_board_info connect ssh set_board_info hostname {self.config.remote_morello_board} @@ -239,15 +272,24 @@ def run_tests(self): # set_board_info exec_shell "gdb-run-noninteractive.sh" # Build tests statically linked so they pick up the local libffi library set TOOL_OPTIONS -static -""", overwrite=True) - self.run_cmd(["make", "check", - f"RUNTESTFLAGS={runtest_flags} --target-board remote-cheribsd --xml"], - env=dict(BOARDSDIR=self.build_dir, DEJAGNU=self.build_dir / "site.exp"), - cwd=str(self.build_dir)) +""", + overwrite=True, + ) + self.run_cmd( + ["make", "check", f"RUNTESTFLAGS={runtest_flags} --target-board remote-cheribsd --xml"], + env=dict(BOARDSDIR=self.build_dir, DEJAGNU=self.build_dir / "site.exp"), + cwd=str(self.build_dir), + ) else: - self.target_info.run_cheribsd_test_script("run_libffi_tests.py", "--test-timeout", str(120 * 60), - mount_builddir=True, mount_sourcedir=True, - mount_sysroot=False, use_full_disk_image=True) + self.target_info.run_cheribsd_test_script( + "run_libffi_tests.py", + "--test-timeout", + str(120 * 60), + mount_builddir=True, + mount_sourcedir=True, + mount_sysroot=False, + use_full_disk_image=True, + ) class BuildWayland(CrossCompileMesonProject): @@ -264,8 +306,13 @@ def dependencies(cls, config: CheriConfig) -> "tuple[str, ...]": if target.target_info_cls.is_freebsd(): deps += ("epoll-shim",) return deps - repository = GitRepository("https://gitlab.freedesktop.org/wayland/wayland.git", default_branch="main", - force_branch=True, old_urls=[b"https://github.com/CTSRD-CHERI/wayland"]) + + repository = GitRepository( + "https://gitlab.freedesktop.org/wayland/wayland.git", + default_branch="main", + force_branch=True, + old_urls=[b"https://github.com/CTSRD-CHERI/wayland"], + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def setup(self): diff --git a/pycheribuild/projects/cross/webkit.py b/pycheribuild/projects/cross/webkit.py index f0a472b79..48ea78eba 100644 --- a/pycheribuild/projects/cross/webkit.py +++ b/pycheribuild/projects/cross/webkit.py @@ -41,30 +41,39 @@ class JsBackend(Enum): class BuildMorelloWebkit(CrossCompileCMakeProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/webkit", - default_branch="master") + repository = GitRepository("https://github.com/CTSRD-CHERI/webkit", default_branch="master") default_directory_basename = "webkit" target = "morello-webkit" dependencies = ("icu4c",) native_install_dir = DefaultInstallDir.DO_NOT_INSTALL cross_install_dir = DefaultInstallDir.ROOTFS_OPTBASE tier2ptrliterals = BoolConfigOption( - "tier2ptrliterals", default=True, show_help=True, + "tier2ptrliterals", + default=True, + show_help=True, help="When true pointers are represented as atomic literals and loaded as data and when false pointers " - "are represented as numeric values which can be splitted and are encoded into instructions. " - "This option only affects the non-purecap tier2 backend.") + "are represented as numeric values which can be splitted and are encoded into instructions. " + "This option only affects the non-purecap tier2 backend.", + ) jsheapoffsets = BoolConfigOption( - "jsheapoffsets", default=False, show_help=True, + "jsheapoffsets", + default=False, + show_help=True, help="Use offsets into the JS heap for object references instead of capabilities. " - "This option only affects the purecap backends.") + "This option only affects the purecap backends.", + ) @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.backend = cls.add_config_option( - "backend", kind=JsBackend, - default=JsBackend.CLOOP, enum_choice_strings=[t.value for t in JsBackend], - show_help=True, help="The JavaScript backend to use for building WebKit") + "backend", + kind=JsBackend, + default=JsBackend.CLOOP, + enum_choice_strings=[t.value for t in JsBackend], + show_help=True, + help="The JavaScript backend to use for building WebKit", + ) @property def build_dir_suffix(self): @@ -109,7 +118,7 @@ def setup(self): ENABLE_YARR_JIT=False, ENABLE_SAMPLING_PROFILER=False, CHERI_PURE_CAPABILITY=self.crosscompile_target.is_cheri_purecap(), - ) + ) if self.crosscompile_target.is_cheri_purecap(): # TODO: we can get this from the pre-processor instead @@ -122,19 +131,29 @@ def setup(self): # Add options for each backend if self.backend == JsBackend.CLOOP: - self.add_cmake_options(ENABLE_C_LOOP=True, ENABLE_ASSEMBLER=False, ENABLE_JIT=False, - ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=False) + self.add_cmake_options( + ENABLE_C_LOOP=True, + ENABLE_ASSEMBLER=False, + ENABLE_JIT=False, + ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=False, + ) elif self.backend == JsBackend.TIER1ASM: - self.add_cmake_options(ENABLE_C_LOOP=False, ENABLE_ASSEMBLER=True, ENABLE_JIT=False, - ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=True) + self.add_cmake_options( + ENABLE_C_LOOP=False, + ENABLE_ASSEMBLER=True, + ENABLE_JIT=False, + ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=True, + ) elif self.backend == JsBackend.TIER2ASM: - self.add_cmake_options(ENABLE_C_LOOP=False, ENABLE_ASSEMBLER=True, ENABLE_DISASSEMBLER=True, - ENABLE_JIT=True) + self.add_cmake_options( + ENABLE_C_LOOP=False, ENABLE_ASSEMBLER=True, ENABLE_DISASSEMBLER=True, ENABLE_JIT=True + ) if self.crosscompile_target.is_cheri_purecap(): if self.jsheapoffsets: self.warning("integer heap offsets are not yet supported for the tier 2 backend!") - self.add_cmake_options(ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=True, - ENABLE_JSHEAP_CHERI_OFFSET_REFS=False) + self.add_cmake_options( + ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=True, ENABLE_JSHEAP_CHERI_OFFSET_REFS=False + ) else: self.add_cmake_options(ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=self.tier2ptrliterals) @@ -143,5 +162,6 @@ def run_tests(self): self.fatal("Running host tests not implemented") else: # full disk image to get icu library - self.target_info.run_cheribsd_test_script("run_morello_webkit_tests.py", mount_sourcedir=True, - use_full_disk_image=True) + self.target_info.run_cheribsd_test_script( + "run_morello_webkit_tests.py", mount_sourcedir=True, use_full_disk_image=True + ) diff --git a/pycheribuild/projects/cross/x11.py b/pycheribuild/projects/cross/x11.py index d17295cf6..a6b9ab9f4 100644 --- a/pycheribuild/projects/cross/x11.py +++ b/pycheribuild/projects/cross/x11.py @@ -110,8 +110,9 @@ def compile(self, **kwargs): if self.config.pretend and not shutil.which("automake"): automake_version = (0, 0, 0) else: - automake_version = get_program_version(Path("automake"), config=self.config, - regex=rb"automake\s+\(GNU automake\)\s+(\d+)\.(\d+)\.?(\d+)?") + automake_version = get_program_version( + Path("automake"), config=self.config, regex=rb"automake\s+\(GNU automake\)\s+(\d+)\.(\d+)\.?(\d+)?" + ) if automake_version >= (1, 16, 4): self.info("Working around https://www.mail-archive.com/bug-automake@gnu.org/msg04957.html") self.replace_in_file(self.build_dir / "xcb-proto.pc", {"${PYTHON_PREFIX}": str(self.install_prefix)}) @@ -311,8 +312,9 @@ def install(self, **kwargs): super().install(**kwargs) if not self.compiling_for_host() and not self.crosscompile_target.is_libcompat_target(): # Ensure that xauth is in the default $PATH, so that ssh -X works - self.create_symlink(self.install_dir / "bin/xauth", self.rootfs_dir / "usr/local/bin/xauth", - print_verbose_only=False) + self.create_symlink( + self.install_dir / "bin/xauth", self.rootfs_dir / "usr/local/bin/xauth", print_verbose_only=False + ) class BuildXEyes(X11AutotoolsProject): @@ -329,12 +331,14 @@ class BuildLibXKBCommon(X11MesonProject): def setup(self): # avoid wayland dep for now super().setup() - self.add_meson_options(**{ - "enable-x11": True, - "enable-xkbregistry": False, # Avoid dependency on libxml2 - "enable-docs": False, # Avoid lots of dependencies to build docs - "enable-tools": False, # No need for wayland deps just for the xkbcli tool - }) + self.add_meson_options( + **{ + "enable-x11": True, + "enable-xkbregistry": False, # Avoid dependency on libxml2 + "enable-docs": False, # Avoid lots of dependencies to build docs + "enable-tools": False, # No need for wayland deps just for the xkbcli tool + } + ) def process(self): newpath = os.getenv("PATH") @@ -353,8 +357,10 @@ class BuildXorgFontUtil(X11AutotoolsProject): class BuildPixman(X11MesonProject): target = "pixman" dependencies = ("libpng",) - repository = GitRepository("https://gitlab.freedesktop.org/pixman/pixman.git", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/pixman.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/pixman/pixman.git", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/pixman.git"], + ) def setup(self): super().setup() @@ -417,8 +423,10 @@ class BuildLibXScrnSaver(X11AutotoolsProject): class BuildLibJpegTurbo(X11CMakeProject): target = "libjpeg-turbo" - repository = GitRepository("https://github.com/libjpeg-turbo/libjpeg-turbo.git", - old_urls=[b"https://github.com/arichardson/libjpeg-turbo.git"]) + repository = GitRepository( + "https://github.com/libjpeg-turbo/libjpeg-turbo.git", + old_urls=[b"https://github.com/arichardson/libjpeg-turbo.git"], + ) def setup(self): super().setup() @@ -487,14 +495,28 @@ class BuildXVncServer(X11AutotoolsProject): target = "xvnc-server" # The actual XVnc source code is part of TigerVNC and not included in the xserver repository. # It also depends on build artifacts from an existing tigervnc build - dependencies = ("libx11", "xorg-font-util", "libxrender", "libxfont", "libxkbfile", "tigervnc", "xkeyboard-config", - "xkbcomp", "dbus") + dependencies = ( + "libx11", + "xorg-font-util", + "libxrender", + "libxfont", + "libxkbfile", + "tigervnc", + "xkeyboard-config", + "xkbcomp", + "dbus", + ) # The tigervnc code requires the 1.20 release - repository = GitRepository("https://gitlab.freedesktop.org/xorg/xserver.git", - default_branch="server-1.20-branch", force_branch=True, - temporary_url_override="https://gitlab.freedesktop.org/arichardson/xserver.git", - url_override_reason=["https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/721", - "https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/720"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/xorg/xserver.git", + default_branch="server-1.20-branch", + force_branch=True, + temporary_url_override="https://gitlab.freedesktop.org/arichardson/xserver.git", + url_override_reason=[ + "https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/721", + "https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/720", + ], + ) def install(self, **kwargs): """ @@ -507,14 +529,22 @@ def install(self, **kwargs): super().install() # Install a script to start the Xvnc so I don't have to remember the arguments # TODO: should we install a service that we can start with `service xvnc start`? - self.write_file(self.install_dir / "bin/startxvnc", overwrite=True, mode=0o755, - contents="#!/bin/sh\nXvnc -geometry 1024x768 -SecurityTypes=None \"$@\"\n") + self.write_file( + self.install_dir / "bin/startxvnc", + overwrite=True, + mode=0o755, + contents='#!/bin/sh\nXvnc -geometry 1024x768 -SecurityTypes=None "$@"\n', + ) if not self.compiling_for_host() and not self.crosscompile_target.is_libcompat_target(): # Ensure that Xvnc is in the default $PATH - self.create_symlink(self.install_dir / "bin/Xvnc", self.rootfs_dir / "usr/local/bin/Xvnc", - print_verbose_only=False) - self.create_symlink(self.install_dir / "bin/startxvnc", self.rootfs_dir / "usr/local/bin/startxvnc", - print_verbose_only=False) + self.create_symlink( + self.install_dir / "bin/Xvnc", self.rootfs_dir / "usr/local/bin/Xvnc", print_verbose_only=False + ) + self.create_symlink( + self.install_dir / "bin/startxvnc", + self.rootfs_dir / "usr/local/bin/startxvnc", + print_verbose_only=False, + ) def update(self): super().update() @@ -523,12 +553,27 @@ def update(self): self.create_symlink(tigervnc_source / "unix/xserver/hw/vnc", self.source_dir / "hw/vnc") try: # Check if the patch was already applied with --dry-run - self.run_cmd("patch", "-p1", "--forward", "--dry-run", "-i", - tigervnc_source / "unix/xserver120.patch", cwd=self.source_dir, capture_error=True, - capture_output=True) - self.run_cmd("patch", "-p1", "--forward", "-i", - tigervnc_source / "unix/xserver120.patch", cwd=self.source_dir, capture_error=True, - capture_output=True) + self.run_cmd( + "patch", + "-p1", + "--forward", + "--dry-run", + "-i", + tigervnc_source / "unix/xserver120.patch", + cwd=self.source_dir, + capture_error=True, + capture_output=True, + ) + self.run_cmd( + "patch", + "-p1", + "--forward", + "-i", + tigervnc_source / "unix/xserver120.patch", + cwd=self.source_dir, + capture_error=True, + capture_output=True, + ) except subprocess.CalledProcessError as e: if b"Skipping patch" in e.stdout: return @@ -537,23 +582,37 @@ def update(self): def setup(self): super().setup() fonts_dir = Path("/", self.target_info.sysroot_install_prefix_relative, "share/fonts") - self.configure_args.extend([ - "--without-dtrace", "--enable-static", "--disable-dri", "--disable-unit-tests", - "--disable-xinerama", "--disable-xvfb", "--disable-xnest", "--disable-xorg", - "--disable-dmx", "--disable-xwin", "--disable-xephyr", "--disable-kdrive", - "--disable-libdrm", - "--disable-config-dbus", "--disable-config-hal", - "--disable-dri", "--disable-dri2", "--disable-dri3", - "--enable-install-libxf86config", - "--disable-glx", # "--enable-glx", - "-with-default-font-path=catalogue:" + str(fonts_dir) + ",built-ins", - "--with-serverconfig-path=" + str(self.install_prefix / "lib/X11"), - "--disable-selective-werror", - "--disable-xwayland", - "--with-fontrootdir=" + str(fonts_dir), - "--with-xkb-path=" + str(BuildXKeyboardConfig.get_instance(self).install_prefix / "share/X11/xkb"), - "--with-xkb-bin-directory=" + str(BuildXKkbcomp.get_instance(self).install_prefix / "bin"), - ]) + self.configure_args.extend( + [ + "--without-dtrace", + "--enable-static", + "--disable-dri", + "--disable-unit-tests", + "--disable-xinerama", + "--disable-xvfb", + "--disable-xnest", + "--disable-xorg", + "--disable-dmx", + "--disable-xwin", + "--disable-xephyr", + "--disable-kdrive", + "--disable-libdrm", + "--disable-config-dbus", + "--disable-config-hal", + "--disable-dri", + "--disable-dri2", + "--disable-dri3", + "--enable-install-libxf86config", + "--disable-glx", # "--enable-glx", + "-with-default-font-path=catalogue:" + str(fonts_dir) + ",built-ins", + "--with-serverconfig-path=" + str(self.install_prefix / "lib/X11"), + "--disable-selective-werror", + "--disable-xwayland", + "--with-fontrootdir=" + str(fonts_dir), + "--with-xkb-path=" + str(BuildXKeyboardConfig.get_instance(self).install_prefix / "share/X11/xkb"), + "--with-xkb-bin-directory=" + str(BuildXKkbcomp.get_instance(self).install_prefix / "bin"), + ] + ) tigervnc = BuildTigerVNC.get_instance(self) self.make_args.set(TIGERVNC_SRCDIR=tigervnc.source_dir, TIGERVNC_BUILDDIR=tigervnc.build_dir) self.COMMON_LDFLAGS.append("-Wl,-rpath," + str(BuildFreeType2.get_instance(self).install_prefix / "lib")) @@ -611,10 +670,17 @@ def setup(self): # Slightly more functional window manager than TWM class BuildIceWM(X11CMakeProject): target = "icewm" - dependencies = ("fontconfig", "libxcomposite", "libxdamage", "libpng", "libjpeg-turbo", - "libxpm", "libxft", "libxrandr") - repository = GitRepository("https://github.com/bbidulock/icewm", - old_urls=[b"https://github.com/arichardson/icewm"]) + dependencies = ( + "fontconfig", + "libxcomposite", + "libxdamage", + "libpng", + "libjpeg-turbo", + "libxpm", + "libxft", + "libxrandr", + ) + repository = GitRepository("https://github.com/bbidulock/icewm", old_urls=[b"https://github.com/arichardson/icewm"]) def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/zlib.py b/pycheribuild/projects/cross/zlib.py index e6e0973ff..978be83a0 100644 --- a/pycheribuild/projects/cross/zlib.py +++ b/pycheribuild/projects/cross/zlib.py @@ -47,5 +47,8 @@ def setup(self): if not self.compiling_for_host(): # If we don't set this, the build will use the macOS host libtool instead of llvm-ar and then complain # because the .o files are not macOS object files. - self.add_configure_vars(uname=self.target_info.cmake_system_name, - AR=self.sdk_bindir / "llvm-ar", RANLIB=self.sdk_bindir / "llvm-ranlib") + self.add_configure_vars( + uname=self.target_info.cmake_system_name, + AR=self.sdk_bindir / "llvm-ar", + RANLIB=self.sdk_bindir / "llvm-ranlib", + )