From f3fae79ad5fd02ca303424d2c4b12f3fdd0b6144 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Sun, 18 Feb 2024 22:17:21 -0800 Subject: [PATCH] Reformat the remaining files --- pycheribuild/__main__.py | 4 +- pycheribuild/boot_cheribsd/__init__.py | 728 ++++++++++++------ pycheribuild/config/chericonfig.py | 609 ++++++++++----- pycheribuild/config/compilation_targets.py | 3 +- pycheribuild/config/computed_default_value.py | 19 +- pycheribuild/config/config_loader_base.py | 130 +++- pycheribuild/config/defaultconfig.py | 201 +++-- pycheribuild/config/jenkinsconfig.py | 183 +++-- pycheribuild/config/loader.py | 326 +++++--- pycheribuild/config/target_info.py | 236 ++++-- pycheribuild/targets.py | 4 +- pycheribuild/utils.py | 9 +- tests/test_argument_parsing.py | 12 +- tests/test_metalog.py | 5 +- 14 files changed, 1694 insertions(+), 775 deletions(-) diff --git a/pycheribuild/__main__.py b/pycheribuild/__main__.py index 9eef0e813..cf77654cc 100644 --- a/pycheribuild/__main__.py +++ b/pycheribuild/__main__.py @@ -88,7 +88,7 @@ def _update_check(config: DefaultCheriConfig, d: Path) -> None: if query_yes_no( config, f"The local {branch_info.local_branch} branch is tracking the obsolete remote 'master'" - f" branch, would you like to switch to 'main'?", + " branch, would you like to switch to 'main'?", force_result=False, ): # Update the remote ref to point to "main". @@ -140,7 +140,7 @@ def get_config_option_value(handle: ConfigOptionHandle, config: DefaultCheriConf if option.is_fallback_only: raise LookupError( f"Option '{option.full_option_name}' cannot be queried since it is a generic fallback value" - f"for a target-specific option. Please use the target-suffixed on instead." + "for a target-specific option. Please use the target-suffixed on instead." ) if option._owning_class is not None: project_cls: "type[SimpleProject]" = option._owning_class diff --git a/pycheribuild/boot_cheribsd/__init__.py b/pycheribuild/boot_cheribsd/__init__.py index 32955ca9d..0e8e6c420 100755 --- a/pycheribuild/boot_cheribsd/__init__.py +++ b/pycheribuild/boot_cheribsd/__init__.py @@ -64,14 +64,18 @@ assert str(_pexpect_dir.resolve()) in sys.path, str(_pexpect_dir) + " not found in " + str(sys.path) import pexpect # noqa: E402 -SUPPORTED_ARCHITECTURES = {x.generic_target_suffix: x for x in (CompilationTargets.CHERIBSD_RISCV_NO_CHERI, - CompilationTargets.CHERIBSD_RISCV_HYBRID, - CompilationTargets.CHERIBSD_RISCV_PURECAP, - CompilationTargets.CHERIBSD_X86_64, - CompilationTargets.CHERIBSD_AARCH64, - CompilationTargets.CHERIBSD_MORELLO_HYBRID, - CompilationTargets.CHERIBSD_MORELLO_PURECAP, - )} +SUPPORTED_ARCHITECTURES = { + x.generic_target_suffix: x + for x in ( + CompilationTargets.CHERIBSD_RISCV_NO_CHERI, + CompilationTargets.CHERIBSD_RISCV_HYBRID, + CompilationTargets.CHERIBSD_RISCV_PURECAP, + CompilationTargets.CHERIBSD_X86_64, + CompilationTargets.CHERIBSD_AARCH64, + CompilationTargets.CHERIBSD_MORELLO_HYBRID, + CompilationTargets.CHERIBSD_MORELLO_PURECAP, + ) +} # boot loader without lua: "Hit [Enter] to boot " # menu.lua before Sep 2019: ", hit [Enter] to boot " @@ -164,12 +168,20 @@ def interact(self, escape_character=chr(29), input_filter=None, output_filter=No info("Interacting with (fake) ", coloured(AnsiColour.yellow, commandline_to_str(self.cmd))) def sendcontrol(self, char): - info("Sending ", coloured(AnsiColour.yellow, "CTRL+", char), coloured(AnsiColour.blue, " to (fake) "), - coloured(AnsiColour.yellow, commandline_to_str(self.cmd))) - - def sendline(self, s=''): - info("Sending ", coloured(AnsiColour.yellow, s), coloured(AnsiColour.blue, " to (fake) "), - coloured(AnsiColour.yellow, commandline_to_str(self.cmd))) + info( + "Sending ", + coloured(AnsiColour.yellow, "CTRL+", char), + coloured(AnsiColour.blue, " to (fake) "), + coloured(AnsiColour.yellow, commandline_to_str(self.cmd)), + ) + + def sendline(self, s=""): + info( + "Sending ", + coloured(AnsiColour.yellow, s), + coloured(AnsiColour.blue, " to (fake) "), + coloured(AnsiColour.yellow, commandline_to_str(self.cmd)), + ) super().sendline(s) @@ -230,32 +242,55 @@ class CheriBSDSpawnMixin(MixinBase): def expect_exact_ignore_panic(self, patterns, *, timeout: int): return super().expect_exact(patterns, timeout=timeout) - def expect(self, patterns: PatternListType, timeout=-1, pretend_result=None, ignore_timeout=False, - log_patterns=True, timeout_msg="timeout", **kwargs): + def expect( + self, + patterns: PatternListType, + timeout=-1, + pretend_result=None, + ignore_timeout=False, + log_patterns=True, + timeout_msg="timeout", + **kwargs, + ): assert isinstance(patterns, list), "expected list and not " + str(patterns) if log_patterns: info("Expecting regex ", coloured(AnsiColour.cyan, str(patterns))) - return self._expect_and_handle_panic_impl(patterns, timeout_msg, ignore_timeout=ignore_timeout, - timeout=timeout, expect_fn=super().expect, **kwargs) - - def expect_exact(self, pattern_list: PatternListType, - timeout=-1, pretend_result=None, ignore_timeout=False, log_patterns=True, timeout_msg="timeout", - **kwargs): + return self._expect_and_handle_panic_impl( + patterns, timeout_msg, ignore_timeout=ignore_timeout, timeout=timeout, expect_fn=super().expect, **kwargs + ) + + def expect_exact( + self, + pattern_list: PatternListType, + timeout=-1, + pretend_result=None, + ignore_timeout=False, + log_patterns=True, + timeout_msg="timeout", + **kwargs, + ): assert isinstance(pattern_list, list), "expected list and not " + str(pattern_list) if log_patterns: info("Expecting literal ", coloured(AnsiColour.blue, str(pattern_list))) - return self._expect_and_handle_panic_impl(pattern_list, timeout_msg, timeout=timeout, - ignore_timeout=ignore_timeout, expect_fn=super().expect_exact, - **kwargs) + return self._expect_and_handle_panic_impl( + pattern_list, + timeout_msg, + timeout=timeout, + ignore_timeout=ignore_timeout, + expect_fn=super().expect_exact, + **kwargs, + ) def expect_prompt(self, timeout=-1, timeout_msg="timeout waiting for prompt", ignore_timeout=False, **kwargs): - result = self.expect_exact([PEXPECT_PROMPT], timeout=timeout, timeout_msg=timeout_msg, - ignore_timeout=ignore_timeout, **kwargs) + result = self.expect_exact( + [PEXPECT_PROMPT], timeout=timeout, timeout_msg=timeout_msg, ignore_timeout=ignore_timeout, **kwargs + ) time.sleep(0.05) # give QEMU a bit of time after printing the prompt (otherwise we might lose some input) return result - def _expect_and_handle_panic_impl(self, options: PatternListType, timeout_msg, *, ignore_timeout=True, - expect_fn, timeout, **kwargs): + def _expect_and_handle_panic_impl( + self, options: PatternListType, timeout_msg, *, ignore_timeout=True, expect_fn, timeout, **kwargs + ): panic_regexes = [PANIC, STOPPED, PANIC_KDB, PANIC_PAGE_FAULT, PANIC_MORELLO_CAP_ABORT, PANIC_IN_BACKTRACE] for i in panic_regexes: assert i not in options @@ -275,15 +310,32 @@ def _expect_and_handle_panic_impl(self, options: PatternListType, timeout_msg, * else: raise e - def run(self, cmd: str, *, expected_output=None, error_output=None, cheri_trap_fatal=True, ignore_cheri_trap=False, - timeout=600): - run_cheribsd_command(self, cmd, expected_output=expected_output, error_output=error_output, - cheri_trap_fatal=cheri_trap_fatal, ignore_cheri_trap=ignore_cheri_trap, timeout=timeout) - - def checked_run(self, cmd: str, *, timeout=600, ignore_cheri_trap=False, error_output: "Optional[str]" = None, - **kwargs): - checked_run_cheribsd_command(self, cmd, timeout=timeout, ignore_cheri_trap=ignore_cheri_trap, - error_output=error_output, **kwargs) + def run( + self, + cmd: str, + *, + expected_output=None, + error_output=None, + cheri_trap_fatal=True, + ignore_cheri_trap=False, + timeout=600, + ): + run_cheribsd_command( + self, + cmd, + expected_output=expected_output, + error_output=error_output, + cheri_trap_fatal=cheri_trap_fatal, + ignore_cheri_trap=ignore_cheri_trap, + timeout=timeout, + ) + + def checked_run( + self, cmd: str, *, timeout=600, ignore_cheri_trap=False, error_output: "Optional[str]" = None, **kwargs + ): + checked_run_cheribsd_command( + self, cmd, timeout=timeout, ignore_cheri_trap=ignore_cheri_trap, error_output=error_output, **kwargs + ) class CheriBSDInstance(CheriBSDSpawnMixin, pexpect.spawn): @@ -298,8 +350,7 @@ class QemuCheriBSDInstance(CheriBSDInstance): smb_dirs: "list[SmbMount]" = None flush_interval = None - def __init__(self, qemu_config: QemuOptions, *args, ssh_port: Optional[int], - ssh_pubkey: Optional[Path], **kwargs): + def __init__(self, qemu_config: QemuOptions, *args, ssh_port: Optional[int], ssh_pubkey: Optional[Path], **kwargs): super().__init__(qemu_config.xtarget, *args, **kwargs) self.qemu_config = qemu_config self.should_quit = False @@ -315,35 +366,61 @@ def __init__(self, qemu_config: QemuOptions, *args, ssh_port: Optional[int], @property def ssh_private_key(self): if self._ssh_private_key is None: - failure("Attempted to use SSH without specifying a key, please pass --test-ssh-key=/path/to/id_foo.pub to " - "cheribuild.", exit=True) + failure( + "Attempted to use SSH without specifying a key, please pass --test-ssh-key=/path/to/id_foo.pub to " + "cheribuild.", + exit=True, + ) assert self._ssh_private_key != self.ssh_public_key, (self._ssh_private_key, "!=", self.ssh_public_key) return self._ssh_private_key @staticmethod def _ssh_options(use_controlmaster: bool): - result = ["-o", "UserKnownHostsFile=/dev/null", - "-o", "StrictHostKeyChecking=no", - "-o", "NoHostAuthenticationForLocalhost=yes", - # "-o", "ConnectTimeout=20", - # "-o", "ConnectionAttempts=2", - ] + result = [ + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "StrictHostKeyChecking=no", + "-o", + "NoHostAuthenticationForLocalhost=yes", + # "-o", "ConnectTimeout=20", + # "-o", "ConnectionAttempts=2", + ] if use_controlmaster: # XXX: always use controlmaster for faster connections? controlmaster_dir = Path.home() / ".ssh/controlmasters" controlmaster_dir.mkdir(exist_ok=True) - result += ["-o", f"ControlPath={controlmaster_dir}/%r@%h:%p", - "-o", "ControlMaster=auto", - # Keep socket open for 10 min (600) or indefinitely (yes) - "-o", "ControlPersist=600"] + result += [ + "-o", + f"ControlPath={controlmaster_dir}/%r@%h:%p", + "-o", + "ControlMaster=auto", + # Keep socket open for 10 min (600) or indefinitely (yes) + "-o", + "ControlPersist=600", + ] return result - def run_command_via_ssh(self, command: "list[str]", *, stdout=None, stderr=None, check=True, verbose=False, - use_controlmaster=False, **kwargs) -> "subprocess.CompletedProcess[bytes]": + def run_command_via_ssh( + self, + command: "list[str]", + *, + stdout=None, + stderr=None, + check=True, + verbose=False, + use_controlmaster=False, + **kwargs, + ) -> "subprocess.CompletedProcess[bytes]": assert self.ssh_port is not None - ssh_command = ["ssh", "{user}@{host}".format(user=self.ssh_user, host="localhost"), - "-p", str(self.ssh_port), - "-i", str(self.ssh_private_key)] + ssh_command = [ + "ssh", + "{user}@{host}".format(user=self.ssh_user, host="localhost"), + "-p", + str(self.ssh_port), + "-i", + str(self.ssh_private_key), + ] if verbose: ssh_command.append("-v") ssh_command.extend(self._ssh_options(use_controlmaster=use_controlmaster)) @@ -354,8 +431,9 @@ def run_command_via_ssh(self, command: "list[str]", *, stdout=None, stderr=None, def check_ssh_connection(self, prefix="SSH connection:"): connection_test_start = datetime.datetime.utcnow() - result = self.run_command_via_ssh(["echo", "connection successful"], check=True, stdout=subprocess.PIPE, - verbose=True) + result = self.run_command_via_ssh( + ["echo", "connection successful"], check=True, stdout=subprocess.PIPE, verbose=True + ) connection_time = (datetime.datetime.utcnow() - connection_test_start).total_seconds() info(prefix, result.stdout) if result.stdout != b"connection successful\n": @@ -444,10 +522,11 @@ def is_newer(path1: Path, path2: Path): def prepend_ld_library_path(qemu: CheriBSDInstance, path: str): - qemu.run("export LD_LIBRARY_PATH=" + path + ":$LD_LIBRARY_PATH; echo \"$LD_LIBRARY_PATH\"", timeout=3) - qemu.run("export LD_64C_LIBRARY_PATH=" + path + ":$LD_64C_LIBRARY_PATH; echo \"$LD_64C_LIBRARY_PATH\"", timeout=3) - qemu.run("export LD_CHERI_LIBRARY_PATH=" + path + ":$LD_CHERI_LIBRARY_PATH; echo \"$LD_CHERI_LIBRARY_PATH\"", - timeout=3) + qemu.run("export LD_LIBRARY_PATH=" + path + ':$LD_LIBRARY_PATH; echo "$LD_LIBRARY_PATH"', timeout=3) + qemu.run("export LD_64C_LIBRARY_PATH=" + path + ':$LD_64C_LIBRARY_PATH; echo "$LD_64C_LIBRARY_PATH"', timeout=3) + qemu.run( + "export LD_CHERI_LIBRARY_PATH=" + path + ':$LD_CHERI_LIBRARY_PATH; echo "$LD_CHERI_LIBRARY_PATH"', timeout=3 + ) def set_ld_library_path_with_sysroot(qemu: CheriBSDInstance): @@ -457,8 +536,11 @@ def set_ld_library_path_with_sysroot(qemu: CheriBSDInstance): local_dir = "usr/local" if qemu.xtarget.target_info_cls.is_cheribsd(): local_dir += "/" + qemu.xtarget.generic_arch_suffix - qemu.run("export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" - "/sysroot/{prefix}/{l}:${var}".format(prefix=local_dir, l="lib", var="LD_LIBRARY_PATH"), timeout=3) + qemu.run( + "export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" + "/sysroot/{prefix}/{l}:${var}".format(prefix=local_dir, l="lib", var="LD_LIBRARY_PATH"), + timeout=3, + ) return purecap_install_prefix = "usr/local/" + qemu.xtarget.get_cheri_purecap_target().generic_arch_suffix @@ -467,21 +549,34 @@ def set_ld_library_path_with_sysroot(qemu: CheriBSDInstance): noncheri_ld_lib_path_var = "LD_LIBRARY_PATH" if not qemu.xtarget.is_cheri_purecap() else "LD_64_LIBRARY_PATH" cheri_ld_lib_path_var = "LD_LIBRARY_PATH" if qemu.xtarget.is_cheri_purecap() else "LD_64C_LIBRARY_PATH" - qemu.run("export {var}=/{lib}:/usr/{lib}:/usr/local/{lib}:/sysroot/{lib}:/sysroot/usr/{lib}:/sysroot/{hybrid}/lib:" - "/sysroot/usr/local/{lib}:/sysroot/{noncheri}/lib:${var}".format( - lib=non_cheri_libdir, hybrid=hybrid_install_prefix, noncheri=nocheri_install_prefix, - var=noncheri_ld_lib_path_var), timeout=3) - qemu.run("export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" - "/sysroot/{prefix}/lib:${var}".format(prefix=purecap_install_prefix, l=cheri_libdir, - var=cheri_ld_lib_path_var), timeout=3) + qemu.run( + "export {var}=/{lib}:/usr/{lib}:/usr/local/{lib}:/sysroot/{lib}:/sysroot/usr/{lib}:/sysroot/{hybrid}/lib:" + "/sysroot/usr/local/{lib}:/sysroot/{noncheri}/lib:${var}".format( + lib=non_cheri_libdir, + hybrid=hybrid_install_prefix, + noncheri=nocheri_install_prefix, + var=noncheri_ld_lib_path_var, + ), + timeout=3, + ) + qemu.run( + "export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" + "/sysroot/{prefix}/lib:${var}".format(prefix=purecap_install_prefix, l=cheri_libdir, var=cheri_ld_lib_path_var), + timeout=3, + ) if cheri_ld_lib_path_var == "LD_64C_LIBRARY_PATH": - qemu.run("export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" - "/sysroot/{prefix}/lib:${var}".format(prefix=purecap_install_prefix, l=cheri_libdir, - var="LD_CHERI_LIBRARY_PATH"), timeout=3) - - -def maybe_decompress(path: Path, force_decompression: bool, keep_archive=True, - args: "Optional[argparse.Namespace]" = None, *, what: str) -> Path: + qemu.run( + "export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" + "/sysroot/{prefix}/lib:${var}".format( + prefix=purecap_install_prefix, l=cheri_libdir, var="LD_CHERI_LIBRARY_PATH" + ), + timeout=3, + ) + + +def maybe_decompress( + path: Path, force_decompression: bool, keep_archive=True, args: "Optional[argparse.Namespace]" = None, *, what: str +) -> Path: # drop the suffix and then try decompressing def bunzip(archive): return decompress(archive, force_decompression, cmd=["bunzip2", "-v", "-f"], keep_archive=keep_archive) @@ -548,18 +643,30 @@ def debug_kernel_panic(qemu: CheriBSDSpawnMixin): SH_PROGRAM_NOT_FOUND = re.compile("/bin/sh: [/\\w\\d_-]+: not found") -RTLD_DSO_NOT_FOUND = re.compile("ld-elf[\\w\\d_-]*.so.1: Shared object \".+\" not found, required by \".+\"") - - -def run_cheribsd_command(qemu: CheriBSDSpawnMixin, cmd: str, expected_output=None, error_output=None, - cheri_trap_fatal=True, ignore_cheri_trap=False, timeout=60): +RTLD_DSO_NOT_FOUND = re.compile('ld-elf[\\w\\d_-]*.so.1: Shared object ".+" not found, required by ".+"') + + +def run_cheribsd_command( + qemu: CheriBSDSpawnMixin, + cmd: str, + expected_output=None, + error_output=None, + cheri_trap_fatal=True, + ignore_cheri_trap=False, + timeout=60, +): qemu.sendline(cmd) # FIXME: allow ignoring CHERI traps if expected_output: qemu.expect([expected_output], timeout=timeout) - results = [SH_PROGRAM_NOT_FOUND, RTLD_DSO_NOT_FOUND, pexpect.TIMEOUT, - PEXPECT_PROMPT_RE, PEXPECT_CONTINUATION_PROMPT_RE] + results = [ + SH_PROGRAM_NOT_FOUND, + RTLD_DSO_NOT_FOUND, + pexpect.TIMEOUT, + PEXPECT_PROMPT_RE, + PEXPECT_CONTINUATION_PROMPT_RE, + ] error_output_index = -1 cheri_trap_indices = tuple() if error_output: @@ -597,11 +704,18 @@ def run_cheribsd_command(qemu: CheriBSDSpawnMixin, cmd: str, expected_output=Non failure("Got CHERI TRAP!", exit=False) -def checked_run_cheribsd_command(qemu: CheriBSDSpawnMixin, cmd: str, timeout=600, ignore_cheri_trap=False, - error_output: "Optional[str]" = None, **kwargs): +def checked_run_cheribsd_command( + qemu: CheriBSDSpawnMixin, + cmd: str, + timeout=600, + ignore_cheri_trap=False, + error_output: "Optional[str]" = None, + **kwargs, +): starttime = datetime.datetime.now() qemu.sendline( - cmd + " ;if test $? -eq 0; then echo '__COMMAND' 'SUCCESSFUL__'; else echo '__COMMAND' 'FAILED__'; fi") + cmd + " ;if test $? -eq 0; then echo '__COMMAND' 'SUCCESSFUL__'; else echo '__COMMAND' 'FAILED__'; fi" + ) cheri_trap_indices = tuple() error_output_index = None results = ["__COMMAND SUCCESSFUL__", "__COMMAND FAILED__", PEXPECT_CONTINUATION_PROMPT_RE, pexpect.TIMEOUT] @@ -622,25 +736,34 @@ def checked_run_cheribsd_command(qemu: CheriBSDSpawnMixin, cmd: str, timeout=600 elif i == 2: raise CheriBSDCommandFailed("Detected line continuation, cannot handle this yet! ", cmd, execution_time=runtime) elif i == 3: - raise CheriBSDCommandTimeout("timeout after ", runtime, " running '", cmd, "': ", str(qemu), - execution_time=runtime) + raise CheriBSDCommandTimeout( + "timeout after ", runtime, " running '", cmd, "': ", str(qemu), execution_time=runtime + ) elif i in cheri_trap_indices: # wait up to 20 seconds for a prompt to ensure the dump output has been printed qemu.expect_prompt(timeout=20, ignore_timeout=True) qemu.flush() - raise CheriBSDCommandFailed("Got CHERI trap running '", cmd, "' (after '", runtime.total_seconds(), "s)", - execution_time=runtime) + raise CheriBSDCommandFailed( + "Got CHERI trap running '", cmd, "' (after '", runtime.total_seconds(), "s)", execution_time=runtime + ) elif i == error_output_index: # wait up to 20 seconds for the shell prompt qemu.expect_prompt(timeout=20, ignore_timeout=True) qemu.flush() assert isinstance(error_output, str) - raise CheriBSDMatchedErrorOutput("Matched error output '" + error_output + "' running '", cmd, "' (after '", - runtime.total_seconds(), ")", execution_time=runtime) + raise CheriBSDMatchedErrorOutput( + "Matched error output '" + error_output + "' running '", + cmd, + "' (after '", + runtime.total_seconds(), + ")", + execution_time=runtime, + ) else: assert i < len(results), str(i) + " >= len(" + str(results) + ")" - raise CheriBSDCommandFailed("error running '", cmd, "' (after '", runtime.total_seconds(), "s)", - execution_time=runtime) + raise CheriBSDCommandFailed( + "error running '", cmd, "' (after '", runtime.total_seconds(), "s)", execution_time=runtime + ) def setup_ssh_for_root_login(qemu: QemuCheriBSDInstance): @@ -655,7 +778,7 @@ def setup_ssh_for_root_login(qemu: QemuCheriBSDInstance): ssh_pubkey_contents = pubkey.read_text(encoding="utf-8").strip() # Handle ssh-pubkeys that might be too long to send as a single line (write 150-char chunks instead): chunk_size = 150 - for part in (ssh_pubkey_contents[i:i + chunk_size] for i in range(0, len(ssh_pubkey_contents), chunk_size)): + for part in (ssh_pubkey_contents[i : i + chunk_size] for i in range(0, len(ssh_pubkey_contents), chunk_size)): qemu.run("printf %s " + shlex.quote(part) + " >> /root/.ssh/authorized_keys") # Add a final newline qemu.run("printf '\\n' >> /root/.ssh/authorized_keys") @@ -740,8 +863,10 @@ def interact(self, escape_character=chr(29), input_filter=None, output_filter=No def start_dhclient(qemu: CheriBSDSpawnMixin, network_iface: str): success("===> Setting up QEMU networking") qemu.sendline(f"ifconfig {network_iface} up && dhclient {network_iface}") - i = qemu.expect([pexpect.TIMEOUT, "DHCPACK from 10.0.2.2", "dhclient already running", - "interface ([\\w\\d]+) does not exist"], timeout=120) + i = qemu.expect( + [pexpect.TIMEOUT, "DHCPACK from 10.0.2.2", "dhclient already running", "interface ([\\w\\d]+) does not exist"], + timeout=120, + ) if i == 0: # Timeout failure("timeout awaiting dhclient ", str(qemu), exit=True) if i == 1: @@ -758,12 +883,24 @@ def start_dhclient(qemu: CheriBSDSpawnMixin, network_iface: str): qemu.expect_prompt(timeout=30) -def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kernel_image: Path, - disk_image: Optional[Path], ssh_port: Optional[int], - ssh_pubkey: Optional[Path], *, write_disk_image_changes: bool, expected_kernel_abi: str, - smp_args: "list[str]", smb_dirs: "Optional[list[SmbMount]]" = None, kernel_init_only=False, - trap_on_unrepresentable=False, skip_ssh_setup=False, bios_path: "Optional[Path]" = None, - boot_alternate_kernel_dir: "Optional[Path]" = None) -> QemuCheriBSDInstance: +def boot_cheribsd( + qemu_options: QemuOptions, + qemu_command: Optional[Path], + kernel_image: Path, + disk_image: Optional[Path], + ssh_port: Optional[int], + ssh_pubkey: Optional[Path], + *, + write_disk_image_changes: bool, + expected_kernel_abi: str, + smp_args: "list[str]", + smb_dirs: "Optional[list[SmbMount]]" = None, + kernel_init_only=False, + trap_on_unrepresentable=False, + skip_ssh_setup=False, + bios_path: "Optional[Path]" = None, + boot_alternate_kernel_dir: "Optional[Path]" = None, +) -> QemuCheriBSDInstance: user_network_args = "" if smb_dirs is None: smb_dirs = [] @@ -784,13 +921,17 @@ def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kerne bios_args = riscv_bios_arguments(qemu_options.xtarget, None) else: bios_args = [] - qemu_args = qemu_options.get_commandline(qemu_command=qemu_command, kernel_file=kernel_image, disk_image=disk_image, - bios_args=bios_args, user_network_args=user_network_args, - write_disk_image_changes=write_disk_image_changes, - add_network_device=True, - trap_on_unrepresentable=trap_on_unrepresentable, # For debugging - add_virtio_rng=True, # faster entropy gathering - ) + qemu_args = qemu_options.get_commandline( + qemu_command=qemu_command, + kernel_file=kernel_image, + disk_image=disk_image, + bios_args=bios_args, + user_network_args=user_network_args, + write_disk_image_changes=write_disk_image_changes, + add_network_device=True, + trap_on_unrepresentable=trap_on_unrepresentable, # For debugging + add_virtio_rng=True, # faster entropy gathering + ) qemu_args.extend(smp_args) kernel_commandline = [] if qemu_options.can_boot_kernel_directly and kernel_image and boot_alternate_kernel_dir: @@ -817,8 +958,16 @@ def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kerne qemu_cls = QemuCheriBSDInstance if get_global_config().pretend: qemu_cls = FakeQemuSpawn - child = qemu_cls(qemu_options, qemu_args[0], qemu_args[1:], ssh_port=ssh_port, ssh_pubkey=ssh_pubkey, - encoding="utf-8", echo=False, timeout=60) + child = qemu_cls( + qemu_options, + qemu_args[0], + qemu_args[1:], + ssh_port=ssh_port, + ssh_pubkey=ssh_pubkey, + encoding="utf-8", + echo=False, + timeout=60, + ) # child.logfile=sys.stdout.buffer child.smb_dirs = smb_dirs if QEMU_LOGFILE: @@ -842,9 +991,15 @@ def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kerne return child -def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=False, - network_iface: Optional[str], expected_kernel_abi_msg: Optional[str] = None, - loader_kernel_dir: "Optional[Path]" = None) -> None: +def boot_and_login( + child: CheriBSDSpawnMixin, + *, + starttime, + kernel_init_only=False, + network_iface: Optional[str], + expected_kernel_abi_msg: Optional[str] = None, + loader_kernel_dir: "Optional[Path]" = None, +) -> None: have_dhclient = False # ignore SIGINT for the python code, the child should still receive it # signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -882,8 +1037,9 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal if loader_kernel_dir: # Stop autoboot and enter console child.send("\x1b") - i = child.expect(loader_boot_prompt_messages, timeout=60, - timeout_msg="timeout before loader prompt") + i = child.expect( + loader_boot_prompt_messages, timeout=60, timeout_msg="timeout before loader prompt" + ) if i != loader_boot_prompt_messages.index(BOOT_LOADER_PROMPT): failure("failed to enter boot loader prompt after stopping autoboot", exit=True) # Fall through to BOOT_LOADER_PROMPT @@ -905,8 +1061,11 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal if i == boot_messages.index(expected_kernel_abi_msg): success(f"Booting correct kernel ABI: {expected_kernel_abi_msg}") else: - failure(f"Did not find expected kernel ABI message '{expected_kernel_abi_msg}'," - f" got '{child.match.group(0)}' instead.", exit=True) + failure( + f"Did not find expected kernel ABI message '{expected_kernel_abi_msg}'," + f" got '{child.match.group(0)}' instead.", + exit=True, + ) i = child.expect(boot_messages, timeout=10 * 60, timeout_msg="timeout mounting rootfs") if i == boot_messages.index(TRYING_TO_MOUNT_ROOT): @@ -919,28 +1078,40 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal success("===> init running (kernel startup time: ", userspace_starttime - starttime, ")") userspace_starttime = datetime.datetime.now() - boot_expect_strings: PatternListType = [LOGIN, LOGIN_AS_ROOT_MINIMAL, SHELL_OPEN, BOOT_FAILURE, - BOOT_FAILURE2, BOOT_FAILURE3] - i = child.expect([*boot_expect_strings, "DHCPACK from ", *FATAL_ERROR_MESSAGES], timeout=90 * 60, - timeout_msg="timeout awaiting login prompt") + boot_expect_strings: PatternListType = [ + LOGIN, + LOGIN_AS_ROOT_MINIMAL, + SHELL_OPEN, + BOOT_FAILURE, + BOOT_FAILURE2, + BOOT_FAILURE3, + ] + i = child.expect( + [*boot_expect_strings, "DHCPACK from ", *FATAL_ERROR_MESSAGES], + timeout=90 * 60, + timeout_msg="timeout awaiting login prompt", + ) if i == len(boot_expect_strings): # DHCPACK from have_dhclient = True success("===> got DHCPACK") # we have a network, keep waiting for the login prompt - i = child.expect(boot_expect_strings + FATAL_ERROR_MESSAGES, timeout=15 * 60, - timeout_msg="timeout awaiting login prompt") + i = child.expect( + boot_expect_strings + FATAL_ERROR_MESSAGES, timeout=15 * 60, timeout_msg="timeout awaiting login prompt" + ) if i == boot_expect_strings.index(LOGIN): success("===> got login prompt") child.sendline("root") - i = child.expect([INITIAL_PROMPT_CSH, INITIAL_PROMPT_SH], timeout=10 * 60, - timeout_msg="timeout awaiting command prompt ") # give CheriABI csh 3 minutes to start + i = child.expect( + [INITIAL_PROMPT_CSH, INITIAL_PROMPT_SH], timeout=10 * 60, timeout_msg="timeout awaiting command prompt " + ) # give CheriABI csh 3 minutes to start if i == 0: # /bin/csh prompt success("===> got csh command prompt, starting POSIX sh") # csh is weird, use the normal POSIX sh instead child.sendline("sh") - i = child.expect([INITIAL_PROMPT_CSH, INITIAL_PROMPT_SH], timeout=3 * 60, - timeout_msg="timeout starting /bin/sh") # give CheriABI sh 3 minutes to start + i = child.expect( + [INITIAL_PROMPT_CSH, INITIAL_PROMPT_SH], timeout=3 * 60, timeout_msg="timeout starting /bin/sh" + ) # give CheriABI sh 3 minutes to start if i == 0: # POSIX sh with PS1 set success("===> started POSIX sh (PS1 already set)") elif i == 1: # POSIX sh without PS1 @@ -956,8 +1127,7 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal child.expect([INITIAL_PROMPT_SH], timeout=3 * 60, timeout_msg="timeout logging in") # Note: the default shell in the minimal images is csh (but without the default prompt). child.sendline("sh") - child.expect([INITIAL_PROMPT_SH], timeout=3 * 60, - timeout_msg="timeout starting /bin/sh") + child.expect([INITIAL_PROMPT_SH], timeout=3 * 60, timeout_msg="timeout starting /bin/sh") success("===> /etc/rc completed, got command prompt") _set_pexpect_sh_prompt(child) else: # BOOT_FAILURE or FATAL_ERROR_MESSAGES @@ -978,9 +1148,13 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal return -def _do_test_setup(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_archives: "list[Path]", - test_ld_preload_files: "list[Path]", - test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None): +def _do_test_setup( + qemu: QemuCheriBSDInstance, + args: argparse.Namespace, + test_archives: "list[Path]", + test_ld_preload_files: "list[Path]", + test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, +): smb_dirs = qemu.smb_dirs setup_tests_starttime = datetime.datetime.now() # Print a backtrace and drop into the debugger on panic @@ -1005,7 +1179,8 @@ def _do_test_setup(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_ar # We can differentiate the two by checking if /boot/kernel/kernel exists since it will be missing in the minimal # image qemu.run( - "if [ ! -e /boot/kernel/kernel ]; then mkdir -p /usr/local && mount -t tmpfs -o size=300m tmpfs /usr/local; fi") + "if [ ! -e /boot/kernel/kernel ]; then mkdir -p /usr/local && mount -t tmpfs -o size=300m tmpfs /usr/local; fi" + ) # Or this: if [ "$(ls -A $DIR)" ]; then echo "Not Empty"; else echo "Empty"; fi qemu.run("if [ ! -e /opt ]; then mkdir -p /opt && mount -t tmpfs -o size=500m tmpfs /opt; fi") qemu.run("df -ih") @@ -1014,9 +1189,21 @@ def _do_test_setup(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_ar def do_scp(src, dst="/"): # CVE-2018-20685 -> Can no longer use '.' See # https://superuser.com/questions/1403473/scp-error-unexpected-filename - scp_cmd = ["scp", "-B", "-r", "-P", str(qemu.ssh_port), "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-i", str(qemu.ssh_private_key), str(src), "root@localhost:" + dst] + scp_cmd = [ + "scp", + "-B", + "-r", + "-P", + str(qemu.ssh_port), + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-i", + str(qemu.ssh_private_key), + str(src), + "root@localhost:" + dst, + ] # use script for a fake tty to get progress output from scp if sys.platform.startswith("linux"): scp_cmd = ["script", "--quiet", "--return", "--command", " ".join(scp_cmd), "/dev/null"] @@ -1048,16 +1235,24 @@ def do_scp(src, dst="/"): mount_command = f"mount_smbfs -I 10.0.2.4 -N //10.0.2.4/qemu{index + 1} '{d.in_target}'" for trial in range(MAX_SMBFS_RETRY if not get_global_config().pretend else 1): # maximum of 3 trials try: - checked_run_cheribsd_command(qemu, mount_command, - error_output="unable to open connection: syserr = ", - pretend_result=0) + checked_run_cheribsd_command( + qemu, mount_command, error_output="unable to open connection: syserr = ", pretend_result=0 + ) qemu.smb_failed = False break except CheriBSDMatchedErrorOutput as e: # If the smbfs connection timed out try once more. This can happen when multiple libc++ test jobs are # running on the same jenkins slaves so one of them might time out - failure("QEMU SMBD failed to mount ", d.in_target, " after ", e.execution_time.total_seconds(), - " seconds. Trying ", (MAX_SMBFS_RETRY - trial - 1), " more time(s)", exit=False) + failure( + "QEMU SMBD failed to mount ", + d.in_target, + " after ", + e.execution_time.total_seconds(), + " seconds. Trying ", + (MAX_SMBFS_RETRY - trial - 1), + " more time(s)", + exit=False, + ) qemu.smb_failed = True info("Waiting for 2-10 seconds before retrying mount_smbfs...") if not get_global_config().pretend: @@ -1074,11 +1269,13 @@ def do_scp(src, dst="/"): # Ensure that the libraries exist checked_run_cheribsd_command(qemu, f"test -x '{lib}'") if ld_preload_target_paths: - checked_run_cheribsd_command(qemu, "export '{}={}'".format(args.test_ld_preload_variable, - ":".join(ld_preload_target_paths))) + checked_run_cheribsd_command( + qemu, "export '{}={}'".format(args.test_ld_preload_variable, ":".join(ld_preload_target_paths)) + ) if args.test_ld_preload_variable == "LD_64C_PRELOAD": - checked_run_cheribsd_command(qemu, "export '{}={}'".format("LD_CHERI_PRELOAD", - ":".join(ld_preload_target_paths))) + checked_run_cheribsd_command( + qemu, "export '{}={}'".format("LD_CHERI_PRELOAD", ":".join(ld_preload_target_paths)) + ) if args.extra_library_paths: prepend_ld_library_path(qemu, ":".join(args.extra_library_paths)) @@ -1089,10 +1286,14 @@ def do_scp(src, dst="/"): success("Additional test enviroment setup took ", datetime.datetime.now() - setup_tests_starttime) -def runtests(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_archives: "list[Path]", - test_ld_preload_files: "list[Path]", - test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, - test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None) -> bool: +def runtests( + qemu: QemuCheriBSDInstance, + args: argparse.Namespace, + test_archives: "list[Path]", + test_ld_preload_files: "list[Path]", + test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, + test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, +) -> bool: try: _do_test_setup(qemu, args, test_archives, test_ld_preload_files, test_setup_function) except KeyboardInterrupt: @@ -1127,13 +1328,13 @@ def runtests(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_archives test_command = args.test_command timeout = args.test_timeout - qemu.sendline(test_command + - " ;if test $? -eq 0; then echo 'TESTS' 'COMPLETED'; else echo 'TESTS' 'FAILED'; fi") + qemu.sendline(test_command + " ;if test $? -eq 0; then echo 'TESTS' 'COMPLETED'; else echo 'TESTS' 'FAILED'; fi") i = qemu.expect([pexpect.TIMEOUT, "TESTS COMPLETED", "TESTS UNSTABLE", "TESTS FAILED"], timeout=timeout) testtime = datetime.datetime.now() - run_tests_starttime if i == 0: # Timeout - return failure("timeout after ", testtime, "waiting for tests (command='", test_command, "'): ", str(qemu), - exit=False) + return failure( + "timeout after ", testtime, "waiting for tests (command='", test_command, "'): ", str(qemu), exit=False + ) elif i == 1 or i == 2: if i == 2: success("===> Tests completed (but with FAILURES)!") @@ -1156,64 +1357,113 @@ def default_ssh_key(): def get_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(allow_abbrev=False) - parser.add_argument("--architecture", help="CPU architecture to be used for this test", required=True, - choices=[x for x in SUPPORTED_ARCHITECTURES.keys()]) + parser.add_argument( + "--architecture", + help="CPU architecture to be used for this test", + required=True, + choices=[x for x in SUPPORTED_ARCHITECTURES.keys()], + ) parser.add_argument("--qemu-cmd", "--qemu", help="Path to QEMU (default: find matching on in $PATH)", default=None) parser.add_argument("--qemu-smp", "--smp", type=int, help="Run QEMU with SMP", default=None) parser.add_argument("--kernel", default=None) parser.add_argument("--bios", default=None) parser.add_argument("--disk-image", default=None) - parser.add_argument("--minimal-image", action="store_true", - help="Set this if tests are being run on the minimal disk image rather than the full one") + parser.add_argument( + "--minimal-image", + action="store_true", + help="Set this if tests are being run on the minimal disk image rather than the full one", + ) parser.add_argument("--extract-images-to", help="Path where the compressed images should be extracted to") parser.add_argument("--reuse-image", action="store_true") parser.add_argument("--keep-compressed-images", action="store_true", default=True, dest="keep_compressed_images") parser.add_argument("--no-keep-compressed-images", action="store_false", dest="keep_compressed_images") - parser.add_argument("--write-disk-image-changes", default=False, action="store_true", - help="Commit changes made to the disk image (by default the image is immutable)") + parser.add_argument( + "--write-disk-image-changes", + default=False, + action="store_true", + help="Commit changes made to the disk image (by default the image is immutable)", + ) parser.add_argument("--no-write-disk-image-changes", action="store_false", dest="write_disk_image_changes") - parser.add_argument("--trap-on-unrepresentable", action="store_true", - help="CHERI trap on unrepresentable caps instead of detagging") + parser.add_argument( + "--trap-on-unrepresentable", action="store_true", help="CHERI trap on unrepresentable caps instead of detagging" + ) parser.add_argument("--ssh-key", "--test-ssh-key", default=default_ssh_key()) parser.add_argument("--ssh-port", type=int, default=None) parser.add_argument("--use-smb-instead-of-ssh", action="store_true") - parser.add_argument("--smb-mount-directory", metavar="HOST_PATH:IN_TARGET", - help="Share a host directory with the QEMU guest via smb. This option can be passed multiple " - "times " - "to share more than one directory. The argument should be colon-separated as follows: " - "':'. Appending '@ro' to HOST_PATH will cause the " - "directory " - "to be mapped as a read-only smb share", action="append", - dest="smb_mount_directories", type=parse_smb_mount, default=[]) + parser.add_argument( + "--smb-mount-directory", + metavar="HOST_PATH:IN_TARGET", + help="Share a host directory with the QEMU guest via smb. This option can be passed multiple " + "times " + "to share more than one directory. The argument should be colon-separated as follows: " + "':'. Appending '@ro' to HOST_PATH will cause the " + "directory " + "to be mapped as a read-only smb share", + action="append", + dest="smb_mount_directories", + type=parse_smb_mount, + default=[], + ) parser.add_argument("--test-archive", "-t", action="append", nargs=1) parser.add_argument("--test-command", "-c") - parser.add_argument('--test-ld-preload', action="append", nargs=1, metavar='LIB', - help="Copy LIB to the guest and LD_PRELOAD it before running tests") - parser.add_argument('--extra-library-path', action="append", dest="extra_library_paths", metavar="DIR", - help="Add DIR as an additional LD_LIBRARY_PATH before running tests") - parser.add_argument('--test-ld-preload-variable', type=str, default=None, - help="The environment variable to set to LD_PRELOAD a library. should be set to either " - "LD_PRELOAD or LD_64C_PRELOAD") - parser.add_argument("--test-timeout", "-tt", type=int, default=60 * 60, - help="Timeout in seconds for running tests") + parser.add_argument( + "--test-ld-preload", + action="append", + nargs=1, + metavar="LIB", + help="Copy LIB to the guest and LD_PRELOAD it before running tests", + ) + parser.add_argument( + "--extra-library-path", + action="append", + dest="extra_library_paths", + metavar="DIR", + help="Add DIR as an additional LD_LIBRARY_PATH before running tests", + ) + parser.add_argument( + "--test-ld-preload-variable", + type=str, + default=None, + help="The environment variable to set to LD_PRELOAD a library. should be set to either " + "LD_PRELOAD or LD_64C_PRELOAD", + ) + parser.add_argument("--test-timeout", "-tt", type=int, default=60 * 60, help="Timeout in seconds for running tests") parser.add_argument("--qemu-logfile", help="File to write all interactions with QEMU to", type=Path) - parser.add_argument("--test-environment-only", action="store_true", - help="Setup mount paths + SSH for tests but don't actually run the tests (implies --interact)") - parser.add_argument("--skip-ssh-setup", action="store_true", - help="Don't start sshd on boot. Saves a few seconds of boot time if not needed.") - parser.add_argument("--pretend", "-p", action="store_true", - help="Don't actually boot CheriBSD just print what would happen") + parser.add_argument( + "--test-environment-only", + action="store_true", + help="Setup mount paths + SSH for tests but don't actually run the tests (implies --interact)", + ) + parser.add_argument( + "--skip-ssh-setup", + action="store_true", + help="Don't start sshd on boot. Saves a few seconds of boot time if not needed.", + ) + parser.add_argument( + "--pretend", "-p", action="store_true", help="Don't actually boot CheriBSD just print what would happen" + ) parser.add_argument("--interact", "-i", action="store_true") - parser.add_argument("--interact-on-kernel-panic", action="store_true", - help="Instead of exiting on kernel panic start interacting with QEMU") + parser.add_argument( + "--interact-on-kernel-panic", + action="store_true", + help="Instead of exiting on kernel panic start interacting with QEMU", + ) parser.add_argument("--test-kernel-init-only", action="store_true") parser.add_argument("--enable-coredumps", action="store_true", dest="enable_coredumps", default=False) parser.add_argument("--disable-coredumps", action="store_false", dest="enable_coredumps") - parser.add_argument("--alternate-kernel-rootfs-path", type=Path, default=None, - help="Path relative to the disk image pointing to the directory " + - "containing the alternate kernel to run and related kernel modules") - parser.add_argument("--expected-kernel-abi", choices=["any", "hybrid", "purecap"], default="any", - help="The kernel kind that is expected ('any' to skip checks)") + parser.add_argument( + "--alternate-kernel-rootfs-path", + type=Path, + default=None, + help="Path relative to the disk image pointing to the directory " + + "containing the alternate kernel to run and related kernel modules", + ) + parser.add_argument( + "--expected-kernel-abi", + choices=["any", "hybrid", "purecap"], + default="any", + help="The kernel kind that is expected ('any' to skip checks)", + ) # Ensure that we don't get a race when running multiple shards: # If we extract the disk image at the same time we might spawn QEMU just between when the # value extracted by one job is unlinked and when it is replaced with a new file @@ -1222,16 +1472,19 @@ def get_argument_parser() -> argparse.ArgumentParser: return parser -def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, - test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, - argparse_setup_callback: "Optional[Callable[[argparse.ArgumentParser], None]]" = None, - argparse_adjust_args_callback: "Optional[Callable[[argparse.Namespace], None]]" = None): +def _main( + test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, + test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, + argparse_setup_callback: "Optional[Callable[[argparse.ArgumentParser], None]]" = None, + argparse_adjust_args_callback: "Optional[Callable[[argparse.Namespace], None]]" = None, +): parser = get_argument_parser() if argparse_setup_callback: argparse_setup_callback(parser) try: # noinspection PyUnresolvedReferences import argcomplete + argcomplete.autocomplete(parser) except ImportError: pass @@ -1287,8 +1540,10 @@ def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespac test_ld_preload_files: "list[Path]" = [] if not args.use_smb_instead_of_ssh and not args.skip_ssh_setup: if args.ssh_key is None: - failure("No SSH key specified, but test script needs SSH. Please pass --test-ssh-key=/path/to/id_foo.pub", - exit=True) + failure( + "No SSH key specified, but test script needs SSH. Please pass --test-ssh-key=/path/to/id_foo.pub", + exit=True, + ) elif not Path(args.ssh_key).exists(): failure("Specified SSH key do not exist: ", args.ssh_key, exit=True) if Path(args.ssh_key).suffix != ".pub": @@ -1340,22 +1595,37 @@ def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespac keep_compressed_images = False kernel = None if args.kernel is not None: - kernel = maybe_decompress(Path(args.kernel), force_decompression, keep_archive=keep_compressed_images, - args=args, what="kernel") + kernel = maybe_decompress( + Path(args.kernel), force_decompression, keep_archive=keep_compressed_images, args=args, what="kernel" + ) diskimg = None if args.disk_image: - diskimg = maybe_decompress(Path(args.disk_image), force_decompression, keep_archive=keep_compressed_images, - args=args, what="disk image") + diskimg = maybe_decompress( + Path(args.disk_image), + force_decompression, + keep_archive=keep_compressed_images, + args=args, + what="disk image", + ) boot_starttime = datetime.datetime.now() - qemu = boot_cheribsd(qemu_options, qemu_command=args.qemu_cmd, kernel_image=kernel, disk_image=diskimg, - ssh_port=args.ssh_port, ssh_pubkey=Path(args.ssh_key) if args.ssh_key is not None else None, - smb_dirs=args.smb_mount_directories, kernel_init_only=args.test_kernel_init_only, - smp_args=["-smp", str(args.qemu_smp)] if args.qemu_smp else [], - trap_on_unrepresentable=args.trap_on_unrepresentable, skip_ssh_setup=args.skip_ssh_setup, - bios_path=args.bios, write_disk_image_changes=args.write_disk_image_changes, - boot_alternate_kernel_dir=args.alternate_kernel_rootfs_path, - expected_kernel_abi=args.expected_kernel_abi) + qemu = boot_cheribsd( + qemu_options, + qemu_command=args.qemu_cmd, + kernel_image=kernel, + disk_image=diskimg, + ssh_port=args.ssh_port, + ssh_pubkey=Path(args.ssh_key) if args.ssh_key is not None else None, + smb_dirs=args.smb_mount_directories, + kernel_init_only=args.test_kernel_init_only, + smp_args=["-smp", str(args.qemu_smp)] if args.qemu_smp else [], + trap_on_unrepresentable=args.trap_on_unrepresentable, + skip_ssh_setup=args.skip_ssh_setup, + bios_path=args.bios, + write_disk_image_changes=args.write_disk_image_changes, + boot_alternate_kernel_dir=args.alternate_kernel_rootfs_path, + expected_kernel_abi=args.expected_kernel_abi, + ) success("Booting CheriBSD took: ", datetime.datetime.now() - boot_starttime) tests_okay = True @@ -1366,8 +1636,14 @@ def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespac setup_ssh_starttime = datetime.datetime.now() setup_ssh_for_root_login(qemu) info("Setting up SSH took: ", datetime.datetime.now() - setup_ssh_starttime) - tests_okay = runtests(qemu, args, test_archives=test_archives, test_function=test_function, - test_setup_function=test_setup_function, test_ld_preload_files=test_ld_preload_files) + tests_okay = runtests( + qemu, + args, + test_archives=test_archives, + test_function=test_function, + test_setup_function=test_setup_function, + test_ld_preload_files=test_ld_preload_files, + ) except CheriBSDCommandFailed as e: failure("Command failed while runnings tests: ", str(e), "\n", str(qemu), exit=False) traceback.print_exc(file=sys.stderr) @@ -1398,16 +1674,22 @@ def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespac sys.exit(2) # different exit code for test failures -def main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, - test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, - argparse_setup_callback: "Optional[Callable[[argparse.ArgumentParser], None]]" = None, - argparse_adjust_args_callback: "Optional[Callable[[argparse.Namespace], None]]" = None): +def main( + test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, + test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, + argparse_setup_callback: "Optional[Callable[[argparse.ArgumentParser], None]]" = None, + argparse_adjust_args_callback: "Optional[Callable[[argparse.Namespace], None]]" = None, +): # Some programs (such as QEMU) can mess up the TTY state if they don't exit cleanly with keep_terminal_sane(): run_and_kill_children_on_exit( - lambda: _main(test_function=test_function, test_setup_function=test_setup_function, - argparse_setup_callback=argparse_setup_callback, - argparse_adjust_args_callback=argparse_adjust_args_callback)) + lambda: _main( + test_function=test_function, + test_setup_function=test_setup_function, + argparse_setup_callback=argparse_setup_callback, + argparse_adjust_args_callback=argparse_adjust_args_callback, + ) + ) if __name__ == "__main__": diff --git a/pycheribuild/config/chericonfig.py b/pycheribuild/config/chericonfig.py index b99a9b725..f46bef9c7 100644 --- a/pycheribuild/config/chericonfig.py +++ b/pycheribuild/config/chericonfig.py @@ -152,94 +152,158 @@ class CheribuildActionEnum(Enum): class CheriConfig(ConfigBase): def __init__(self, loader, action_class: "type[CheribuildActionEnum]") -> None: - super().__init__(pretend=DoNotUseInIfStmt(), verbose=DoNotUseInIfStmt(), quiet=DoNotUseInIfStmt(), - force=DoNotUseInIfStmt()) + super().__init__( + pretend=DoNotUseInIfStmt(), verbose=DoNotUseInIfStmt(), quiet=DoNotUseInIfStmt(), force=DoNotUseInIfStmt() + ) self._cached_deps = collections.defaultdict(dict) assert isinstance(loader, ConfigLoaderBase) loader._cheri_config = self self.loader = loader - self.pretend = loader.add_commandline_only_bool_option("pretend", "p", - help="Only print the commands instead of running them") + self.pretend = loader.add_commandline_only_bool_option( + "pretend", "p", help="Only print the commands instead of running them" + ) # add the actions: self._action_class = action_class - self.action = loader.add_option("action", default=[], action="append", type=action_class, help_hidden=True, - help="The action to perform by cheribuild", group=loader.action_group) + self.action = loader.add_option( + "action", + default=[], + action="append", + type=action_class, + help_hidden=True, + help="The action to perform by cheribuild", + group=loader.action_group, + ) self.default_action: Optional[CheribuildActionEnum] = None # Add aliases (e.g. --test = --action=test): for action in action_class: if action.altname: - loader.action_group.add_argument(action.option_name, action.altname, help=action.help_message, - dest="action", action="append_const", const=action.actions) + loader.action_group.add_argument( + action.option_name, + action.altname, + help=action.help_message, + dest="action", + action="append_const", + const=action.actions, + ) else: - loader.action_group.add_argument(action.option_name, help=action.help_message, dest="action", - action="append_const", const=action.actions) + loader.action_group.add_argument( + action.option_name, + help=action.help_message, + dest="action", + action="append_const", + const=action.actions, + ) self.print_targets_only = loader.add_commandline_only_bool_option( - "print-targets-only", help_hidden=False, group=loader.action_group, - help="Don't run the build but instead only print the targets that would be executed") - - self.clang_path = loader.add_path_option("clang-path", shortname="-cc-path", - default=lambda c, _: latest_system_clang_tool(c, "clang", "cc"), - group=loader.path_group, - help="The C compiler to use for host binaries (must be compatible " - "with Clang >= 3.7)") - self.clang_plusplus_path = loader.add_path_option("clang++-path", shortname="-c++-path", - default=lambda c, _: latest_system_clang_tool(c, "clang++", - "c++"), - group=loader.path_group, - help="The C++ compiler to use for host binaries (must be " - "compatible with Clang >= 3.7)") - self.clang_cpp_path = loader.add_path_option("clang-cpp-path", shortname="-cpp-path", - default=lambda c, _: latest_system_clang_tool(c, "clang-cpp", - "cpp"), - group=loader.path_group, - help="The C preprocessor to use for host binaries (must be " - "compatible with Clang >= 3.7)") + "print-targets-only", + help_hidden=False, + group=loader.action_group, + help="Don't run the build but instead only print the targets that would be executed", + ) + + self.clang_path = loader.add_path_option( + "clang-path", + shortname="-cc-path", + default=lambda c, _: latest_system_clang_tool(c, "clang", "cc"), + group=loader.path_group, + help="The C compiler to use for host binaries (must be compatible " "with Clang >= 3.7)", + ) + self.clang_plusplus_path = loader.add_path_option( + "clang++-path", + shortname="-c++-path", + default=lambda c, _: latest_system_clang_tool(c, "clang++", "c++"), + group=loader.path_group, + help="The C++ compiler to use for host binaries (must be " "compatible with Clang >= 3.7)", + ) + self.clang_cpp_path = loader.add_path_option( + "clang-cpp-path", + shortname="-cpp-path", + default=lambda c, _: latest_system_clang_tool(c, "clang-cpp", "cpp"), + group=loader.path_group, + help="The C preprocessor to use for host binaries (must be " "compatible with Clang >= 3.7)", + ) self.pass_dash_k_to_make = loader.add_commandline_only_bool_option( - "pass-k-to-make", "k", help="Pass the -k flag to make to continue after the first error") - self.with_libstatcounters = loader.add_bool_option("with-libstatcounters", - group=loader.cross_compile_options_group, - help="Link cross compiled CHERI project with " - "libstatcounters.") + "pass-k-to-make", "k", help="Pass the -k flag to make to continue after the first error" + ) + self.with_libstatcounters = loader.add_bool_option( + "with-libstatcounters", + group=loader.cross_compile_options_group, + help="Link cross compiled CHERI project with " "libstatcounters.", + ) self.skip_world = loader.add_bool_option( - "skip-world", "-skip-buildworld", group=loader.freebsd_group, - help="Skip the buildworld-related steps when building FreeBSD or CheriBSD") + "skip-world", + "-skip-buildworld", + group=loader.freebsd_group, + help="Skip the buildworld-related steps when building FreeBSD or CheriBSD", + ) self.skip_kernel = loader.add_bool_option( - "skip-kernel", "-skip-buildkernel", group=loader.freebsd_group, - help="Skip the buildkernel step when building FreeBSD or CheriBSD") + "skip-kernel", + "-skip-buildkernel", + group=loader.freebsd_group, + help="Skip the buildkernel step when building FreeBSD or CheriBSD", + ) self.freebsd_kernconf = loader.add_commandline_only_option( - "kernel-config", "-kernconf", group=loader.freebsd_group, help_hidden=True, - help="Override the default FreeBSD/CheriBSD kernel config.") + "kernel-config", + "-kernconf", + group=loader.freebsd_group, + help_hidden=True, + help="Override the default FreeBSD/CheriBSD kernel config.", + ) self.freebsd_subdir = loader.add_commandline_only_option( - "freebsd-subdir", "-subdir", group=loader.freebsd_group, type=list, metavar="SUBDIRS", + "freebsd-subdir", + "-subdir", + group=loader.freebsd_group, + type=list, + metavar="SUBDIRS", help="Only build subdirs SUBDIRS of FreeBSD/CheriBSD instead of the full tree. Useful for quickly " - "rebuilding individual programs/libraries. If more than one dir is passed they will be processed in " - "order. Note: This will break if not all dependencies have been built.") + "rebuilding individual programs/libraries. If more than one dir is passed they will be processed in " + "order. Note: This will break if not all dependencies have been built.", + ) self.freebsd_host_tools_only = loader.add_commandline_only_bool_option( - "freebsd-host-tools-only", help_hidden=True, group=loader.freebsd_group, - help="Stop the FreeBSD/CheriBSD build after the host tools have been built") + "freebsd-host-tools-only", + help_hidden=True, + group=loader.freebsd_group, + help="Stop the FreeBSD/CheriBSD build after the host tools have been built", + ) self.buildenv = loader.add_commandline_only_bool_option( - "buildenv", group=loader.freebsd_group, + "buildenv", + group=loader.freebsd_group, help="Open a shell with the right environment for building the project. Currently only works for " - "FreeBSD/CheriBSD") + "FreeBSD/CheriBSD", + ) self.libcompat_buildenv = loader.add_commandline_only_bool_option( - "libcompat-buildenv", "-libcheri-buildenv", group=loader.freebsd_group, - help="Open a shell with the right environment for building compat libraries.") - - self.cheri_cap_table_abi = loader.add_option("cap-table-abi", help_hidden=True, - choices=("pcrel", "plt", "fn-desc"), - help="The ABI to use for cap-table mode") - self.cross_target_suffix = loader.add_option("cross-target-suffix", help_hidden=True, default="", - help="Add a suffix to the cross build and install directories.") - self.allow_running_as_root = loader.add_bool_option("allow-running-as-root", help_hidden=True, default=False, - help="Allow running cheribuild as root (not recommended!)") + "libcompat-buildenv", + "-libcheri-buildenv", + group=loader.freebsd_group, + help="Open a shell with the right environment for building compat libraries.", + ) + + self.cheri_cap_table_abi = loader.add_option( + "cap-table-abi", + help_hidden=True, + choices=("pcrel", "plt", "fn-desc"), + help="The ABI to use for cap-table mode", + ) + self.cross_target_suffix = loader.add_option( + "cross-target-suffix", + help_hidden=True, + default="", + help="Add a suffix to the cross build and install directories.", + ) + self.allow_running_as_root = loader.add_bool_option( + "allow-running-as-root", + help_hidden=True, + default=False, + help="Allow running cheribuild as root (not recommended!)", + ) # Attributes for code completion: self.verbose: Optional[bool] = None - self.debug_output = loader.add_commandline_only_bool_option("debug-output", "vv", - help="Extremely verbose output") + self.debug_output = loader.add_commandline_only_bool_option( + "debug-output", "vv", help="Extremely verbose output" + ) self.quiet: "Optional[bool] " = None self.clean: "Optional[bool] " = None self.force: "Optional[bool] " = None @@ -250,79 +314,121 @@ def __init__(self, loader, action_class: "type[CheribuildActionEnum]") -> None: self.skip_configure: "Optional[bool] " = None self.force_configure: "Optional[bool] " = None self.force_update: "Optional[bool] " = None - self.mips_float_abi = loader.add_option("mips-float-abi", default=MipsFloatAbi.SOFT, type=MipsFloatAbi, - group=loader.cross_compile_options_group, - help="The floating point ABI to use for building MIPS+CHERI programs") + self.mips_float_abi = loader.add_option( + "mips-float-abi", + default=MipsFloatAbi.SOFT, + type=MipsFloatAbi, + group=loader.cross_compile_options_group, + help="The floating point ABI to use for building MIPS+CHERI programs", + ) self.aarch64_fp_and_simd_options = loader.add_option( - "aarch64-fp-and-simd-options", default=AArch64FloatSimdOptions.DEFAULT, type=AArch64FloatSimdOptions, + "aarch64-fp-and-simd-options", + default=AArch64FloatSimdOptions.DEFAULT, + type=AArch64FloatSimdOptions, group=loader.cross_compile_options_group, - help="The floating point/SIMD mode to use for building AArch64 programs") - self.crosscompile_linkage = loader.add_option("cross-compile-linkage", default=Linkage.DEFAULT, type=Linkage, - group=loader.cross_compile_options_group, - enum_choices=(Linkage.DEFAULT, Linkage.DYNAMIC, Linkage.STATIC), - help="Whether to link cross-compile projects static or dynamic " - "by default") - self.csetbounds_stats = loader.add_bool_option("collect-csetbounds-stats", - group=loader.cross_compile_options_group, help_hidden=True, - help="Whether to log CSetBounds statistics in csv format") - self.subobject_bounds = loader.add_option("subobject-bounds", type=str, - group=loader.cross_compile_options_group, - choices=( - "conservative", "subobject-safe", "aggressive", "very-aggressive", - "everywhere-unsafe"), - help="Whether to add additional CSetBounds to subobject " - "references/&-operator") + help="The floating point/SIMD mode to use for building AArch64 programs", + ) + self.crosscompile_linkage = loader.add_option( + "cross-compile-linkage", + default=Linkage.DEFAULT, + type=Linkage, + group=loader.cross_compile_options_group, + enum_choices=(Linkage.DEFAULT, Linkage.DYNAMIC, Linkage.STATIC), + help="Whether to link cross-compile projects static or dynamic " "by default", + ) + self.csetbounds_stats = loader.add_bool_option( + "collect-csetbounds-stats", + group=loader.cross_compile_options_group, + help_hidden=True, + help="Whether to log CSetBounds statistics in csv format", + ) + self.subobject_bounds = loader.add_option( + "subobject-bounds", + type=str, + group=loader.cross_compile_options_group, + choices=("conservative", "subobject-safe", "aggressive", "very-aggressive", "everywhere-unsafe"), + help="Whether to add additional CSetBounds to subobject " "references/&-operator", + ) self.use_cheri_ubsan = loader.add_bool_option( - "use-cheri-ubsan", group=loader.cross_compile_options_group, - help="Add compiler flags to detect certain undefined CHERI behaviour at runtime") + "use-cheri-ubsan", + group=loader.cross_compile_options_group, + help="Add compiler flags to detect certain undefined CHERI behaviour at runtime", + ) self.use_cheri_ubsan_runtime = loader.add_bool_option( - "use-cheri-ubsan-runtime", group=loader.cross_compile_options_group, default=False, + "use-cheri-ubsan-runtime", + group=loader.cross_compile_options_group, + default=False, help="Use the UBSan runtime to provide more detailed information on undefined CHERI behaviour." - "If false (the default) the compiler will generate a trap instruction instead.") - self.subobject_debug = loader.add_bool_option("subobject-debug", group=loader.cross_compile_options_group, - default=True, help_hidden=False, - help="Clear software permission bit 2 when subobject bounds " - "reduced size" - " (Note: this should be turned off for benchmarks!)") - - self.clang_colour_diags = loader.add_bool_option("clang-colour-diags", "-clang-color-diags", default=True, - help="Force CHERI clang to emit coloured diagnostics") - self.use_sdk_clang_for_native_xbuild = loader.add_bool_option("use-sdk-clang-for-native-xbuild", - group=loader.cross_compile_options_group, - help="Compile cross-compile project with CHERI " - "clang from the SDK instead of host " - "compiler") - - self.configure_only = loader.add_bool_option("configure-only", - help="Only run the configure step (skip build and install)") + "If false (the default) the compiler will generate a trap instruction instead.", + ) + self.subobject_debug = loader.add_bool_option( + "subobject-debug", + group=loader.cross_compile_options_group, + default=True, + help_hidden=False, + help="Clear software permission bit 2 when subobject bounds " + "reduced size" + " (Note: this should be turned off for benchmarks!)", + ) + + self.clang_colour_diags = loader.add_bool_option( + "clang-colour-diags", + "-clang-color-diags", + default=True, + help="Force CHERI clang to emit coloured diagnostics", + ) + self.use_sdk_clang_for_native_xbuild = loader.add_bool_option( + "use-sdk-clang-for-native-xbuild", + group=loader.cross_compile_options_group, + help="Compile cross-compile project with CHERI " "clang from the SDK instead of host " "compiler", + ) + + self.configure_only = loader.add_bool_option( + "configure-only", help="Only run the configure step (skip build and install)" + ) self.skip_install = loader.add_bool_option("skip-install", help="Skip the install step (only do the build)") self.skip_build = loader.add_bool_option("skip-build", help="Skip the build step (only do the install)") self.skip_sdk = loader.add_bool_option( - "skip-sdk", group=loader.dependencies_group, + "skip-sdk", + group=loader.dependencies_group, help="When building with --include-dependencies ignore the SDK dependencies. Saves a lot of time " - "when building libc++, etc. with dependencies but the sdk is already up-to-date. " - "This is like --no-include-toolchain-depedencies but also skips the target that builds the sysroot.") + "when building libc++, etc. with dependencies but the sdk is already up-to-date. " + "This is like --no-include-toolchain-depedencies but also skips the target that builds the sysroot.", + ) self.skip_dependency_filters: "list[re.Pattern]" = loader.add_option( - "skip-dependency-filter", group=loader.dependencies_group, action="append", default=[], - type=_skip_dependency_filter_arg, metavar="REGEX", + "skip-dependency-filter", + group=loader.dependencies_group, + action="append", + default=[], + type=_skip_dependency_filter_arg, + metavar="REGEX", help="A regular expression to match against to target names that should be skipped when using" - "--include-dependency. Can be passed multiple times to add more patterns.") + "--include-dependency. Can be passed multiple times to add more patterns.", + ) self.trap_on_unrepresentable = loader.add_bool_option( - "trap-on-unrepresentable", default=False, group=loader.run_group, + "trap-on-unrepresentable", + default=False, + group=loader.run_group, help="Raise a CHERI exception when capabilities become unreprestable instead of detagging. Useful for " - "debugging, but deviates from the spec, and therefore off by default.") + "debugging, but deviates from the spec, and therefore off by default.", + ) self.debugger_on_cheri_trap = loader.add_bool_option( - "qemu-gdb-break-on-cheri-trap", default=False, group=loader.run_group, - help="Drop into GDB attached to QEMU when a CHERI exception is triggered (QEMU only).") + "qemu-gdb-break-on-cheri-trap", + default=False, + group=loader.run_group, + help="Drop into GDB attached to QEMU when a CHERI exception is triggered (QEMU only).", + ) self.qemu_debug_program = loader.add_option( - "qemu-gdb-debug-userspace-program", group=loader.run_group, - help="Print the command to debug the following userspace program in GDB attaced to QEMU") + "qemu-gdb-debug-userspace-program", + group=loader.run_group, + help="Print the command to debug the following userspace program in GDB attaced to QEMU", + ) self.include_dependencies: Optional[bool] = None self.include_toolchain_dependencies = True self.enable_hybrid_targets = False - self.only_dependencies = loader.add_bool_option("only-dependencies", - help="Only build dependencies of targets, " - "not the targets themselves") + self.only_dependencies = loader.add_bool_option( + "only-dependencies", help="Only build dependencies of targets, " "not the targets themselves" + ) self.start_with: Optional[str] = None self.start_after: Optional[str] = None self.make_without_nice: Optional[bool] = None @@ -339,135 +445,201 @@ def __init__(self, loader, action_class: "type[CheribuildActionEnum]") -> None: self.morello_sdk_dir: Optional[Path] = None self.other_tools_dir: Optional[Path] = None self.sysroot_output_root: Optional[Path] = None - self.docker = loader.add_bool_option("docker", help="Run the build inside a docker container", - group=loader.docker_group) - self.docker_container = loader.add_option("docker-container", help="Name of the docker container to use", - default="ctsrd/cheribuild-docker", group=loader.docker_group) - self.docker_reuse_container = loader.add_bool_option("docker-reuse-container", group=loader.docker_group, - help="Attach to the same container again (note: " - "docker-container option must be an id rather than " - "a container name") + self.docker = loader.add_bool_option( + "docker", help="Run the build inside a docker container", group=loader.docker_group + ) + self.docker_container = loader.add_option( + "docker-container", + help="Name of the docker container to use", + default="ctsrd/cheribuild-docker", + group=loader.docker_group, + ) + self.docker_reuse_container = loader.add_bool_option( + "docker-reuse-container", + group=loader.docker_group, + help="Attach to the same container again (note: " + "docker-container option must be an id rather than " + "a container name", + ) # compilation db options: self.create_compilation_db = loader.add_bool_option( - "compilation-db", "-cdb", help="Create a compile_commands.json file in the build dir " - "(requires Bear for non-CMake projects)") + "compilation-db", + "-cdb", + help="Create a compile_commands.json file in the build dir " "(requires Bear for non-CMake projects)", + ) self.copy_compilation_db_to_source_dir = None # False for jenkins, an option for cheribuild self.generate_cmakelists = False # False for jenkins, an option for cheribuild # Run QEMU options - self.wait_for_debugger = loader.add_bool_option("wait-for-debugger", group=loader.run_group, - help="Start QEMU in the 'wait for a debugger' state when" - "launching CheriBSD,FreeBSD, etc.") - - self.debugger_in_tmux_pane = loader.add_bool_option("debugger-in-tmux-pane", group=loader.run_group, - help="Start Qemu and gdb in another tmux split") - - self.gdb_random_port = loader.add_bool_option("gdb-random-port", default=True, group=loader.run_group, - help="Wait for gdb using a random port") - - self.run_under_gdb = loader.add_bool_option("run-under-gdb", group=loader.run_group, - help="Run tests/benchmarks under GDB. Note: currently most " - "targets ignore this flag.") + self.wait_for_debugger = loader.add_bool_option( + "wait-for-debugger", + group=loader.run_group, + help="Start QEMU in the 'wait for a debugger' state when" "launching CheriBSD,FreeBSD, etc.", + ) + + self.debugger_in_tmux_pane = loader.add_bool_option( + "debugger-in-tmux-pane", group=loader.run_group, help="Start Qemu and gdb in another tmux split" + ) + + self.gdb_random_port = loader.add_bool_option( + "gdb-random-port", default=True, group=loader.run_group, help="Wait for gdb using a random port" + ) + + self.run_under_gdb = loader.add_bool_option( + "run-under-gdb", + group=loader.run_group, + help="Run tests/benchmarks under GDB. Note: currently most " "targets ignore this flag.", + ) # Test options: self._test_ssh_key = loader.add_optional_path_option( - "test-ssh-key", group=loader.tests_group, + "test-ssh-key", + group=loader.tests_group, help="The SSH key to used to connect to the QEMU instance when running tests on CheriBSD. If not specified" - " a key will be generated in the build-root directory on-demand.") - self.use_minimal_benchmark_kernel = loader.add_bool_option("use-minimal-benchmark-kernel", - help="Use a CHERI BENCHMARK version of the " - "cheribsd-mfs-root-kernel (without " - "INVARIATES) for the " - "run-minimal target and for tests. This can " - "speed up longer running tests. This is the " - "default for " - "PostgreSQL and libc++ tests (passing " - "use-minimal-benchmark-kernel can force these " - "tests to use " - "an INVARIANTS kernel).", - group=loader.tests_group, default=False) - - self.test_extra_args = loader.add_commandline_only_option("test-extra-args", group=loader.tests_group, - type=list, - metavar="ARGS", - help="Additional flags to pass to the test script " - "in --test") - self.tests_interact = loader.add_commandline_only_bool_option("interact-after-tests", group=loader.tests_group, - help="Interact with the CheriBSD instance after " - "running the tests on QEMU (only for " - "--test)") - self.tests_env_only = loader.add_commandline_only_bool_option("test-environment-only", group=loader.tests_group, - help="Don't actually run the tests. Instead " - "setup a QEMU instance with the right " - "paths set up.") - self.test_ld_preload = loader.add_optional_path_option("test-ld-preload", group=loader.tests_group, - help="Preload the given library before running tests") + " a key will be generated in the build-root directory on-demand.", + ) + self.use_minimal_benchmark_kernel = loader.add_bool_option( + "use-minimal-benchmark-kernel", + help="Use a CHERI BENCHMARK version of the " + "cheribsd-mfs-root-kernel (without " + "INVARIATES) for the " + "run-minimal target and for tests. This can " + "speed up longer running tests. This is the " + "default for " + "PostgreSQL and libc++ tests (passing " + "use-minimal-benchmark-kernel can force these " + "tests to use " + "an INVARIANTS kernel).", + group=loader.tests_group, + default=False, + ) + + self.test_extra_args = loader.add_commandline_only_option( + "test-extra-args", + group=loader.tests_group, + type=list, + metavar="ARGS", + help="Additional flags to pass to the test script " "in --test", + ) + self.tests_interact = loader.add_commandline_only_bool_option( + "interact-after-tests", + group=loader.tests_group, + help="Interact with the CheriBSD instance after " "running the tests on QEMU (only for " "--test)", + ) + self.tests_env_only = loader.add_commandline_only_bool_option( + "test-environment-only", + group=loader.tests_group, + help="Don't actually run the tests. Instead " "setup a QEMU instance with the right " "paths set up.", + ) + self.test_ld_preload = loader.add_optional_path_option( + "test-ld-preload", group=loader.tests_group, help="Preload the given library before running tests" + ) self.benchmark_fpga_extra_args = loader.add_commandline_only_option( - "benchmark-fpga-extra-args", group=loader.benchmark_group, type=list, metavar="ARGS", - help="Extra options for the FPGA management script") - self.benchmark_clean_boot = loader.add_bool_option("benchmark-clean-boot", group=loader.benchmark_group, - help="Reboot the FPGA with a new bitfile and kernel before " - "running benchmarks. " - "If not set, assume the FPGA is running.") + "benchmark-fpga-extra-args", + group=loader.benchmark_group, + type=list, + metavar="ARGS", + help="Extra options for the FPGA management script", + ) + self.benchmark_clean_boot = loader.add_bool_option( + "benchmark-clean-boot", + group=loader.benchmark_group, + help="Reboot the FPGA with a new bitfile and kernel before " + "running benchmarks. " + "If not set, assume the FPGA is running.", + ) self.benchmark_extra_args = loader.add_commandline_only_option( - "benchmark-extra-args", group=loader.benchmark_group, type=list, - metavar="ARGS", help="Additional flags to pass to the program executed in --benchmark") + "benchmark-extra-args", + group=loader.benchmark_group, + type=list, + metavar="ARGS", + help="Additional flags to pass to the program executed in --benchmark", + ) self.benchmark_ssh_host = loader.add_option( - "benchmark-ssh-host", group=loader.benchmark_group, type=str, - default="cheri-fpga", help="The SSH hostname/IP for the benchmark FPGA") + "benchmark-ssh-host", + group=loader.benchmark_group, + type=str, + default="cheri-fpga", + help="The SSH hostname/IP for the benchmark FPGA", + ) self.benchmark_statcounters_suffix = loader.add_option( - "benchmark-csv-suffix", group=loader.benchmark_group, - help="Add a custom suffix for the statcounters CSV.") + "benchmark-csv-suffix", group=loader.benchmark_group, help="Add a custom suffix for the statcounters CSV." + ) self.benchmark_ld_preload = loader.add_optional_path_option( - "benchmark-ld-preload", group=loader.benchmark_group, - help="Preload the given library before running benchmarks") + "benchmark-ld-preload", + group=loader.benchmark_group, + help="Preload the given library before running benchmarks", + ) self.benchmark_with_debug_kernel = loader.add_bool_option( - "benchmark-with-debug-kernel", group=loader.benchmark_group, - help="Run the benchmark with a kernel that has assertions enabled.") + "benchmark-with-debug-kernel", + group=loader.benchmark_group, + help="Run the benchmark with a kernel that has assertions enabled.", + ) self.benchmark_lazy_binding = loader.add_bool_option( - "benchmark-lazy-binding", group=loader.benchmark_group, - help="Run the benchmark without setting LD_BIND_NOW.") + "benchmark-lazy-binding", + group=loader.benchmark_group, + help="Run the benchmark without setting LD_BIND_NOW.", + ) self.benchmark_iterations = loader.add_option( - "benchmark-iterations", type=int, group=loader.benchmark_group, - help="Override the number of iterations for the benchmark. " - "Note: not all benchmarks support this option") + "benchmark-iterations", + type=int, + group=loader.benchmark_group, + help="Override the number of iterations for the benchmark. " "Note: not all benchmarks support this option", + ) self.benchmark_with_qemu = loader.add_bool_option( - "benchmark-with-qemu", group=loader.benchmark_group, + "benchmark-with-qemu", + group=loader.benchmark_group, help="Run the benchmarks on QEMU instead of the FPGA (only useful to collect instruction counts or test " - "the benchmarks)") + "the benchmarks)", + ) self.shallow_clone = loader.add_bool_option( - "shallow-clone", default=True, + "shallow-clone", + default=True, help="Perform a shallow `git clone` when cloning new projects. This can save a lot of time for large" - "repositories such as FreeBSD or LLVM. Use `git fetch --unshallow` to convert to a non-shallow clone") + "repositories such as FreeBSD or LLVM. Use `git fetch --unshallow` to convert to a non-shallow clone", + ) self.fpga_custom_env_setup_script = loader.add_optional_path_option( - "beri-fpga-env-setup-script", group=loader.path_group, - help="Custom script to source to setup PATH and quartus, default to using cheri-cpu/cheri/setup.sh") + "beri-fpga-env-setup-script", + group=loader.path_group, + help="Custom script to source to setup PATH and quartus, default to using cheri-cpu/cheri/setup.sh", + ) self.local_arm_none_eabi_toolchain_relpath = Path("arm-none-eabi-sdk") self.arm_none_eabi_toolchain_prefix = loader.add_option( - "arm-none-eabi-prefix", default=ComputedDefaultValue(_default_arm_none_eabi_prefix, ""), + "arm-none-eabi-prefix", + default=ComputedDefaultValue(_default_arm_none_eabi_prefix, ""), group=loader.path_group, help="Prefix for arm-none-eabi-gcc binaries (e.g. /usr/bin/arm-none-eabi-). Available at" - "https://developer.arm.com/tools-and-software/open-source-software/" - "developer-tools/gnu-toolchain/gnu-rm/downloads") + "https://developer.arm.com/tools-and-software/open-source-software/" + "developer-tools/gnu-toolchain/gnu-rm/downloads", + ) self.build_morello_firmware_from_source = loader.add_bool_option( - "build-morello-firmware-from-source", help_hidden=False, - help="Build the firmware from source instead of downloading the latest release.") + "build-morello-firmware-from-source", + help_hidden=False, + help="Build the firmware from source instead of downloading the latest release.", + ) - self.list_kernels = loader.add_bool_option("list-kernels", group=loader.action_group, - help="List available kernel configs to run and exit") + self.list_kernels = loader.add_bool_option( + "list-kernels", group=loader.action_group, help="List available kernel configs to run and exit" + ) self.remote_morello_board = loader.add_option( - "remote-morello-board", help="SSH hostname of a Morello board. When set, some projects will run their " - "test suites on the remote board instead of QEMU.") + "remote-morello-board", + help="SSH hostname of a Morello board. When set, some projects will run their " + "test suites on the remote board instead of QEMU.", + ) self.targets: "Optional[list[str]]" = None - self.__optional_properties = ["internet_connection_last_checked_at", "start_after", "start_with", - "default_action"] + self.__optional_properties = [ + "internet_connection_last_checked_at", + "start_after", + "start_with", + "default_action", + ] def load(self) -> None: self.loader.load() @@ -494,8 +666,11 @@ def load(self) -> None: # flatten the potentially nested list if not self.action: if self.default_action is None: - fatal_error("Missing action, please pass one of", - ", ".join([str(action.option_name) for action in self._action_class]), pretend=False) + fatal_error( + "Missing action, please pass one of", + ", ".join([str(action.option_name) for action in self._action_class]), + pretend=False, + ) self.action = [self.default_action] else: assert isinstance(self.action, list) @@ -569,8 +744,20 @@ def test_ssh_key(self) -> Path: default_test_ssh_key_path = self.build_root / "insecure_test_ssh_key.pub" if not default_test_ssh_key_path.exists(): status_update("Generating SSH key for testing:", default_test_ssh_key_path) - run_command(["ssh-keygen", "-t", "ed25519", "-N", "", "-f", default_test_ssh_key_path.with_suffix(""), - "-C", "Test SSH key for cheribuild"], config=self) + run_command( + [ + "ssh-keygen", + "-t", + "ed25519", + "-N", + "", + "-f", + default_test_ssh_key_path.with_suffix(""), + "-C", + "Test SSH key for cheribuild", + ], + config=self, + ) return default_test_ssh_key_path def _ensure_required_properties_set(self) -> bool: @@ -600,7 +787,7 @@ def should_skip_dependency(self, target_name: str, requested_by: str) -> bool: # FIXME: not sure why this is needed def __getattribute__(self, item) -> "typing.Any": v = object.__getattribute__(self, item) - if hasattr(v, '__get__'): + if hasattr(v, "__get__"): # noinspection PyCallingNonCallable return v.__get__(self, self.__class__) # pytype: disable=attribute-error return v diff --git a/pycheribuild/config/compilation_targets.py b/pycheribuild/config/compilation_targets.py index fb283e3c6..85ce88e3d 100644 --- a/pycheribuild/config/compilation_targets.py +++ b/pycheribuild/config/compilation_targets.py @@ -182,7 +182,8 @@ def triple_for_target( config: "CheriConfig", *, include_version: bool, - ) -> str: ... + ) -> str: + ... def get_target_triple(self, *, include_version: bool) -> str: return self.triple_for_target(self.target, self.config, include_version=include_version) diff --git a/pycheribuild/config/computed_default_value.py b/pycheribuild/config/computed_default_value.py index 89fe81f08..f36bcdeed 100644 --- a/pycheribuild/config/computed_default_value.py +++ b/pycheribuild/config/computed_default_value.py @@ -33,20 +33,26 @@ T = typing.TypeVar("T") if typing.TYPE_CHECKING: # no-combine from ..utils import ConfigBase # no-combine + ConfigTy = typing.TypeVar("ConfigTy", bound=ConfigBase) # no-combine class ComputedDefaultValue(typing.Generic[T]): - def __init__(self, function: "Callable[[ConfigTy, Any], T]", - as_string: "Union[str, Callable[[Any], str]]", - as_readme_string: "Union[str, Callable[[Any], str], None]" = None, - inherit: "typing.Optional[ComputedDefaultValue[T]]" = None): + def __init__( + self, + function: "Callable[[ConfigTy, Any], T]", + as_string: "Union[str, Callable[[Any], str]]", + as_readme_string: "Union[str, Callable[[Any], str], None]" = None, + inherit: "typing.Optional[ComputedDefaultValue[T]]" = None, + ): if inherit is not None: + def inheriting_function(config, project): val = function(config, project) if val is None: val = inherit.function(config, project) return val + self.function = inheriting_function else: assert function is not None, "Must provide function or inherit" @@ -56,8 +62,10 @@ def inheriting_function(config, project): assert callable(as_string), "Inheriting only makes sense with callable as_string" if not callable(inherit.as_string): + def inherit_as_string_wrapper(cls): return inherit.as_string + inherited_as_string = inherit_as_string_wrapper else: inherited_as_string = inherit.as_string @@ -79,11 +87,14 @@ def inheriting_as_string(cls): def as_readme_string_none_wrapper(cls): return None + as_readme_string = as_readme_string_none_wrapper if not callable(inherit.as_readme_string): + def inherit_as_readme_string_wrapper(cls): return inherit.as_readme_string + inherited_as_readme_string = inherit_as_readme_string_wrapper else: inherited_as_readme_string = inherit.as_readme_string diff --git a/pycheribuild/config/config_loader_base.py b/pycheribuild/config/config_loader_base.py index ecba3f6fa..e5bcd600b 100644 --- a/pycheribuild/config/config_loader_base.py +++ b/pycheribuild/config/config_loader_base.py @@ -39,7 +39,7 @@ from .computed_default_value import ComputedDefaultValue from ..utils import ConfigBase, fatal_error, warning_message -T = typing.TypeVar('T') +T = typing.TypeVar("T") if typing.TYPE_CHECKING: import argparse @@ -73,8 +73,9 @@ class ConfigLoaderBase(ABC): # argparse groups used in the command line loader - def __init__(self, *, option_cls: "type[ConfigOptionBase]", - command_line_only_options_cls: "type[ConfigOptionBase]"): + def __init__( + self, *, option_cls: "type[ConfigOptionBase]", command_line_only_options_cls: "type[ConfigOptionBase]" + ): self.__option_cls: "type[ConfigOptionBase]" = option_cls self.__command_line_only_options_cls: "type[ConfigOptionBase]" = command_line_only_options_cls self.unknown_config_option_is_error = False @@ -84,7 +85,8 @@ def __init__(self, *, option_cls: "type[ConfigOptionBase]", self.dependencies_group = self.add_argument_group("Selecting which dependencies are built") self.path_group = self.add_argument_group("Configuration of default paths") self.cross_compile_options_group = self.add_argument_group( - "Adjust flags used when compiling MIPS/CHERI projects") + "Adjust flags used when compiling MIPS/CHERI projects" + ) self.tests_group = self.add_argument_group("Configuration for running tests") self.benchmark_group = self.add_argument_group("Configuration for running benchmarks") self.run_group = self.add_argument_group("Configuration for launching QEMU (and other simulators)") @@ -101,15 +103,30 @@ def add_commandline_only_option(self, *args, type: "Callable[[str], T]" = str, * def add_commandline_only_bool_option(self, *args, default=False, **kwargs) -> bool: assert default is False or kwargs.get("negatable") is True - return self.add_option(*args, option_cls=self.__command_line_only_options_cls, default=default, - negatable=kwargs.pop("negatable", False), type=bool, **kwargs) + return self.add_option( + *args, + option_cls=self.__command_line_only_options_cls, + default=default, + negatable=kwargs.pop("negatable", False), + type=bool, + **kwargs, + ) # noinspection PyShadowingBuiltins - def add_option(self, name: str, shortname=None, *, type: "Union[type[T], Callable[[str], T]]" = str, - default: "Union[ComputedDefaultValue[T], Optional[T], Callable[[ConfigBase, typing.Any], T]]" = None, - _owning_class: "Optional[type]" = None, _fallback_names: "Optional[list[str]]" = None, - option_cls: "Optional[type[ConfigOptionBase[T]]]" = None, replaceable=False, - fallback_replaceable: "Optional[bool]" = None, **kwargs) -> T: + def add_option( + self, + name: str, + shortname=None, + *, + type: "Union[type[T], Callable[[str], T]]" = str, + default: "Union[ComputedDefaultValue[T], Optional[T], Callable[[ConfigBase, typing.Any], T]]" = None, + _owning_class: "Optional[type]" = None, + _fallback_names: "Optional[list[str]]" = None, + option_cls: "Optional[type[ConfigOptionBase[T]]]" = None, + replaceable=False, + fallback_replaceable: "Optional[bool]" = None, + **kwargs, + ) -> T: if option_cls is None: option_cls = self.__option_cls if fallback_replaceable is None: @@ -124,10 +141,16 @@ def add_option(self, name: str, shortname=None, *, type: "Union[type[T], Callabl fallback_handle = self.option_handles.get(fallback_name) if fallback_handle is None or fallback_handle.replaceable: # Do not assign an owning class or a default value to this implicitly added fallback option. - self.add_option(fallback_name, type=type, option_cls=option_cls, replaceable=fallback_replaceable, - is_fallback=True) - option = option_cls(name, shortname, default, type, _owning_class, _loader=self, - _fallback_names=_fallback_names, **kwargs) + self.add_option( + fallback_name, + type=type, + option_cls=option_cls, + replaceable=fallback_replaceable, + is_fallback=True, + ) + option = option_cls( + name, shortname, default, type, _owning_class, _loader=self, _fallback_names=_fallback_names, **kwargs + ) if name in self.option_handles: self.option_handles[name]._replace_option(option, replaceable) else: @@ -138,14 +161,20 @@ def add_bool_option(self, name: str, shortname=None, default=False, **kwargs) -> # noinspection PyTypeChecker return self.add_option(name, shortname, default=default, type=bool, **kwargs) - def add_path_option(self, name: str, *, - default: "Union[ComputedDefaultValue[Path], Path, Callable[[ConfigBase, typing.Any], Path]]", - shortname=None, **kwargs) -> Path: + def add_path_option( + self, + name: str, + *, + default: "Union[ComputedDefaultValue[Path], Path, Callable[[ConfigBase, typing.Any], Path]]", + shortname=None, + **kwargs, + ) -> Path: # we have to make sure we resolve this to an absolute path because otherwise steps where CWD is different fail! return typing.cast(Path, self.add_option(name, shortname, type=Path, default=default, **kwargs)) - def add_optional_path_option(self, name: str, *, default: "Optional[Path]" = None, shortname=None, - **kwargs) -> Path: + def add_optional_path_option( + self, name: str, *, default: "Optional[Path]" = None, shortname=None, **kwargs + ) -> Path: # we have to make sure we resolve this to an absolute path because otherwise steps where CWD is different fail! return self.add_option(name, shortname, type=Path, default=default, **kwargs) @@ -193,8 +222,9 @@ def targets(self) -> "list[str]": class AbstractConfigOption(typing.Generic[T], metaclass=ABCMeta): @abstractmethod - def load_option(self, config: "ConfigBase", instance: "Optional[object]", _: type, - return_none_if_default=False) -> T: + def load_option( + self, config: "ConfigBase", instance: "Optional[object]", _: type, return_none_if_default=False + ) -> T: ... @abstractmethod @@ -229,10 +259,19 @@ def _convert_type(self, loaded_result: _LoadedConfigValue) -> "Optional[T]": class ConfigOptionBase(AbstractConfigOption[T]): - def __init__(self, name: str, shortname: Optional[str], default, - value_type: "Union[type[T], Callable[[typing.Any], T]]", _owning_class=None, *, - _loader: "Optional[ConfigLoaderBase]" = None, _fallback_names: "Optional[list[str]]" = None, - _legacy_alias_names: "Optional[list[str]]" = None, is_fallback: bool = False): + def __init__( + self, + name: str, + shortname: Optional[str], + default, + value_type: "Union[type[T], Callable[[typing.Any], T]]", + _owning_class=None, + *, + _loader: "Optional[ConfigLoaderBase]" = None, + _fallback_names: "Optional[list[str]]" = None, + _legacy_alias_names: "Optional[list[str]]" = None, + is_fallback: bool = False, + ): self.name = name self.shortname = shortname self.default = default @@ -250,8 +289,9 @@ def __init__(self, name: str, shortname: Optional[str], default, self._is_default_value = False self.is_fallback_only = is_fallback - def load_option(self, config: "ConfigBase", instance: "Optional[object]", _: type, - return_none_if_default=False) -> T: + def load_option( + self, config: "ConfigBase", instance: "Optional[object]", _: type, return_none_if_default=False + ) -> T: result = self._load_option_impl(config, self.full_option_name) # fall back from --qtbase-mips/foo to --qtbase/foo # Try aliases first: @@ -283,8 +323,16 @@ def load_option(self, config: "ConfigBase", instance: "Optional[object]", _: typ try: result = self._convert_type(result) except ValueError as e: - fatal_error("Invalid value for option '", self.full_option_name, - "': could not convert '", result, "': ", str(e), sep="", pretend=config.pretend) + fatal_error( + "Invalid value for option '", + self.full_option_name, + "': could not convert '", + result, + "': ", + str(e), + sep="", + pretend=config.pretend, + ) sys.exit() return result @@ -305,9 +353,10 @@ def is_default_value(self) -> bool: return self._is_default_value def __get__(self, instance, owner) -> T: - assert instance is not None or not callable(self.default), \ - f"Tried to access read config option {self.full_option_name} without an object instance. " \ + assert instance is not None or not callable(self.default), ( + f"Tried to access read config option {self.full_option_name} without an object instance. " f"Config options using computed defaults can only be used with an object instance. Owner = {owner}" + ) # TODO: would be nice if this was possible (but too much depends on accessing values without instances) # if instance is None: @@ -335,8 +384,16 @@ def _convert_type(self, loaded_result: _LoadedConfigValue) -> "Optional[T]": if isinstance(self.value_type, type) and issubclass(self.value_type, collections.abc.Sequence): string_value = result result = shlex.split(string_value) - warning_message("Config option ", self.full_option_name, " (", string_value, ") should be a list, ", - "got a string instead -> assuming the correct value is ", result, sep="") + warning_message( + "Config option ", + self.full_option_name, + " (", + string_value, + ") should be a list, ", + "got a string instead -> assuming the correct value is ", + result, + sep="", + ) if isinstance(self.value_type, type) and issubclass(self.value_type, Path): expanded = os.path.expanduser(os.path.expandvars(str(result))) while expanded.startswith("//"): @@ -391,8 +448,9 @@ def _replace_option(self, option: "ConfigOptionBase[T]", replaceable: bool): def replaceable(self) -> bool: return self.__replaceable - def load_option(self, config: "ConfigBase", instance: "Optional[object]", _: type, - return_none_if_default=False) -> T: + def load_option( + self, config: "ConfigBase", instance: "Optional[object]", _: type, return_none_if_default=False + ) -> T: return self._get_option().load_option(config, instance, _, return_none_if_default) def _load_option_impl(self, config: "ConfigBase", target_option_name) -> "Optional[_LoadedConfigValue]": diff --git a/pycheribuild/config/defaultconfig.py b/pycheribuild/config/defaultconfig.py index 299b10166..8b01aad4a 100644 --- a/pycheribuild/config/defaultconfig.py +++ b/pycheribuild/config/defaultconfig.py @@ -42,12 +42,19 @@ class CheribuildAction(CheribuildActionEnum): BUILD = ("--build", "Run (usually build+install) chosen targets (default)") TEST = ("--test", "Run tests for the passed targets instead of building them", "--run-tests") BENCHMARK = ("--benchmark", "Run tests for the passed targets instead of building them") - BUILD_AND_TEST = ("--build-and-test", "Run chosen targets and then run any tests afterwards", None, - # can get the other instances yet -> use strings - ["build", "test"]) + BUILD_AND_TEST = ( + "--build-and-test", + "Run chosen targets and then run any tests afterwards", + None, + # can get the other instances yet -> use strings + ["build", "test"], + ) LIST_TARGETS = ("--list-targets", "List all available targets and exit") - DUMP_CONFIGURATION = ("--dump-configuration", "Print the current configuration as JSON. This can be saved to " - "~/.config/cheribuild.json to make it persistent") + DUMP_CONFIGURATION = ( + "--dump-configuration", + "Print the current configuration as JSON. This can be saved to " + "~/.config/cheribuild.json to make it persistent", + ) def __init__(self, option_name, help_message, altname=None, actions=None) -> None: self.option_name = option_name @@ -61,14 +68,18 @@ def __init__(self, option_name, help_message, altname=None, actions=None) -> Non class DefaultCheribuildConfigLoader(JsonAndCommandLineConfigLoader): def finalize_options(self, available_targets: "list[str]", **kwargs) -> None: - target_option = self._parser.add_argument("targets", metavar="TARGET", nargs=argparse.ZERO_OR_MORE, - help="The targets to build") + target_option = self._parser.add_argument( + "targets", metavar="TARGET", nargs=argparse.ZERO_OR_MORE, help="The targets to build" + ) if argcomplete and self.is_completing_arguments: # if OSInfo.IS_FREEBSD: # FIXME: for some reason this won't work self.completion_excludes = ["-t", "--skip-dependencies"] if sys.platform.startswith("freebsd"): - self.completion_excludes += ["--freebsd-builder-copy-only", "--freebsd-builder-hostname", - "--freebsd-builder-output-path"] + self.completion_excludes += [ + "--freebsd-builder-copy-only", + "--freebsd-builder-hostname", + "--freebsd-builder-output-path", + ] visible_targets = available_targets.copy() visible_targets.remove("__run_everything__") @@ -76,8 +87,14 @@ def finalize_options(self, available_targets: "list[str]", **kwargs) -> None: target_option.completer = target_completer # make sure we get target completion for the unparsed args too by adding another zero_or more options # not sure why this works but it's a nice hack - unparsed = self._parser.add_argument("targets", metavar="TARGET", type=list, nargs=argparse.ZERO_OR_MORE, - help=argparse.SUPPRESS, choices=available_targets) + unparsed = self._parser.add_argument( + "targets", + metavar="TARGET", + type=list, + nargs=argparse.ZERO_OR_MORE, + help=argparse.SUPPRESS, + choices=available_targets, + ) unparsed.completer = target_completer @@ -86,106 +103,152 @@ def __init__(self, loader: ConfigLoaderBase, available_targets: "list[str]") -> super().__init__(loader, action_class=CheribuildAction) self.default_action = CheribuildAction.BUILD # The run mode: - self.get_config_option = loader.add_option("get-config-option", type=str, metavar="KEY", - group=loader.action_group, - help="Print the value of config option KEY and exit") + self.get_config_option = loader.add_option( + "get-config-option", + type=str, + metavar="KEY", + group=loader.action_group, + help="Print the value of config option KEY and exit", + ) # boolean flags self.quiet = loader.add_bool_option("quiet", "q", help="Don't show stdout of the commands that are executed") self.verbose = loader.add_bool_option("verbose", "v", help="Print all commmands that are executed") self.clean = loader.add_bool_option("clean", "c", help="Remove the build directory before build") self.force = loader.add_bool_option("force", "f", help="Don't prompt for user input but use the default action") - self.write_logfile = loader.add_bool_option("logfile", help="Write a logfile for the build steps", - default=False) + self.write_logfile = loader.add_bool_option( + "logfile", help="Write a logfile for the build steps", default=False + ) self.skip_update = loader.add_bool_option("skip-update", help="Skip the git pull step") self.skip_clone = False self.confirm_clone = loader.add_bool_option( - "confirm-clone", help="Ask for confirmation before cloning repositories.") - self.force_update = loader.add_bool_option("force-update", help="Always update (with autostash) even if there " - "are uncommitted changes") + "confirm-clone", help="Ask for confirmation before cloning repositories." + ) + self.force_update = loader.add_bool_option( + "force-update", help="Always update (with autostash) even if there " "are uncommitted changes" + ) self.presume_connectivity = loader.add_bool_option( "presume-connectivity", - help="Do not probe for network connectivity and just assume that we are suitably connected") + help="Do not probe for network connectivity and just assume that we are suitably connected", + ) # TODO: should replace this group with a tristate value configure_group = loader.add_mutually_exclusive_group() - self.skip_configure = loader.add_bool_option("skip-configure", help="Skip the configure step", - group=configure_group) - self.force_configure = loader.add_bool_option("reconfigure", "-force-configure", group=configure_group, - help="Always run the configure step, even for CMake projects " - "with a valid cache.") + self.skip_configure = loader.add_bool_option( + "skip-configure", help="Skip the configure step", group=configure_group + ) + self.force_configure = loader.add_bool_option( + "reconfigure", + "-force-configure", + group=configure_group, + help="Always run the configure step, even for CMake projects " "with a valid cache.", + ) self.include_dependencies = loader.add_commandline_only_bool_option( - "include-dependencies", "d", group=loader.dependencies_group, + "include-dependencies", + "d", + group=loader.dependencies_group, help="Also build the dependencies of targets passed on the command line. Targets passed on the command " - "line will be reordered and processed in an order that ensures dependencies are built before the " - "real target. (run --list-targets for more information). By default this does not build toolchain " - "targets such as LLVM. Pass --include-toolchain-dependencies to also build those.") + "line will be reordered and processed in an order that ensures dependencies are built before the " + "real target. (run --list-targets for more information). By default this does not build toolchain " + "targets such as LLVM. Pass --include-toolchain-dependencies to also build those.", + ) self.include_toolchain_dependencies = loader.add_bool_option( - "include-toolchain-dependencies", default=True, group=loader.dependencies_group, - help="Include toolchain targets such as LLVM and QEMU when --include-dependencies is set.") + "include-toolchain-dependencies", + default=True, + group=loader.dependencies_group, + help="Include toolchain targets such as LLVM and QEMU when --include-dependencies is set.", + ) self.enable_hybrid_targets = loader.add_bool_option( - "enable-hybrid-targets", default=False, help_hidden=True, + "enable-hybrid-targets", + default=False, + help_hidden=True, help="Enable building hybrid targets. This is highly discouraged, " - "only enable if you know what you're doing.") + "only enable if you know what you're doing.", + ) start_after_group = loader.dependencies_group.add_mutually_exclusive_group() self.start_with = loader.add_commandline_only_option( - "start-with", metavar="TARGET", group=start_after_group, - help="Start building at TARGET (useful when resuming an interrupted --include-depedencies build)") + "start-with", + metavar="TARGET", + group=start_after_group, + help="Start building at TARGET (useful when resuming an interrupted --include-depedencies build)", + ) self.start_after = loader.add_commandline_only_option( - "start-after", metavar="TARGET", group=start_after_group, - help="Start building after TARGET (useful when resuming an interrupted --include-depedencies build)") + "start-after", + metavar="TARGET", + group=start_after_group, + help="Start building after TARGET (useful when resuming an interrupted --include-depedencies build)", + ) self.copy_compilation_db_to_source_dir = loader.add_commandline_only_bool_option( "compilation-db-in-source-dir", - help="Generate a compile_commands.json and also copy it to the source directory") + help="Generate a compile_commands.json and also copy it to the source directory", + ) self.generate_cmakelists = loader.add_bool_option( "generate-cmakelists", - help="Generate a CMakeLists.txt that just calls cheribuild. Useful for IDEs that only support CMake") + help="Generate a CMakeLists.txt that just calls cheribuild. Useful for IDEs that only support CMake", + ) self.make_without_nice = loader.add_bool_option("make-without-nice", help="Run make/ninja without nice(1)") default_make_jobs = default_make_jobs_count() - default_make_jobs_computed = ComputedDefaultValue(lambda p, cls: default_make_jobs, - as_string=str(default_make_jobs), - as_readme_string="") - self.make_jobs: int = loader.add_option("make-jobs", "j", type=int, default=default_make_jobs_computed, - help="Number of jobs to use for compiling") + default_make_jobs_computed = ComputedDefaultValue( + lambda p, cls: default_make_jobs, as_string=str(default_make_jobs), as_readme_string="" + ) + self.make_jobs: int = loader.add_option( + "make-jobs", "j", type=int, default=default_make_jobs_computed, help="Number of jobs to use for compiling" + ) # configurable paths - self.source_root = loader.add_path_option("source-root", - default=Path(os.path.expanduser("~/cheri")), group=loader.path_group, - help="The directory to store all sources") - self.output_root = loader.add_path_option("output-root", - default=lambda p, cls: (p.source_root / "output"), - group=loader.path_group, - help="The directory to store all output (default: " - "'/output')") - self.build_root = loader.add_path_option("build-root", - default=lambda p, cls: (p.source_root / "build"), - group=loader.path_group, - help="The directory for all the builds (default: " - "'/build')") - self.tools_root = loader.add_path_option("tools-root", - default=lambda p, cls: p.output_root, group=loader.path_group, - help="The directory to find sdk and bootstrap tools (default: " - "'')") + self.source_root = loader.add_path_option( + "source-root", + default=Path(os.path.expanduser("~/cheri")), + group=loader.path_group, + help="The directory to store all sources", + ) + self.output_root = loader.add_path_option( + "output-root", + default=lambda p, cls: (p.source_root / "output"), + group=loader.path_group, + help="The directory to store all output (default: " "'/output')", + ) + self.build_root = loader.add_path_option( + "build-root", + default=lambda p, cls: (p.source_root / "build"), + group=loader.path_group, + help="The directory for all the builds (default: " "'/build')", + ) + self.tools_root = loader.add_path_option( + "tools-root", + default=lambda p, cls: p.output_root, + group=loader.path_group, + help="The directory to find sdk and bootstrap tools (default: " "'')", + ) default_morello_sdk = ComputedDefaultValue( function=lambda p, cls: (p.tools_root / p.default_morello_sdk_directory_name), - as_string="'/morello-sdk'") - self.morello_sdk_dir = loader.add_path_option("morello-sdk-root", - default=default_morello_sdk, group=loader.path_group, - help="The directory to find/install the Morello SDK") - self.sysroot_output_root = loader.add_path_option("sysroot-install-root", shortname="-sysroot-install-dir", - default=lambda p, cls: p.tools_root, group=loader.path_group, - help="Sysroot prefix (default: '')") + as_string="'/morello-sdk'", + ) + self.morello_sdk_dir = loader.add_path_option( + "morello-sdk-root", + default=default_morello_sdk, + group=loader.path_group, + help="The directory to find/install the Morello SDK", + ) + self.sysroot_output_root = loader.add_path_option( + "sysroot-install-root", + shortname="-sysroot-install-dir", + default=lambda p, cls: p.tools_root, + group=loader.path_group, + help="Sysroot prefix (default: '')", + ) # Hidden option to enable foo-hybrid-for-purecap-rootfs targets for all projects. This option is not actually # uses since we have to look at sys.argv[] directly due to depedency cycles. However, we do still need it since # we would otherwise get an invalid command line argument error. self.enable_hybrid_for_purecap_rootfs_targets = loader.add_commandline_only_bool_option( - "enable-hybrid-for-purecap-rootfs-targets", default=False, help_hidden=True) + "enable-hybrid-for-purecap-rootfs-targets", default=False, help_hidden=True + ) loader.finalize_options(available_targets) def load(self) -> None: diff --git a/pycheribuild/config/jenkinsconfig.py b/pycheribuild/config/jenkinsconfig.py index 9bb3a5133..692d6f372 100644 --- a/pycheribuild/config/jenkinsconfig.py +++ b/pycheribuild/config/jenkinsconfig.py @@ -90,74 +90,120 @@ class JenkinsConfig(CheriConfig): def __init__(self, loader: ConfigLoaderBase, available_targets: "list[str]") -> None: super().__init__(loader, action_class=JenkinsAction) self.cpu = loader.add_commandline_only_option( - "cpu", default=os.getenv("CPU", "default"), - help="Only used for backwards compatibility with old jenkins jobs") + "cpu", + default=os.getenv("CPU", "default"), + help="Only used for backwards compatibility with old jenkins jobs", + ) self.workspace = loader.add_commandline_only_option( - "workspace", default=os.getenv("WORKSPACE"), type=Path, - help="The root directory for building (defaults to $WORKSPACE)") + "workspace", + default=os.getenv("WORKSPACE"), + type=Path, + help="The root directory for building (defaults to $WORKSPACE)", + ) self.compiler_archive_name = loader.add_commandline_only_option( - "compiler-archive", type=str, default="cheri-clang-llvm.tar.xz", - help="The name of the archive containing the compiler") + "compiler-archive", + type=str, + default="cheri-clang-llvm.tar.xz", + help="The name of the archive containing the compiler", + ) self.compiler_archive_output_path = loader.add_commandline_only_option( - "compiler-archive-output-path", type=Path, default=_infer_compiler_output_path, - help="The path where to extract the compiler") + "compiler-archive-output-path", + type=Path, + default=_infer_compiler_output_path, + help="The path where to extract the compiler", + ) self.compiler_type = loader.add_commandline_only_option( - "compiler-type", type=CompilerType, default=CompilerType.CHERI_LLVM, + "compiler-type", + type=CompilerType, + default=CompilerType.CHERI_LLVM, enum_choices=[CompilerType.CHERI_LLVM, CompilerType.MORELLO_LLVM, CompilerType.UPSTREAM_LLVM], - help="The type of the compiler to extract (used to infer the output " - " path)") + help="The type of the compiler to extract (used to infer the output " " path)", + ) self.sysroot_archive_name = loader.add_commandline_only_option( - "sysroot-archive", type=str, default="cheribsd-sysroot.tar.xz", - help="The name of the archive containing the sysroot") + "sysroot-archive", + type=str, + default="cheribsd-sysroot.tar.xz", + help="The name of the archive containing the sysroot", + ) self.sysroot_archive_output_path = loader.add_commandline_only_option( - "sysroot-archive-output-path", type=Path, - default=ComputedDefaultValue(lambda c, _: c.compiler_archive_output_path / "sysroot", - as_string="/sysroot"), - help="The path where to extract the sysroot (default=") + "sysroot-archive-output-path", + type=Path, + default=ComputedDefaultValue( + lambda c, _: c.compiler_archive_output_path / "sysroot", as_string="/sysroot" + ), + help="The path where to extract the sysroot (default=", + ) self.keep_install_dir = loader.add_commandline_only_bool_option( - "keep-install-dir", help="Don't delete the install dir prior to build") + "keep-install-dir", help="Don't delete the install dir prior to build" + ) self.keep_sdk_dir = loader.add_commandline_only_bool_option( - "keep-sdk-dir", help="Don't delete existing SDK dir even if there is a newer archive") + "keep-sdk-dir", help="Don't delete existing SDK dir even if there is a newer archive" + ) self.force_update = loader.add_commandline_only_bool_option( - "force-update", help="Do the updating (not recommended in jenkins!)") + "force-update", help="Do the updating (not recommended in jenkins!)" + ) self.copy_compilation_db_to_source_dir = False self.make_without_nice = False - self.make_jobs = loader.add_commandline_only_option("make-jobs", "j", type=int, - default=default_jenkins_make_jobs_count, - help="Number of jobs to use for compiling") - self.use_all_cores = loader.add_commandline_only_bool_option("use-all-cores", - help="Use all available cores for building (" - "Note: Should only be used for LLVM or " - "short-running jobs!)") + self.make_jobs = loader.add_commandline_only_option( + "make-jobs", + "j", + type=int, + default=default_jenkins_make_jobs_count, + help="Number of jobs to use for compiling", + ) + self.use_all_cores = loader.add_commandline_only_bool_option( + "use-all-cores", + help="Use all available cores for building (" + "Note: Should only be used for LLVM or " + "short-running jobs!)", + ) self.installation_prefix = loader.add_commandline_only_option( - "install-prefix", type=absolute_path_only, default=default_install_prefix, - help="The install prefix for cross compiled projects (the path in the install image)") + "install-prefix", + type=absolute_path_only, + default=default_install_prefix, + help="The install prefix for cross compiled projects (the path in the install image)", + ) self.use_system_compiler_for_native = loader.add_commandline_only_bool_option( - "use-system-compiler-for-native", "-without-sdk", - help="Don't use the CHERI SDK -> only /usr (for native builds)") + "use-system-compiler-for-native", + "-without-sdk", + help="Don't use the CHERI SDK -> only /usr (for native builds)", + ) self.strip_elf_files = loader.add_commandline_only_bool_option( - "strip-elf-files", help="Strip ELF files before creating the tarball", default=True, negatable=True) + "strip-elf-files", help="Strip ELF files before creating the tarball", default=True, negatable=True + ) self._cheri_sdk_dir_override = loader.add_commandline_only_option( - "cheri-sdk-path", default=None, type=Path, - help="Override the path to the CHERI SDK (default is $WORKSPACE/cherisdk)") + "cheri-sdk-path", + default=None, + type=Path, + help="Override the path to the CHERI SDK (default is $WORKSPACE/cherisdk)", + ) self._morello_sdk_dir_override = loader.add_commandline_only_option( - "morello-sdk-path", default=None, type=Path, - help="Override the path to the Morello SDK (default is $WORKSPACE/morello-sdk)") + "morello-sdk-path", + default=None, + type=Path, + help="Override the path to the Morello SDK (default is $WORKSPACE/morello-sdk)", + ) self.extract_compiler_only = loader.add_commandline_only_bool_option( - "extract-compiler-only", help="Don't attempt to extract a sysroot") + "extract-compiler-only", help="Don't attempt to extract a sysroot" + ) self.tarball_name = loader.add_commandline_only_option( - "tarball-name", default=lambda conf, cls: conf.targets[0] + "-" + conf.cpu + ".tar.xz") + "tarball-name", default=lambda conf, cls: conf.targets[0] + "-" + conf.cpu + ".tar.xz" + ) self.default_output_path = "tarball" - default_output = ComputedDefaultValue(lambda c, _: c.workspace / c.default_output_path, - "$WORKSPACE/" + self.default_output_path) - self.output_root = loader.add_commandline_only_option("output-path", default=default_output, type=Path, - help="Path for the output (relative to $WORKSPACE)") + default_output = ComputedDefaultValue( + lambda c, _: c.workspace / c.default_output_path, "$WORKSPACE/" + self.default_output_path + ) + self.output_root = loader.add_commandline_only_option( + "output-path", default=default_output, type=Path, help="Path for the output (relative to $WORKSPACE)" + ) self.sysroot_output_root = loader.add_commandline_only_option( "sysroot-output-path", default=ComputedDefaultValue(function=lambda c, _: c.workspace, as_string="$WORKSPACE"), - type=Path, help="Path for the installed sysroot (defaults to the same value as --output-path)") + type=Path, + help="Path for the installed sysroot (defaults to the same value as --output-path)", + ) # self.strip_install_prefix_from_archive = loader.add_commandline_only_bool_option( # "strip-install-prefix-from-archive", # help="Only put the files inside the install prefix into the tarball (stripping the leading @@ -167,8 +213,9 @@ def __init__(self, loader: ConfigLoaderBase, available_targets: "list[str]") -> self.confirm_clone = False self.verbose = True self.quiet = False - self.clean = loader.add_commandline_only_bool_option("clean", default=True, negatable=True, - help="Clean build directory before building") + self.clean = loader.add_commandline_only_bool_option( + "clean", default=True, negatable=True, help="Clean build directory before building" + ) self.force = True # no user input in jenkins self.write_logfile = False # jenkins stores the output anyway self.skip_configure = loader.add_bool_option("skip-configure", help="Skip the configure step") @@ -178,7 +225,8 @@ def __init__(self, loader: ConfigLoaderBase, available_targets: "list[str]") -> self.allow_more_than_one_target = loader.add_commandline_only_bool_option( "allow-more-than-one-target", # help_hidden=True, Note: setting this to True seems to break argparse help="Allow more than one target on the command line. This should only be used for testing since " - "dependencies are not resolved!") + "dependencies are not resolved!", + ) loader.finalize_options(available_targets) self.FS = FileSystemUtils(self) @@ -208,8 +256,12 @@ def load(self) -> None: super().load() if not self.workspace or not self.workspace.is_dir(): - fatal_error("WORKSPACE is not set to a valid directory:", self.workspace, pretend=self.pretend, - fatal_when_pretending=True) + fatal_error( + "WORKSPACE is not set to a valid directory:", + self.workspace, + pretend=self.pretend, + fatal_when_pretending=True, + ) self.source_root = self.workspace self.build_root = self.workspace if self.output_root != self.workspace / self.default_output_path: @@ -219,8 +271,14 @@ def load(self) -> None: if os.path.relpath(str(self.output_root), str(self.workspace)).startswith(".."): fatal_error("Output path", self.output_root, "must be inside workspace", self.workspace, pretend=False) if os.path.relpath(str(self.sysroot_output_root), str(self.workspace)).startswith(".."): - fatal_error("Sysroot output path", self.sysroot_output_root, "must be inside workspace", self.workspace, - pretend=False, fatal_when_pretending=True) + fatal_error( + "Sysroot output path", + self.sysroot_output_root, + "must be inside workspace", + self.workspace, + pretend=False, + fatal_when_pretending=True, + ) # expect the CheriBSD disk images in the workspace root self.cheribsd_image_root = self.workspace @@ -254,14 +312,26 @@ def load(self) -> None: self.clang_plusplus_path = Path(os.getenv("HOST_CXX", self.clang_plusplus_path)) self.clang_cpp_path = Path(os.getenv("HOST_CPP", self.clang_cpp_path)) if not self.clang_path.exists(): - fatal_error("C compiler", self.clang_path, - "does not exit. Pass --clang-path or set $HOST_CC", pretend=self.pretend) + fatal_error( + "C compiler", + self.clang_path, + "does not exit. Pass --clang-path or set $HOST_CC", + pretend=self.pretend, + ) if not self.clang_plusplus_path.exists(): - fatal_error("C++ compiler", self.clang_plusplus_path, - "does not exit. Pass --clang++-path or set $HOST_CXX", pretend=self.pretend) + fatal_error( + "C++ compiler", + self.clang_plusplus_path, + "does not exit. Pass --clang++-path or set $HOST_CXX", + pretend=self.pretend, + ) if not self.clang_cpp_path.exists(): - fatal_error("C pre-processor", self.clang_cpp_path, - "does not exit. Pass --clang-cpp-path or set $HOST_CPP", pretend=self.pretend) + fatal_error( + "C pre-processor", + self.clang_cpp_path, + "does not exit. Pass --clang-cpp-path or set $HOST_CPP", + pretend=self.pretend, + ) else: # always use the CHERI clang built by jenkins (if available) # Prefix $WORKSPACE/native-sdk, but fall back to CHERI/Morello LLVM if that does not exist @@ -285,6 +355,7 @@ def load(self) -> None: assert self._ensure_required_properties_set() if os.getenv("DEBUG") is not None: import pprint + for k, v in self.__dict__.items(): if hasattr(v, "__get__"): # noinspection PyCallingNonCallable diff --git a/pycheribuild/config/loader.py b/pycheribuild/config/loader.py index a4c970f80..d5a2b8cb9 100644 --- a/pycheribuild/config/loader.py +++ b/pycheribuild/config/loader.py @@ -55,14 +55,13 @@ from ..colour import AnsiColour, coloured from ..utils import ConfigBase, error_message, fatal_error, status_update, warning_message -T = typing.TypeVar('T') -EnumTy = typing.TypeVar('EnumTy', bound=Enum) +T = typing.TypeVar("T") +EnumTy = typing.TypeVar("EnumTy", bound=Enum) # From https://bugs.python.org/issue25061 class _EnumArgparseType(typing.Generic[EnumTy]): - """Factory for creating enum object types - """ + """Factory for creating enum object types""" def __init__(self, enumclass: "type[EnumTy]"): self.enums: "type[EnumTy]" = enumclass @@ -74,8 +73,15 @@ def __init__(self, enumclass: "type[EnumTy]"): continue if c.isalpha() and c.isupper(): continue - raise RuntimeError("Invalid character '" + c + "' found in enum " + str(enumclass) + - " member " + member.name + ": must all be upper case letters or _ or digits.") + raise RuntimeError( + "Invalid character '" + + c + + "' found in enum " + + str(enumclass) + + " member " + + member.name + + ": must all be upper case letters or _ or digits." + ) # self.action = action def __call__(self, astring: "Union[str, list[str], EnumTy]") -> "Union[EnumTy, list[EnumTy]]": @@ -94,16 +100,16 @@ def __call__(self, astring: "Union[str, list[str], EnumTy]") -> "Union[EnumTy, l return e v = self.enums[enum_value_name] except KeyError: - msg = ', '.join([t.name.lower() for t in self.enums]) - msg = f'{name}: use one of {{{msg}}}' + msg = ", ".join([t.name.lower() for t in self.enums]) + msg = f"{name}: use one of {{{msg}}}" raise argparse.ArgumentTypeError(msg) # else: # self.action.choices = None # hugly hack to prevent post validation from choices return v def __repr__(self) -> str: - astr = ', '.join([t.name.lower() for t in self.enums]) - return f'{self.enums.__name__}({astr})' + astr = ", ".join([t.name.lower() for t in self.enums]) + return f"{self.enums.__name__}({astr})" # custom encoder to handle pathlib.Path and _LoadedConfigValue objects @@ -151,66 +157,123 @@ def get_argcomplete_prefix() -> str: # Based on Python 3.9 BooleanOptionalAction, but places the "no" after the first / class BooleanNegatableAction(argparse.Action): # noinspection PyShadowingBuiltins - def __init__(self, option_strings: "list[str]", dest, default=None, type=None, choices=None, required=False, - help=None, metavar=None): + def __init__( + self, + option_strings: "list[str]", + dest, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None, + ): # Add the negated option, placing the "no" after the / instead of the start -> --cheribsd/no-build-tests def collect_option_strings(original_strings): for opt in original_strings: all_option_strings.append(opt) - if opt.startswith('--'): + if opt.startswith("--"): slash_index = opt.rfind("/") if slash_index == -1: negated_opt = "--no-" + opt[2:] else: - negated_opt = opt[:slash_index + 1] + "no-" + opt[slash_index + 1:] + negated_opt = opt[: slash_index + 1] + "no-" + opt[slash_index + 1 :] all_option_strings.append(negated_opt) self._negated_option_strings.append(negated_opt) + all_option_strings = [] self._negated_option_strings = [] collect_option_strings(option_strings) # Don't show the alias options in --help output self.displayed_option_count = len(all_option_strings) - super().__init__(option_strings=all_option_strings, dest=dest, nargs=0, - default=default, type=type, choices=choices, required=required, help=help, metavar=metavar) + super().__init__( + option_strings=all_option_strings, + dest=dest, + nargs=0, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar, + ) def __call__(self, parser, namespace, values, option_string=None) -> None: if option_string in self.option_strings: setattr(namespace, self.dest, option_string not in self._negated_option_strings) def format_usage(self) -> str: - return ' | '.join(self.option_strings[:self.displayed_option_count]) + return " | ".join(self.option_strings[: self.displayed_option_count]) # argparse._StoreAction but with a possible list of aliases class StoreActionWithPossibleAliases(argparse.Action): # noinspection PyShadowingBuiltins - def __init__(self, option_strings: "list[str]", dest, nargs=None, default=None, type=None, choices=None, - required=False, help=None, metavar=None): + def __init__( + self, + option_strings: "list[str]", + dest, + nargs=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None, + ): if nargs == 1: raise ValueError("nargs for store actions must be 1") self.displayed_option_count = len(option_strings) - super().__init__(option_strings=option_strings, dest=dest, nargs=nargs, default=default, type=type, - choices=choices, required=required, help=help, metavar=metavar) + super().__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar, + ) def __call__(self, parser, namespace, values, option_string=None) -> None: setattr(namespace, self.dest, values) def format_usage(self) -> str: - return ' | '.join(self.option_strings[:self.displayed_option_count]) + return " | ".join(self.option_strings[: self.displayed_option_count]) class CommandLineConfigOption(ConfigOptionBase[T]): _loader: "JsonAndCommandLineConfigLoader" # noinspection PyProtectedMember,PyUnresolvedReferences - def __init__(self, name: str, shortname: "Optional[str]", default, - value_type: "Union[type[T], Callable[[Any], T]]", _owning_class, *, - _loader: "JsonAndCommandLineConfigLoader", help_hidden: bool, - group: "Optional[argparse._ArgumentGroup]", _fallback_names: "Optional[list[str]]" = None, - _legacy_alias_names: "Optional[list[str]]" = None, is_fallback: bool = False, **kwargs): - super().__init__(name, shortname, default, value_type, _owning_class, _loader=_loader, - _fallback_names=_fallback_names, _legacy_alias_names=_legacy_alias_names, - is_fallback=is_fallback) + def __init__( + self, + name: str, + shortname: "Optional[str]", + default, + value_type: "Union[type[T], Callable[[Any], T]]", + _owning_class, + *, + _loader: "JsonAndCommandLineConfigLoader", + help_hidden: bool, + group: "Optional[argparse._ArgumentGroup]", + _fallback_names: "Optional[list[str]]" = None, + _legacy_alias_names: "Optional[list[str]]" = None, + is_fallback: bool = False, + **kwargs, + ): + super().__init__( + name, + shortname, + default, + value_type, + _owning_class, + _loader=_loader, + _fallback_names=_fallback_names, + _legacy_alias_names=_legacy_alias_names, + is_fallback=is_fallback, + ) # hide obscure options unless --help-hidden/--help/all is passed if help_hidden and not self._loader.show_all_help: kwargs["help"] = argparse.SUPPRESS @@ -230,7 +293,8 @@ def __init__(self, name: str, shortname: "Optional[str]", default, self.default_str = "[]" else: assert isinstance(value_type, _EnumArgparseType), "default is enum but value type isn't: " + str( - value_type) + value_type + ) assert isinstance(default, Enum), "Should use enum constant for default and not " + str(default) self.default_str = default.name.lower() else: @@ -265,7 +329,7 @@ def _add_argparse_action(self, name, shortname, group, **kwargs) -> "argparse.Ac action = parser_obj.add_argument("--" + name, **kwargs) if self.default is not None and action.help is not None and has_default_help_text: if action.help != argparse.SUPPRESS: - action.help = action.help + " (default: \'" + self.default_str + "\')" + action.help = action.help + " (default: '" + self.default_str + "')" action.default = None # we don't want argparse default values! assert not action.type # we handle the type of the value manually return action @@ -305,8 +369,15 @@ def _load_option_impl(self, config: ConfigBase, target_option_name: str): # self.debug_msg(full_option_name, "from JSON:", from_json) if from_json is not None: if not config.quiet: - status_update("Overriding default value for", target_option_name, "with value from JSON key", - from_json.used_key, "->", from_json.value, file=sys.stderr) + status_update( + "Overriding default value for", + target_option_name, + "with value from JSON key", + from_json.used_key, + "->", + from_json.value, + file=sys.stderr, + ) return from_json return None # not found -> fall back to default @@ -348,8 +419,9 @@ def _load_from_json(self, full_option_name: str) -> "Optional[_LoadedConfigValue class DefaultValueOnlyConfigLoader(ConfigLoaderBase): def __init__(self) -> None: - super().__init__(option_cls=DefaultValueOnlyConfigOption, - command_line_only_options_cls=DefaultValueOnlyConfigOption) + super().__init__( + option_cls=DefaultValueOnlyConfigOption, command_line_only_options_cls=DefaultValueOnlyConfigOption + ) # Ignore options stored in other classes self.option_handles = dict() @@ -383,42 +455,63 @@ def dict_raise_on_duplicates_and_store_src(ordered_pairs, src_file) -> "dict[Any class ArgparseSetGivenAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None) -> None: setattr(namespace, self.dest, values) - setattr(namespace, self.dest + '_given', True) + setattr(namespace, self.dest + "_given", True) class CommandLineConfigLoader(ConfigLoaderBase): _parsed_args: argparse.Namespace - show_all_help: bool = any( - s in sys.argv for s in ("--help-all", "--help-hidden")) or ConfigLoaderBase.is_completing_arguments + show_all_help: bool = ( + any(s in sys.argv for s in ("--help-all", "--help-hidden")) or ConfigLoaderBase.is_completing_arguments + ) _argcomplete_prefix: "Optional[str]" = ( - get_argcomplete_prefix() if ConfigLoaderBase.is_completing_arguments else None) + get_argcomplete_prefix() if ConfigLoaderBase.is_completing_arguments else None + ) _argcomplete_prefix_includes_slash: bool = "/" in _argcomplete_prefix if _argcomplete_prefix else False - def __init__(self, argparser_class: "type[argparse.ArgumentParser]" = argparse.ArgumentParser, *, - option_cls=CommandLineConfigOption, command_line_only_options_cls=CommandLineConfigOption): + def __init__( + self, + argparser_class: "type[argparse.ArgumentParser]" = argparse.ArgumentParser, + *, + option_cls=CommandLineConfigOption, + command_line_only_options_cls=CommandLineConfigOption, + ): if self.is_completing_arguments or self.is_running_unit_tests: self._parser = argparser_class(formatter_class=NoOpHelpFormatter) else: terminal_width = shutil.get_terminal_size(fallback=(120, 24))[0] self._parser = argparser_class( - formatter_class=lambda prog: argparse.HelpFormatter(prog, width=terminal_width)) + formatter_class=lambda prog: argparse.HelpFormatter(prog, width=terminal_width) + ) super().__init__(option_cls=option_cls, command_line_only_options_cls=command_line_only_options_cls) - self._parser.add_argument("--help-all", "--help-hidden", action="help", help="Show all help options, including" - " the target-specific ones.") + self._parser.add_argument( + "--help-all", + "--help-hidden", + action="help", + help="Show all help options, including" " the target-specific ones.", + ) # noinspection PyShadowingBuiltins - def add_option(self, name: str, shortname=None, *, type: "Union[type[T], Callable[[str], T]]" = str, - default: "Union[ComputedDefaultValue[T], Optional[T], Callable[[ConfigBase, typing.Any], T]]" = None, - group=None, help_hidden=False, **kwargs) -> T: + def add_option( + self, + name: str, + shortname=None, + *, + type: "Union[type[T], Callable[[str], T]]" = str, + default: "Union[ComputedDefaultValue[T], Optional[T], Callable[[ConfigBase, typing.Any], T]]" = None, + group=None, + help_hidden=False, + **kwargs, + ) -> T: if not self.is_needed_for_completion(name, shortname, type): # We are autocompleting and there is a prefix that won't match this option, so we just return the # default value since it won't be displayed anyway. This should noticeably speed up tab-completion. return default # pytype: disable=bad-return-type if isinstance(type, builtins.type) and issubclass(type, Enum): # Handle enums as the argparse type - assert "action" not in kwargs or kwargs[ - "action"] == "append", "action should be none or append for Enum options" + assert ( + "action" not in kwargs or kwargs["action"] == "append" + ), "action should be none or append for Enum options" assert "choices" not in kwargs, "for enum options choices are the enum names (or set enum_choices)!" if "enum_choices" in kwargs: kwargs["choices"] = tuple(t.name.lower().replace("_", "-") for t in kwargs["enum_choices"]) @@ -432,8 +525,9 @@ def add_option(self, name: str, shortname=None, *, type: "Union[type[T], Callabl # noinspection PyTypeChecker kwargs["choices"] = tuple(t.name.lower() for t in type) type = _EnumArgparseType(type) - return super().add_option(name, shortname, default=default, type=type, group=group, help_hidden=help_hidden, - **kwargs) + return super().add_option( + name, shortname, default=default, type=type, group=group, help_hidden=help_hidden, **kwargs + ) def debug_msg(self, *args, sep=" ", **kwargs) -> None: if self._parsed_args and self._parsed_args.verbose is True: @@ -453,7 +547,8 @@ def _load_command_line_args(self) -> None: exclude=self.completion_excludes, # hide these options from the output print_suppressed=True, # also include target-specific options output_stream=output, - exit_method=sys.exit) # ensure that cprofile data is written + exit_method=sys.exit, + ) # ensure that cprofile data is written else: argcomplete.autocomplete( self._parser, @@ -470,8 +565,9 @@ def _load_command_line_args(self) -> None: for x in trailing: # filter out unknown options (like -b) # exit with error - if x.startswith('-'): + if x.startswith("-"): import difflib + # There is no officially supported API to get back all option strings, but fortunately we store # all the actions here anyway all_options = getattr(self._parser, "_option_string_actions", {}).keys() @@ -512,7 +608,7 @@ def is_needed_for_completion(self, name: str, shortname: str, option_type) -> bo return True # Okay, prefix matches shortname elif option_type is bool and (comp_prefix.startswith("--no-") or self._argcomplete_prefix_includes_slash): slash_index = name.rfind("/") - negated_name = name[:slash_index + 1] + "no-" + name[slash_index + 1:] + negated_name = name[: slash_index + 1] + "no-" + name[slash_index + 1 :] if negated_name.startswith(comp_prefix[2:]): # self.debug_msg("comp_prefix '", comp_prefix, "' matches negated option: ", negated_name, sep="") return True # Okay, prefix matches negated long name @@ -524,10 +620,16 @@ def load(self) -> None: class JsonAndCommandLineConfigLoader(CommandLineConfigLoader): - def __init__(self, argparser_class: "type[argparse.ArgumentParser]" = argparse.ArgumentParser, *, - option_cls=JsonAndCommandLineConfigOption, command_line_only_options_cls=CommandLineConfigOption): - super().__init__(argparser_class, option_cls=option_cls, - command_line_only_options_cls=command_line_only_options_cls) + def __init__( + self, + argparser_class: "type[argparse.ArgumentParser]" = argparse.ArgumentParser, + *, + option_cls=JsonAndCommandLineConfigOption, + command_line_only_options_cls=CommandLineConfigOption, + ): + super().__init__( + argparser_class, option_cls=option_cls, command_line_only_options_cls=command_line_only_options_cls + ) self._config_path: "Optional[Path]" = None # Choose the default config file based on argv[0] # This allows me to have symlinks for e.g. stable-cheribuild.py release-cheribuild.py debug-cheribuild.py @@ -535,10 +637,16 @@ def __init__(self, argparser_class: "type[argparse.ArgumentParser]" = argparse.A cheribuild_rootdir = Path(__file__).absolute().parent.parent.parent self._inferred_config_prefix = self.get_config_prefix() self.default_config_path = Path(cheribuild_rootdir, self._inferred_config_prefix + "cheribuild.json") - self.path_group.add_argument("--config-file", metavar="FILE", type=str, default=str(self.default_config_path), - action=ArgparseSetGivenAction, - help="The config file that is used to load the default settings (default: '" + - str(self.default_config_path) + "')") + self.path_group.add_argument( + "--config-file", + metavar="FILE", + type=str, + default=str(self.default_config_path), + action=ArgparseSetGivenAction, + help="The config file that is used to load the default settings (default: '" + + str(self.default_config_path) + + "')", + ) @staticmethod def get_config_prefix() -> str: @@ -546,7 +654,7 @@ def get_config_prefix() -> str: suffixes = ["cheribuild", "cheribuild.py"] for suffix in suffixes: if program.endswith(suffix): - return program[0:-len(suffix)] + return program[0 : -len(suffix)] return "" def __load_json_with_comments(self, config_path: Path) -> "dict[str, Any]": @@ -565,15 +673,24 @@ def __load_json_with_comments(self, config_path: Path) -> "dict[str, Any]": result = dict() status_update("JSON config file", config_path, "was empty.") else: - result = json.loads("".join(json_lines), - object_pairs_hook=lambda o: dict_raise_on_duplicates_and_store_src(o, config_path)) - self.debug_msg("Parsed", config_path, "as", - coloured(AnsiColour.cyan, json.dumps(result, cls=MyJsonEncoder))) + result = json.loads( + "".join(json_lines), + object_pairs_hook=lambda o: dict_raise_on_duplicates_and_store_src(o, config_path), + ) + self.debug_msg( + "Parsed", config_path, "as", coloured(AnsiColour.cyan, json.dumps(result, cls=MyJsonEncoder)) + ) return result # Based on https://stackoverflow.com/a/7205107/894271 - def merge_dict_recursive(self, a: "dict[str, _LoadedConfigValue]", b: "dict[str, _LoadedConfigValue]", - included_file: Path, base_file: Path, path=None) -> dict: + def merge_dict_recursive( + self, + a: "dict[str, _LoadedConfigValue]", + b: "dict[str, _LoadedConfigValue]", + included_file: Path, + base_file: Path, + path=None, + ) -> dict: """merges b into a""" if path is None: path = [] @@ -585,8 +702,16 @@ def merge_dict_recursive(self, a: "dict[str, _LoadedConfigValue]", b: "dict[str, self.merge_dict_recursive(a[key].value, b[key].value, included_file, base_file, [*path, str(key)]) elif a[key] != b[key]: if self._parsed_args: - self.debug_msg("Overriding '" + '.'.join([*path, str(key)]) + "' value", b[key], " from", - included_file, "with value ", a[key], "from", base_file) + self.debug_msg( + "Overriding '" + ".".join([*path, str(key)]) + "' value", + b[key], + " from", + included_file, + "with value ", + a[key], + "from", + base_file, + ) else: pass # same leaf value else: @@ -599,8 +724,9 @@ def __load_json_with_includes(self, config_path: Path): result = self.__load_json_with_comments(config_path) except Exception as e: error_message("Could not load config file ", config_path, ": ", e, sep="") - if not sys.__stdin__.isatty() or not input("Invalid config file " + str(config_path) + - ". Continue? y/[N]").lower().startswith("y"): + if not sys.__stdin__.isatty() or not input( + "Invalid config file " + str(config_path) + ". Continue? y/[N]" + ).lower().startswith("y"): raise include_value = result.get("#include") if include_value: @@ -632,31 +758,51 @@ def _load_json_config_file(self) -> None: # XXX: Ideally we would always load this file and merge the two if # both exist, with the bundled config file setting new defaults. configdir = os.getenv("XDG_CONFIG_HOME") or os.path.expanduser("~/.config") - print("Checking", Path(configdir, self._config_path.name), "since", self._config_path, "doesn't exist", - file=sys.stderr) + print( + "Checking", + Path(configdir, self._config_path.name), + "since", + self._config_path, + "doesn't exist", + file=sys.stderr, + ) self._config_path = Path(configdir, self._config_path.name) if self._inferred_config_prefix: - print(coloured(AnsiColour.green, "Note: Configuration file path inferred as"), - coloured(AnsiColour.blue, self._config_path), - coloured(AnsiColour.green, "based on command name"), - file=sys.stderr) + print( + coloured(AnsiColour.green, "Note: Configuration file path inferred as"), + coloured(AnsiColour.blue, self._config_path), + coloured(AnsiColour.green, "based on command name"), + file=sys.stderr, + ) if self._config_path.exists(): self._json = self.__load_json_with_includes(self._config_path) else: if self._inferred_config_prefix: # If the user invoked foo-cheribuild.py but foo-cheribuild.json does not exist that is almost # certainly an error. Report it as such and don't - print(coloured(AnsiColour.green, "Note: Configuration file path inferred as"), - coloured(AnsiColour.blue, self._config_path), - coloured(AnsiColour.green, "based on command name"), - file=sys.stderr) - fatal_error("Configuration file ", self._config_path, "matching prefixed command was not found.", - "If this is intended pass an explicit `--config-file=/dev/null` argument.", - pretend=False) + print( + coloured(AnsiColour.green, "Note: Configuration file path inferred as"), + coloured(AnsiColour.blue, self._config_path), + coloured(AnsiColour.green, "based on command name"), + file=sys.stderr, + ) + fatal_error( + "Configuration file ", + self._config_path, + "matching prefixed command was not found.", + "If this is intended pass an explicit `--config-file=/dev/null` argument.", + pretend=False, + ) raise FileNotFoundError(self._parsed_args.config_file) - print(coloured(AnsiColour.green, "Note: Configuration file", self._config_path, - "does not exist, using only command line arguments."), - file=sys.stderr) + print( + coloured( + AnsiColour.green, + "Note: Configuration file", + self._config_path, + "does not exist, using only command line arguments.", + ), + file=sys.stderr, + ) def load(self) -> None: super().load() diff --git a/pycheribuild/config/target_info.py b/pycheribuild/config/target_info.py index d93a8d7c5..cdc24dd08 100644 --- a/pycheribuild/config/target_info.py +++ b/pycheribuild/config/target_info.py @@ -51,7 +51,8 @@ "DefaultInstallDir", "MipsFloatAbi", "NativeTargetInfo", - "TargetInfo"] + "TargetInfo", +] class CPUArchitecture(Enum): @@ -101,6 +102,7 @@ class CompilerType(Enum): """ Used by the jenkins script to detect which compiler directory should be used """ + DEFAULT_COMPILER = "default-compiler" # Default system compiler (i.e. the argument passed to cheribuild) CHERI_LLVM = "cheri-llvm" # Compile with CHERI LLVM built by cheribuild MORELLO_LLVM = "morello-llvm" # Compile with Morello LLVM built by cheribuild @@ -120,8 +122,10 @@ def clang_flags(self) -> "list[str]": if self is None: return [] # Equivalent to -ftrivial-auto-var-init=uninitialized elif self is AutoVarInit.ZERO: - return ["-ftrivial-auto-var-init=zero", - "-enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang"] + return [ + "-ftrivial-auto-var-init=zero", + "-enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang", + ] elif self is AutoVarInit.PATTERN: return ["-ftrivial-auto-var-init=pattern"] else: @@ -152,6 +156,7 @@ class DefaultInstallDir(Enum): class AbstractProject(FileSystemUtils): """A base class for (Simple)Project that exposes only the fields/methods needed in target_info.""" + _xtarget: "ClassVar[Optional[CrossCompileTarget]]" = None default_architecture: "ClassVar[Optional[CrossCompileTarget]]" needs_sysroot: "ClassVar[bool]" @@ -184,8 +189,13 @@ def warning(*args, **kwargs) -> None: warning_message(*args, **kwargs) def fatal(self, *args, sep=" ", fixit_hint=None, fatal_when_pretending=False) -> None: - fatal_error(*args, sep=sep, fixit_hint=fixit_hint, fatal_when_pretending=fatal_when_pretending, - pretend=self.config.pretend) + fatal_error( + *args, + sep=sep, + fixit_hint=fixit_hint, + fatal_when_pretending=fatal_when_pretending, + pretend=self.config.pretend, + ) @classmethod def get_crosscompile_target(cls) -> "CrossCompileTarget": @@ -194,9 +204,12 @@ def get_crosscompile_target(cls) -> "CrossCompileTarget": return target @classmethod - def get_instance(cls: "type[_AnyProject]", caller: "Optional[AbstractProject]", - config: "Optional[CheriConfig]" = None, - cross_target: "Optional[CrossCompileTarget]" = None) -> "_AnyProject": + def get_instance( + cls: "type[_AnyProject]", + caller: "Optional[AbstractProject]", + config: "Optional[CheriConfig]" = None, + cross_target: "Optional[CrossCompileTarget]" = None, + ) -> "_AnyProject": raise NotImplementedError() @classmethod @@ -316,20 +329,34 @@ def strip_tool(self) -> Path: @classmethod @abstractmethod - def essential_compiler_and_linker_flags_impl(cls, instance: "TargetInfo", *, xtarget: "CrossCompileTarget", - perform_sanity_checks=True, default_flags_only=False, - softfloat: Optional[bool] = None) -> "list[str]": + def essential_compiler_and_linker_flags_impl( + cls, + instance: "TargetInfo", + *, + xtarget: "CrossCompileTarget", + perform_sanity_checks=True, + default_flags_only=False, + softfloat: Optional[bool] = None, + ) -> "list[str]": """ :return: flags such as -target + -mabi which are needed for both compiler and linker """ ... - def get_essential_compiler_and_linker_flags(self, xtarget: "Optional[CrossCompileTarget]" = None, - perform_sanity_checks=True, default_flags_only=False, - softfloat: Optional[bool] = None) -> "list[str]": - return self.essential_compiler_and_linker_flags_impl(self, perform_sanity_checks=perform_sanity_checks, - xtarget=xtarget if xtarget is not None else self.target, - default_flags_only=default_flags_only, softfloat=softfloat) + def get_essential_compiler_and_linker_flags( + self, + xtarget: "Optional[CrossCompileTarget]" = None, + perform_sanity_checks=True, + default_flags_only=False, + softfloat: Optional[bool] = None, + ) -> "list[str]": + return self.essential_compiler_and_linker_flags_impl( + self, + perform_sanity_checks=perform_sanity_checks, + xtarget=xtarget if xtarget is not None else self.target, + default_flags_only=default_flags_only, + softfloat=softfloat, + ) @property def additional_executable_link_flags(self) -> "list[str]": @@ -412,8 +439,9 @@ def must_link_statically(self) -> bool: return False @final - def get_rootfs_project(self, *, t: "type[_AnyProject]", caller: AbstractProject, - xtarget: "Optional[CrossCompileTarget]" = None) -> _AnyProject: + def get_rootfs_project( + self, *, t: "type[_AnyProject]", caller: AbstractProject, xtarget: "Optional[CrossCompileTarget]" = None + ) -> _AnyProject: if xtarget is None: xtarget = self.target xtarget = xtarget.get_rootfs_target() @@ -448,11 +476,20 @@ def is_freebsd(cls) -> bool: def is_cheribsd(cls) -> bool: return False - def run_cheribsd_test_script(self, script_name, *script_args, kernel_path=None, disk_image_path=None, - mount_builddir=True, mount_sourcedir=False, mount_sysroot=False, - use_full_disk_image=False, mount_installdir=False, - use_benchmark_kernel_by_default=False, - rootfs_alternate_kernel_dir=None) -> None: + def run_cheribsd_test_script( + self, + script_name, + *script_args, + kernel_path=None, + disk_image_path=None, + mount_builddir=True, + mount_sourcedir=False, + mount_sysroot=False, + use_full_disk_image=False, + mount_installdir=False, + use_benchmark_kernel_by_default=False, + rootfs_alternate_kernel_dir=None, + ) -> None: raise ValueError("run_cheribsd_test_script only supports CheriBSD targets") @classmethod @@ -512,8 +549,11 @@ def host_c_preprocessor(config: CheriConfig) -> Path: def pkgconfig_candidates(self, prefix: Path) -> "list[str]": """:return: a list of potential candidates for pkgconfig .pc files inside prefix""" - return [str(prefix / self.default_libdir / "pkgconfig"), str(prefix / "share/pkgconfig"), - str(prefix / "libdata/pkgconfig")] + return [ + str(prefix / self.default_libdir / "pkgconfig"), + str(prefix / "share/pkgconfig"), + str(prefix / "libdata/pkgconfig"), + ] class NativeTargetInfo(TargetInfo): @@ -646,8 +686,9 @@ def _compat_abi_suffix(self) -> str: # Directory suffix for compat ABI (currently only "64"/"" should be valid) if _is_native_purecap() and not self.target.is_cheri_purecap(): return "64" - assert _is_native_purecap() == self.target.is_cheri_purecap(), \ - "Building purecap natively is only supported on purecap installations" + assert ( + _is_native_purecap() == self.target.is_cheri_purecap() + ), "Building purecap natively is only supported on purecap installations" return "" @property @@ -689,9 +730,15 @@ def pkgconfig_candidates(self, prefix: Path) -> "list[str]": return result @classmethod - def essential_compiler_and_linker_flags_impl(cls, instance: "TargetInfo", *, xtarget: "CrossCompileTarget", - perform_sanity_checks=True, default_flags_only=False, - softfloat: Optional[bool] = None) -> "list[str]": + def essential_compiler_and_linker_flags_impl( + cls, + instance: "TargetInfo", + *, + xtarget: "CrossCompileTarget", + perform_sanity_checks=True, + default_flags_only=False, + softfloat: Optional[bool] = None, + ) -> "list[str]": result = [] if instance.project.auto_var_init != AutoVarInit.NONE: compiler = instance.project.get_compiler_info(instance.c_compiler) @@ -704,8 +751,9 @@ def essential_compiler_and_linker_flags_impl(cls, instance: "TargetInfo", *, xta if valid_clang_version: result += instance.project.auto_var_init.clang_flags() else: - instance.project.fatal("Requested automatic variable initialization, but don't know how to for", - compiler) + instance.project.fatal( + "Requested automatic variable initialization, but don't know how to for", compiler + ) if xtarget.is_cheri_hybrid(): if xtarget.is_aarch64(include_purecap=True): result.append("-mabi=aapcs") @@ -718,17 +766,25 @@ class CrossCompileTarget: # Currently the same for all targets DEFAULT_SUBOBJECT_BOUNDS: str = "conservative" - def __init__(self, arch_suffix: str, cpu_architecture: CPUArchitecture, target_info_cls: "type[TargetInfo]", - *, is_cheri_purecap=False, is_cheri_hybrid=False, extra_target_suffix: str = "", - check_conflict_with: "Optional[CrossCompileTarget]" = None, - rootfs_target: "Optional[CrossCompileTarget]" = None, - non_cheri_target: "Optional[CrossCompileTarget]" = None, - hybrid_target: "Optional[CrossCompileTarget]" = None, - purecap_target: "Optional[CrossCompileTarget]" = None, - non_cheri_for_hybrid_rootfs_target: "Optional[CrossCompileTarget]" = None, - non_cheri_for_purecap_rootfs_target: "Optional[CrossCompileTarget]" = None, - hybrid_for_purecap_rootfs_target: "Optional[CrossCompileTarget]" = None, - purecap_for_hybrid_rootfs_target: "Optional[CrossCompileTarget]" = None) -> None: + def __init__( + self, + arch_suffix: str, + cpu_architecture: CPUArchitecture, + target_info_cls: "type[TargetInfo]", + *, + is_cheri_purecap=False, + is_cheri_hybrid=False, + extra_target_suffix: str = "", + check_conflict_with: "Optional[CrossCompileTarget]" = None, + rootfs_target: "Optional[CrossCompileTarget]" = None, + non_cheri_target: "Optional[CrossCompileTarget]" = None, + hybrid_target: "Optional[CrossCompileTarget]" = None, + purecap_target: "Optional[CrossCompileTarget]" = None, + non_cheri_for_hybrid_rootfs_target: "Optional[CrossCompileTarget]" = None, + non_cheri_for_purecap_rootfs_target: "Optional[CrossCompileTarget]" = None, + hybrid_for_purecap_rootfs_target: "Optional[CrossCompileTarget]" = None, + purecap_for_hybrid_rootfs_target: "Optional[CrossCompileTarget]" = None, + ) -> None: assert not arch_suffix.startswith("-"), arch_suffix assert not extra_target_suffix or extra_target_suffix.startswith("-"), extra_target_suffix name_prefix = target_info_cls.shortname @@ -774,9 +830,15 @@ def __init__(self, arch_suffix: str, cpu_architecture: CPUArchitecture, target_i def _set_from(self, other_target: "CrossCompileTarget") -> None: if self is other_target: return - for attr in ("_hybrid_target", "_non_cheri_target", "_purecap_target", "_non_cheri_for_hybrid_rootfs_target", - "_non_cheri_for_purecap_rootfs_target", "_hybrid_for_purecap_rootfs_target", - "_purecap_for_hybrid_rootfs_target"): + for attr in ( + "_hybrid_target", + "_non_cheri_target", + "_purecap_target", + "_non_cheri_for_hybrid_rootfs_target", + "_non_cheri_for_purecap_rootfs_target", + "_hybrid_for_purecap_rootfs_target", + "_purecap_for_hybrid_rootfs_target", + ): if getattr(self, attr) is None and getattr(other_target, attr) is not None: setattr(self, attr, getattr(other_target, attr)) # noinspection PyProtectedMember @@ -787,10 +849,13 @@ def _set_for(self, other_target: "Optional[CrossCompileTarget]", also_set_other= if other_target is not None and self is not other_target: if self._is_cheri_hybrid: if self._rootfs_target is not None: - assert self._rootfs_target._is_cheri_purecap, \ - "Only support purecap separate rootfs for hybrid targets" - assert other_target._hybrid_for_purecap_rootfs_target is None or \ - other_target._hybrid_for_purecap_rootfs_target is self, "Already set?" + assert ( + self._rootfs_target._is_cheri_purecap + ), "Only support purecap separate rootfs for hybrid targets" + assert ( + other_target._hybrid_for_purecap_rootfs_target is None + or other_target._hybrid_for_purecap_rootfs_target is self + ), "Already set?" other_target._hybrid_for_purecap_rootfs_target = self self._hybrid_for_purecap_rootfs_target = self else: @@ -799,10 +864,13 @@ def _set_for(self, other_target: "Optional[CrossCompileTarget]", also_set_other= self._hybrid_target = self elif self._is_cheri_purecap: if self._rootfs_target is not None: - assert self._rootfs_target._is_cheri_hybrid, \ - "Only support hybrid separate rootfs for purecap targets" - assert other_target._purecap_for_hybrid_rootfs_target is None or \ - other_target._purecap_for_hybrid_rootfs_target is self, "Already set?" + assert ( + self._rootfs_target._is_cheri_hybrid + ), "Only support hybrid separate rootfs for purecap targets" + assert ( + other_target._purecap_for_hybrid_rootfs_target is None + or other_target._purecap_for_hybrid_rootfs_target is self + ), "Already set?" other_target._purecap_for_hybrid_rootfs_target = self self._purecap_for_hybrid_rootfs_target = self else: @@ -812,20 +880,25 @@ def _set_for(self, other_target: "Optional[CrossCompileTarget]", also_set_other= else: if self._rootfs_target is not None: if self._rootfs_target._is_cheri_hybrid: - assert other_target._non_cheri_for_hybrid_rootfs_target is None or \ - other_target._non_cheri_for_hybrid_rootfs_target is self, "Already set?" + assert ( + other_target._non_cheri_for_hybrid_rootfs_target is None + or other_target._non_cheri_for_hybrid_rootfs_target is self + ), "Already set?" other_target._non_cheri_for_hybrid_rootfs_target = self self._non_cheri_for_hybrid_rootfs_target = self else: assert self._rootfs_target._is_cheri_purecap, "Separate non-CHERI rootfs for non-CHERI target?" - assert other_target._non_cheri_for_purecap_rootfs_target is None or \ - other_target._non_cheri_for_purecap_rootfs_target is self, "Already set?" + assert ( + other_target._non_cheri_for_purecap_rootfs_target is None + or other_target._non_cheri_for_purecap_rootfs_target is self + ), "Already set?" other_target._non_cheri_for_purecap_rootfs_target = self self._non_cheri_for_purecap_rootfs_target = self else: assert self._rootfs_target is None, "Separate rootfs targets only supported for CHERI targets" - assert other_target._non_cheri_target is None or other_target._non_cheri_target is self, \ - "Already set?" + assert ( + other_target._non_cheri_target is None or other_target._non_cheri_target is self + ), "Already set?" other_target._non_cheri_target = self self._non_cheri_target = self if also_set_other: @@ -962,16 +1035,24 @@ def get_non_cheri_target(self) -> "CrossCompileTarget": raise ValueError("Don't know non-CHERI version of " + repr(self)) def get_non_cheri_for_hybrid_rootfs_target(self) -> "CrossCompileTarget": - if not self._is_cheri_purecap and not self._is_cheri_hybrid and self._rootfs_target is not None and \ - self._rootfs_target._is_cheri_hybrid: + if ( + not self._is_cheri_purecap + and not self._is_cheri_hybrid + and self._rootfs_target is not None + and self._rootfs_target._is_cheri_hybrid + ): return self elif self._non_cheri_for_hybrid_rootfs_target is not None: return self._non_cheri_for_hybrid_rootfs_target raise ValueError("Don't know non-CHERI for hybrid rootfs version of " + repr(self)) def get_non_cheri_for_purecap_rootfs_target(self) -> "CrossCompileTarget": - if not self._is_cheri_purecap and not self._is_cheri_hybrid and self._rootfs_target is not None and \ - self._rootfs_target._is_cheri_purecap: + if ( + not self._is_cheri_purecap + and not self._is_cheri_hybrid + and self._rootfs_target is not None + and self._rootfs_target._is_cheri_purecap + ): return self elif self._non_cheri_for_purecap_rootfs_target is not None: return self._non_cheri_for_purecap_rootfs_target @@ -1006,9 +1087,16 @@ def __repr__(self) -> str: def _dump_target_relations(self) -> None: self_repr = repr(self) - for n in ('non_cheri', 'hybrid', 'purecap', 'non_cheri_for_hybrid_rootfs', 'non_cheri_for_purecap_rootfs', - 'hybrid_for_purecap_rootfs', 'purecap_for_hybrid_rootfs'): - k = '_' + n + '_target' + for n in ( + "non_cheri", + "hybrid", + "purecap", + "non_cheri_for_hybrid_rootfs", + "non_cheri_for_purecap_rootfs", + "hybrid_for_purecap_rootfs", + "purecap_for_hybrid_rootfs", + ): + k = "_" + n + "_target" v = self.__dict__[k] print(self_repr + "." + n + ": " + repr(v)) @@ -1041,14 +1129,20 @@ class BasicCompilationTargets: # Some projects (LLVM, QEMU, GDB, etc.) don't build as purecap binaries, so we have to build them hybrid instead. if _is_native_purecap(): NATIVE = CrossCompileTarget("native", _native_cpu_arch(), NativeTargetInfo, is_cheri_purecap=True) - NATIVE_HYBRID = CrossCompileTarget("native-hybrid", _native_cpu_arch(), NativeTargetInfo, is_cheri_hybrid=True, - purecap_target=NATIVE, check_conflict_with=NATIVE) + NATIVE_HYBRID = CrossCompileTarget( + "native-hybrid", + _native_cpu_arch(), + NativeTargetInfo, + is_cheri_hybrid=True, + purecap_target=NATIVE, + check_conflict_with=NATIVE, + ) NATIVE_NON_PURECAP = NATIVE_HYBRID ALL_NATIVE = (NATIVE, NATIVE_HYBRID) else: NATIVE = CrossCompileTarget("native", _native_cpu_arch(), NativeTargetInfo) NATIVE_NON_PURECAP = NATIVE - ALL_NATIVE = (NATIVE, ) + ALL_NATIVE = (NATIVE,) NATIVE_IF_FREEBSD = ALL_NATIVE if OSInfo.IS_FREEBSD else tuple() NATIVE_IF_LINUX = ALL_NATIVE if OSInfo.IS_LINUX else tuple() NATIVE_IF_MACOS = ALL_NATIVE if OSInfo.IS_MAC else tuple() diff --git a/pycheribuild/targets.py b/pycheribuild/targets.py index 5144b94f4..3fb0dcdcc 100644 --- a/pycheribuild/targets.py +++ b/pycheribuild/targets.py @@ -429,7 +429,7 @@ def target_disabled_reason(target: Target, config: CheriConfig) -> Optional[str] ): return ( f"{target.name} is a hybrid target, which should not be used unless you know what you're doing. " - f"If you are still sure you want to build this, use --enable-hybrid-targets." + "If you are still sure you want to build this, use --enable-hybrid-targets." ) return None @@ -482,7 +482,7 @@ def get_target( target = target.get_real_target(arch_for_unqualified_targets, config, caller=caller) if required_arch is not None and target.xtarget != required_arch: raise LookupError( - f"Target {target.name} has wrong architecture:" f"{target.xtarget} but expected {required_arch}" + f"Target {target.name} has wrong architecture: {target.xtarget} but expected {required_arch}" ) # print(" ->", target) return target diff --git a/pycheribuild/utils.py b/pycheribuild/utils.py index aaf677685..91a871fdb 100644 --- a/pycheribuild/utils.py +++ b/pycheribuild/utils.py @@ -165,7 +165,7 @@ def __set_name__(self, _, name) -> None: # XXX: requires python 3.6 self.attrname = name elif name != self.attrname: raise TypeError( - "Cannot assign the same cached_property to two different names " f"({self.attrname} and {name})." + f"Cannot assign the same cached_property to two different names ({self.attrname} and {name})." ) def __get__(self, instance, owner=None) -> Type_T: @@ -529,10 +529,7 @@ def command_not_found(): if msg_start: hint = hint[msg_start:] return hint - return ( - "Could not find package for program " + name + ". " - "Maybe `zypper in " + name + "` will work." - ) + return f"Could not find package for program {name}. Maybe `zypper in {name}` will work." return InstallInstructions(command_not_found, cheribuild_target, alternative) guessed_package = True @@ -548,7 +545,7 @@ def command_not_found(): # not sure if the package name is correct: return InstallInstructions( f"Possibly running `{cls.package_manager(compat_abi)} install {install_name}" - f"` fixes this. Note: package name may not be correct.", + "` fixes this. Note: package name may not be correct.", cheribuild_target, alternative, ) diff --git a/tests/test_argument_parsing.py b/tests/test_argument_parsing.py index b7f6a01a9..6da3e8fcd 100644 --- a/tests/test_argument_parsing.py +++ b/tests/test_argument_parsing.py @@ -937,8 +937,12 @@ def test_kernel_configs(target, config_options: "list[str]", expected_kernels: " pytest.param( "cheribsd-mfs-root-kernel-riscv64-purecap", ["--cheribsd/build-nocaprevoke-kernel"], - ["CHERI-QEMU-MFS-ROOT", "CHERI-NOCAPREVOKE-QEMU-MFS-ROOT", "CHERI-PURECAP-QEMU-MFS-ROOT", - "CHERI-PURECAP-NOCAPREVOKE-QEMU-MFS-ROOT"], + [ + "CHERI-QEMU-MFS-ROOT", + "CHERI-NOCAPREVOKE-QEMU-MFS-ROOT", + "CHERI-PURECAP-QEMU-MFS-ROOT", + "CHERI-PURECAP-NOCAPREVOKE-QEMU-MFS-ROOT", + ], ), ], ) @@ -1404,7 +1408,9 @@ def test_jenkins_hack_disk_image(): config = _parse_arguments(args) jenkins_override_install_dirs_hack(config, Path("/rootfs")) disk_image = _get_target_instance( - "disk-image-aarch64", config, BuildCheriBSDDiskImage, + "disk-image-aarch64", + config, + BuildCheriBSDDiskImage, ) assert disk_image.disk_image_path == Path("/tmp/tarball/cheribsd-aarch64.img") assert disk_image.rootfs_dir == Path("/tmp/tarball/rootfs") diff --git a/tests/test_metalog.py b/tests/test_metalog.py index 45609c265..19a121d9c 100644 --- a/tests/test_metalog.py +++ b/tests/test_metalog.py @@ -207,13 +207,16 @@ def test_contents_root(): # END """ mtree = MtreeFile(file=io.StringIO(file), contents_root=Path("/path/to/rootfs"), verbose=False) - assert _get_as_str(mtree) == """#mtree 2.0 + assert ( + _get_as_str(mtree) + == """#mtree 2.0 . type=dir uname=root gname=wheel mode=0755 ./bin type=dir uname=root gname=wheel mode=0755 ./bin/cat type=file uname=root gname=wheel mode=0755 contents=/path/to/rootfs/bin/cheribsdbox ./bin/cheribsdbox type=file uname=root gname=wheel mode=0755 contents=/path/to/rootfs/bin/cheribsdbox # END """ + ) def test_add_file():