From d7b24514d7301f86031b7d1e2215cf8c2476bec0 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 27 Aug 2023 23:20:13 +0100 Subject: [PATCH] Fixes to stubtest's new check for missing stdlib modules (#15960) - It's not easy to predict where stdlib modules are going to be located. (It varies between platforms, and between venvs and conda envs; on some platforms it's in a completely different directory to the Python executable.) - Some modules appear to raise `SystemExit` when stubtest tries to import them in CI, leading stubtest to instantly exit without logging a message to the terminal. - Importing some `test.*` submodules leads to unraisable exceptions being printed to the terminal at the end of the stubtest run, which is somewhat annoying. --- mypy/stubtest.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 34bb985b702e..a804835a632b 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -23,6 +23,7 @@ import typing import typing_extensions import warnings +from collections import defaultdict from contextlib import redirect_stderr, redirect_stdout from functools import singledispatch from pathlib import Path @@ -1679,16 +1680,22 @@ def get_importable_stdlib_modules() -> set[str]: all_stdlib_modules = sys.stdlib_module_names else: all_stdlib_modules = set(sys.builtin_module_names) - python_exe_dir = Path(sys.executable).parent + modules_by_finder: defaultdict[importlib.machinery.FileFinder, set[str]] = defaultdict(set) for m in pkgutil.iter_modules(): - finder = m.module_finder - if isinstance(finder, importlib.machinery.FileFinder): - finder_path = Path(finder.path) - if ( - python_exe_dir in finder_path.parents - and "site-packages" not in finder_path.parts - ): - all_stdlib_modules.add(m.name) + if isinstance(m.module_finder, importlib.machinery.FileFinder): + modules_by_finder[m.module_finder].add(m.name) + for finder, module_group in modules_by_finder.items(): + if ( + "site-packages" not in Path(finder.path).parents + # if "_queue" is present, it's most likely the module finder + # for stdlib extension modules; + # if "queue" is present, it's most likely the module finder + # for pure-Python stdlib modules. + # In either case, we'll want to add all the modules that the finder has to offer us. + # This is a bit hacky, but seems to work well in a cross-platform way. + and {"_queue", "queue"} & module_group + ): + all_stdlib_modules.update(module_group) importable_stdlib_modules: set[str] = set() for module_name in all_stdlib_modules: @@ -1719,13 +1726,25 @@ def get_importable_stdlib_modules() -> set[str]: # The idlelib.* submodules are similarly annoying in opening random tkinter windows, # and we're unlikely to ever add stubs for idlelib in typeshed # (see discussion in https://github.com/python/typeshed/pull/9193) - if submodule_name.endswith(".__main__") or submodule_name.startswith("idlelib."): + # + # test.* modules do weird things like raising exceptions in __del__ methods, + # leading to unraisable exceptions being logged to the terminal + # as a warning at the end of the stubtest run + if ( + submodule_name.endswith(".__main__") + or submodule_name.startswith("idlelib.") + or submodule_name.startswith("test.") + ): continue try: silent_import_module(submodule_name) + except KeyboardInterrupt: + raise # importing multiprocessing.popen_forkserver on Windows raises AttributeError... - except Exception: + # some submodules also appear to raise SystemExit as well on some Python versions + # (not sure exactly which) + except BaseException: continue else: importable_stdlib_modules.add(submodule_name)