185 lines
6.2 KiB
Python
185 lines
6.2 KiB
Python
|
"""Utilities for mypy.stubgen, mypy.stubgenc, and mypy.stubdoc modules."""
|
||
|
|
||
|
from __future__ import annotations
|
||
|
|
||
|
import os.path
|
||
|
import re
|
||
|
import sys
|
||
|
from contextlib import contextmanager
|
||
|
from typing import Iterator
|
||
|
from typing_extensions import overload
|
||
|
|
||
|
from mypy.modulefinder import ModuleNotFoundReason
|
||
|
from mypy.moduleinspect import InspectError, ModuleInspect
|
||
|
|
||
|
# Modules that may fail when imported, or that may have side effects (fully qualified).
|
||
|
NOT_IMPORTABLE_MODULES = ()
|
||
|
|
||
|
|
||
|
class CantImport(Exception):
|
||
|
def __init__(self, module: str, message: str):
|
||
|
self.module = module
|
||
|
self.message = message
|
||
|
|
||
|
|
||
|
def walk_packages(
|
||
|
inspect: ModuleInspect, packages: list[str], verbose: bool = False
|
||
|
) -> Iterator[str]:
|
||
|
"""Iterates through all packages and sub-packages in the given list.
|
||
|
|
||
|
This uses runtime imports (in another process) to find both Python and C modules.
|
||
|
For Python packages we simply pass the __path__ attribute to pkgutil.walk_packages() to
|
||
|
get the content of the package (all subpackages and modules). However, packages in C
|
||
|
extensions do not have this attribute, so we have to roll out our own logic: recursively
|
||
|
find all modules imported in the package that have matching names.
|
||
|
"""
|
||
|
for package_name in packages:
|
||
|
if package_name in NOT_IMPORTABLE_MODULES:
|
||
|
print(f"{package_name}: Skipped (blacklisted)")
|
||
|
continue
|
||
|
if verbose:
|
||
|
print(f"Trying to import {package_name!r} for runtime introspection")
|
||
|
try:
|
||
|
prop = inspect.get_package_properties(package_name)
|
||
|
except InspectError:
|
||
|
report_missing(package_name)
|
||
|
continue
|
||
|
yield prop.name
|
||
|
if prop.is_c_module:
|
||
|
# Recursively iterate through the subpackages
|
||
|
yield from walk_packages(inspect, prop.subpackages, verbose)
|
||
|
else:
|
||
|
yield from prop.subpackages
|
||
|
|
||
|
|
||
|
def find_module_path_using_sys_path(module: str, sys_path: list[str]) -> str | None:
|
||
|
relative_candidates = (
|
||
|
module.replace(".", "/") + ".py",
|
||
|
os.path.join(module.replace(".", "/"), "__init__.py"),
|
||
|
)
|
||
|
for base in sys_path:
|
||
|
for relative_path in relative_candidates:
|
||
|
path = os.path.join(base, relative_path)
|
||
|
if os.path.isfile(path):
|
||
|
return path
|
||
|
return None
|
||
|
|
||
|
|
||
|
def find_module_path_and_all_py3(
|
||
|
inspect: ModuleInspect, module: str, verbose: bool
|
||
|
) -> tuple[str | None, list[str] | None] | None:
|
||
|
"""Find module and determine __all__ for a Python 3 module.
|
||
|
|
||
|
Return None if the module is a C module. Return (module_path, __all__) if
|
||
|
it is a Python module. Raise CantImport if import failed.
|
||
|
"""
|
||
|
if module in NOT_IMPORTABLE_MODULES:
|
||
|
raise CantImport(module, "")
|
||
|
|
||
|
# TODO: Support custom interpreters.
|
||
|
if verbose:
|
||
|
print(f"Trying to import {module!r} for runtime introspection")
|
||
|
try:
|
||
|
mod = inspect.get_package_properties(module)
|
||
|
except InspectError as e:
|
||
|
# Fall back to finding the module using sys.path.
|
||
|
path = find_module_path_using_sys_path(module, sys.path)
|
||
|
if path is None:
|
||
|
raise CantImport(module, str(e)) from e
|
||
|
return path, None
|
||
|
if mod.is_c_module:
|
||
|
return None
|
||
|
return mod.file, mod.all
|
||
|
|
||
|
|
||
|
@contextmanager
|
||
|
def generate_guarded(
|
||
|
mod: str, target: str, ignore_errors: bool = True, verbose: bool = False
|
||
|
) -> Iterator[None]:
|
||
|
"""Ignore or report errors during stub generation.
|
||
|
|
||
|
Optionally report success.
|
||
|
"""
|
||
|
if verbose:
|
||
|
print(f"Processing {mod}")
|
||
|
try:
|
||
|
yield
|
||
|
except Exception as e:
|
||
|
if not ignore_errors:
|
||
|
raise e
|
||
|
else:
|
||
|
# --ignore-errors was passed
|
||
|
print("Stub generation failed for", mod, file=sys.stderr)
|
||
|
else:
|
||
|
if verbose:
|
||
|
print(f"Created {target}")
|
||
|
|
||
|
|
||
|
def report_missing(mod: str, message: str | None = "", traceback: str = "") -> None:
|
||
|
if message:
|
||
|
message = " with error: " + message
|
||
|
print(f"{mod}: Failed to import, skipping{message}")
|
||
|
|
||
|
|
||
|
def fail_missing(mod: str, reason: ModuleNotFoundReason) -> None:
|
||
|
if reason is ModuleNotFoundReason.NOT_FOUND:
|
||
|
clarification = "(consider using --search-path)"
|
||
|
elif reason is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS:
|
||
|
clarification = "(module likely exists, but is not PEP 561 compatible)"
|
||
|
else:
|
||
|
clarification = f"(unknown reason '{reason}')"
|
||
|
raise SystemExit(f"Can't find module '{mod}' {clarification}")
|
||
|
|
||
|
|
||
|
@overload
|
||
|
def remove_misplaced_type_comments(source: bytes) -> bytes:
|
||
|
...
|
||
|
|
||
|
|
||
|
@overload
|
||
|
def remove_misplaced_type_comments(source: str) -> str:
|
||
|
...
|
||
|
|
||
|
|
||
|
def remove_misplaced_type_comments(source: str | bytes) -> str | bytes:
|
||
|
"""Remove comments from source that could be understood as misplaced type comments.
|
||
|
|
||
|
Normal comments may look like misplaced type comments, and since they cause blocking
|
||
|
parse errors, we want to avoid them.
|
||
|
"""
|
||
|
if isinstance(source, bytes):
|
||
|
# This gives us a 1-1 character code mapping, so it's roundtrippable.
|
||
|
text = source.decode("latin1")
|
||
|
else:
|
||
|
text = source
|
||
|
|
||
|
# Remove something that looks like a variable type comment but that's by itself
|
||
|
# on a line, as it will often generate a parse error (unless it's # type: ignore).
|
||
|
text = re.sub(r'^[ \t]*# +type: +["\'a-zA-Z_].*$', "", text, flags=re.MULTILINE)
|
||
|
|
||
|
# Remove something that looks like a function type comment after docstring,
|
||
|
# which will result in a parse error.
|
||
|
text = re.sub(r'""" *\n[ \t\n]*# +type: +\(.*$', '"""\n', text, flags=re.MULTILINE)
|
||
|
text = re.sub(r"''' *\n[ \t\n]*# +type: +\(.*$", "'''\n", text, flags=re.MULTILINE)
|
||
|
|
||
|
# Remove something that looks like a badly formed function type comment.
|
||
|
text = re.sub(r"^[ \t]*# +type: +\([^()]+(\)[ \t]*)?$", "", text, flags=re.MULTILINE)
|
||
|
|
||
|
if isinstance(source, bytes):
|
||
|
return text.encode("latin1")
|
||
|
else:
|
||
|
return text
|
||
|
|
||
|
|
||
|
def common_dir_prefix(paths: list[str]) -> str:
|
||
|
if not paths:
|
||
|
return "."
|
||
|
cur = os.path.dirname(os.path.normpath(paths[0]))
|
||
|
for path in paths[1:]:
|
||
|
while True:
|
||
|
path = os.path.dirname(os.path.normpath(path))
|
||
|
if (cur + os.sep).startswith(path + os.sep):
|
||
|
cur = path
|
||
|
break
|
||
|
return cur or "."
|