py_task/venv/lib/python3.9/site-packages/mypy/stubgenc.py

#!/usr/bin/env python3
"""Stub generator for C modules.

The public interface is via the mypy.stubgen module.
"""

from __future__ import annotations

import importlib
import inspect
import os.path
import re
from abc import abstractmethod
from types import ModuleType
from typing import Any, Iterable, Mapping
from typing_extensions import Final

from mypy.moduleinspect import is_c_module
from mypy.stubdoc import (
    ArgSig,
    FunctionSig,
    infer_arg_sig_from_anon_docstring,
    infer_prop_type_from_docstring,
    infer_ret_type_sig_from_anon_docstring,
    infer_ret_type_sig_from_docstring,
    infer_sig_from_docstring,
)

# Members of the typing module to consider for importing by default.
_DEFAULT_TYPING_IMPORTS: Final = (
    "Any",
    "Callable",
    "ClassVar",
    "Dict",
    "Iterable",
    "Iterator",
    "List",
    "Optional",
    "Tuple",
    "Union",
)


class SignatureGenerator:
    """Abstract base class for extracting a list of FunctionSigs for each function."""

    def remove_self_type(
        self, inferred: list[FunctionSig] | None, self_var: str
    ) -> list[FunctionSig] | None:
        """Remove type annotation from self/cls argument"""
        if inferred:
            for signature in inferred:
                if signature.args:
                    if signature.args[0].name == self_var:
                        signature.args[0].type = None
        return inferred

    @abstractmethod
    def get_function_sig(
        self, func: object, module_name: str, name: str
    ) -> list[FunctionSig] | None:
        pass

    @abstractmethod
    def get_method_sig(
        self, cls: type, func: object, module_name: str, class_name: str, name: str, self_var: str
    ) -> list[FunctionSig] | None:
        pass


class ExternalSignatureGenerator(SignatureGenerator):
    def __init__(
        self, func_sigs: dict[str, str] | None = None, class_sigs: dict[str, str] | None = None
    ):
        """
        Takes a mapping of function/method names to signatures and class name to
        class signatures (usually corresponds to __init__).
        """
        self.func_sigs = func_sigs or {}
        self.class_sigs = class_sigs or {}

    def get_function_sig(
        self, func: object, module_name: str, name: str
    ) -> list[FunctionSig] | None:
        if name in self.func_sigs:
            return [
                FunctionSig(
                    name=name,
                    args=infer_arg_sig_from_anon_docstring(self.func_sigs[name]),
                    ret_type="Any",
                )
            ]
        else:
            return None

    def get_method_sig(
        self, cls: type, func: object, module_name: str, class_name: str, name: str, self_var: str
    ) -> list[FunctionSig] | None:
        if (
            name in ("__new__", "__init__")
            and name not in self.func_sigs
            and class_name in self.class_sigs
        ):
            return [
                FunctionSig(
                    name=name,
                    args=infer_arg_sig_from_anon_docstring(self.class_sigs[class_name]),
                    ret_type=infer_method_ret_type(name),
                )
            ]
        inferred = self.get_function_sig(func, module_name, name)
        return self.remove_self_type(inferred, self_var)


class DocstringSignatureGenerator(SignatureGenerator):
    def get_function_sig(
        self, func: object, module_name: str, name: str
    ) -> list[FunctionSig] | None:
        docstr = getattr(func, "__doc__", None)
        inferred = infer_sig_from_docstring(docstr, name)
        if inferred:
            assert docstr is not None
            if is_pybind11_overloaded_function_docstring(docstr, name):
                # Remove pybind11 umbrella (*args, **kwargs) for overloaded functions
                del inferred[-1]
        return inferred

    def get_method_sig(
        self,
        cls: type,
        func: object,
        module_name: str,
        class_name: str,
        func_name: str,
        self_var: str,
    ) -> list[FunctionSig] | None:
        inferred = self.get_function_sig(func, module_name, func_name)
        if not inferred and func_name == "__init__":
            # look for class-level constructor signatures of the form <class_name>(<signature>)
            inferred = self.get_function_sig(cls, module_name, class_name)
        return self.remove_self_type(inferred, self_var)


class FallbackSignatureGenerator(SignatureGenerator):
    def get_function_sig(
        self, func: object, module_name: str, name: str
    ) -> list[FunctionSig] | None:
        return [
            FunctionSig(
                name=name,
                args=infer_arg_sig_from_anon_docstring("(*args, **kwargs)"),
                ret_type="Any",
            )
        ]

    def get_method_sig(
        self, cls: type, func: object, module_name: str, class_name: str, name: str, self_var: str
    ) -> list[FunctionSig] | None:
        return [
            FunctionSig(
                name=name,
                args=infer_method_args(name, self_var),
                ret_type=infer_method_ret_type(name),
            )
        ]


def generate_stub_for_c_module(
    module_name: str,
    target: str,
    known_modules: list[str],
    sig_generators: Iterable[SignatureGenerator],
) -> None:
    """Generate stub for C module.

    Signature generators are called in order until a list of signatures is returned.  The order
    is:
    - signatures inferred from .rst documentation (if given)
    - simple runtime introspection (looking for docstrings and attributes
      with simple builtin types)
    - fallback based special method names or "(*args, **kwargs)"

    If directory for target doesn't exist it will be created. Existing stub
    will be overwritten.
    """
    module = importlib.import_module(module_name)
    assert is_c_module(module), f"{module_name} is not a C module"
    subdir = os.path.dirname(target)
    if subdir and not os.path.isdir(subdir):
        os.makedirs(subdir)
    imports: list[str] = []
    functions: list[str] = []
    done = set()
    items = sorted(get_members(module), key=lambda x: x[0])
    for name, obj in items:
        if is_c_function(obj):
            generate_c_function_stub(
                module,
                name,
                obj,
                output=functions,
                known_modules=known_modules,
                imports=imports,
                sig_generators=sig_generators,
            )
            done.add(name)
    types: list[str] = []
    for name, obj in items:
        if name.startswith("__") and name.endswith("__"):
            continue
        if is_c_type(obj):
            generate_c_type_stub(
                module,
                name,
                obj,
                output=types,
                known_modules=known_modules,
                imports=imports,
                sig_generators=sig_generators,
            )
            done.add(name)
    variables = []
    for name, obj in items:
        if name.startswith("__") and name.endswith("__"):
            continue
        if name not in done and not inspect.ismodule(obj):
            type_str = strip_or_import(
                get_type_fullname(type(obj)), module, known_modules, imports
            )
            variables.append(f"{name}: {type_str}")
    output = sorted(set(imports))
    for line in variables:
        output.append(line)
    for line in types:
        if line.startswith("class") and output and output[-1]:
            output.append("")
        output.append(line)
    if output and functions:
        output.append("")
    for line in functions:
        output.append(line)
    output = add_typing_import(output)
    with open(target, "w") as file:
        for line in output:
            file.write(f"{line}\n")


def add_typing_import(output: list[str]) -> list[str]:
    """Add typing imports for collections/types that occur in the generated stub."""
    names = []
    for name in _DEFAULT_TYPING_IMPORTS:
        if any(re.search(r"\b%s\b" % name, line) for line in output):
            names.append(name)
    if names:
        return [f"from typing import {', '.join(names)}", ""] + output
    else:
        return output.copy()


def get_members(obj: object) -> list[tuple[str, Any]]:
    obj_dict: Mapping[str, Any] = getattr(obj, "__dict__")  # noqa: B009
    results = []
    for name in obj_dict:
        if is_skipped_attribute(name):
            continue
        # Try to get the value via getattr
        try:
            value = getattr(obj, name)
        except AttributeError:
            continue
        else:
            results.append((name, value))
    return results


def is_c_function(obj: object) -> bool:
    return inspect.isbuiltin(obj) or type(obj) is type(ord)


def is_c_method(obj: object) -> bool:
    return inspect.ismethoddescriptor(obj) or type(obj) in (
        type(str.index),
        type(str.__add__),
        type(str.__new__),
    )


def is_c_classmethod(obj: object) -> bool:
    return inspect.isbuiltin(obj) or type(obj).__name__ in (
        "classmethod",
        "classmethod_descriptor",
    )


def is_c_property(obj: object) -> bool:
    return inspect.isdatadescriptor(obj) or hasattr(obj, "fget")


def is_c_property_readonly(prop: Any) -> bool:
    return hasattr(prop, "fset") and prop.fset is None


def is_c_type(obj: object) -> bool:
    return inspect.isclass(obj) or type(obj) is type(int)


def is_pybind11_overloaded_function_docstring(docstr: str, name: str) -> bool:
    return docstr.startswith(f"{name}(*args, **kwargs)\n" + "Overloaded function.\n\n")


def generate_c_function_stub(
    module: ModuleType,
    name: str,
    obj: object,
    *,
    known_modules: list[str],
    sig_generators: Iterable[SignatureGenerator],
    output: list[str],
    imports: list[str],
    self_var: str | None = None,
    cls: type | None = None,
    class_name: str | None = None,
) -> None:
    """Generate stub for a single function or method.

    The result (always a single line) will be appended to 'output'.
    If necessary, any required names will be added to 'imports'.
    The 'class_name' is used to find signature of __init__ or __new__ in
    'class_sigs'.
    """
    inferred: list[FunctionSig] | None = None
    if class_name:
        # method:
        assert cls is not None, "cls should be provided for methods"
        assert self_var is not None, "self_var should be provided for methods"
        for sig_gen in sig_generators:
            inferred = sig_gen.get_method_sig(
                cls, obj, module.__name__, class_name, name, self_var
            )
            if inferred:
                # add self/cls var, if not present
                for sig in inferred:
                    if not sig.args or sig.args[0].name not in ("self", "cls"):
                        sig.args.insert(0, ArgSig(name=self_var))
                break
    else:
        # function:
        for sig_gen in sig_generators:
            inferred = sig_gen.get_function_sig(obj, module.__name__, name)
            if inferred:
                break

    if not inferred:
        raise ValueError(
            "No signature was found. This should never happen "
            "if FallbackSignatureGenerator is provided"
        )

    is_overloaded = len(inferred) > 1 if inferred else False
    if is_overloaded:
        imports.append("from typing import overload")
    if inferred:
        for signature in inferred:
            args: list[str] = []
            for arg in signature.args:
                arg_def = arg.name
                if arg_def == "None":
                    arg_def = "_none"  # None is not a valid argument name

                if arg.type:
                    arg_def += ": " + strip_or_import(arg.type, module, known_modules, imports)

                if arg.default:
                    arg_def += " = ..."

                args.append(arg_def)

            if is_overloaded:
                output.append("@overload")
            # a sig generator indicates @classmethod by specifying the cls arg
            if class_name and signature.args and signature.args[0].name == "cls":
                output.append("@classmethod")
            output.append(
                "def {function}({args}) -> {ret}: ...".format(
                    function=name,
                    args=", ".join(args),
                    ret=strip_or_import(signature.ret_type, module, known_modules, imports),
                )
            )


def strip_or_import(
    typ: str, module: ModuleType, known_modules: list[str], imports: list[str]
) -> str:
    """Strips unnecessary module names from typ.

    If typ represents a type that is inside module or is a type coming from builtins, remove
    module declaration from it. Return stripped name of the type.

    Arguments:
        typ: name of the type
        module: in which this type is used
        known_modules: other modules being processed
        imports: list of import statements (may be modified during the call)
    """
    local_modules = ["builtins"]
    if module:
        local_modules.append(module.__name__)

    stripped_type = typ
    if any(c in typ for c in "[,"):
        for subtyp in re.split(r"[\[,\]]", typ):
            stripped_subtyp = strip_or_import(subtyp.strip(), module, known_modules, imports)
            if stripped_subtyp != subtyp:
                stripped_type = re.sub(
                    r"(^|[\[, ]+)" + re.escape(subtyp) + r"($|[\], ]+)",
                    r"\1" + stripped_subtyp + r"\2",
                    stripped_type,
                )
    elif "." in typ:
        for module_name in local_modules + list(reversed(known_modules)):
            if typ.startswith(module_name + "."):
                if module_name in local_modules:
                    stripped_type = typ[len(module_name) + 1 :]
                arg_module = module_name
                break
        else:
            arg_module = typ[: typ.rindex(".")]
        if arg_module not in local_modules:
            imports.append(f"import {arg_module}")
    if stripped_type == "NoneType":
        stripped_type = "None"
    return stripped_type


def is_static_property(obj: object) -> bool:
    return type(obj).__name__ == "pybind11_static_property"


def generate_c_property_stub(
    name: str,
    obj: object,
    static_properties: list[str],
    rw_properties: list[str],
    ro_properties: list[str],
    readonly: bool,
    module: ModuleType | None = None,
    known_modules: list[str] | None = None,
    imports: list[str] | None = None,
) -> None:
    """Generate property stub using introspection of 'obj'.

    Try to infer type from docstring, append resulting lines to 'output'.
    """

    def infer_prop_type(docstr: str | None) -> str | None:
        """Infer property type from docstring or docstring signature."""
        if docstr is not None:
            inferred = infer_ret_type_sig_from_anon_docstring(docstr)
            if not inferred:
                inferred = infer_ret_type_sig_from_docstring(docstr, name)
            if not inferred:
                inferred = infer_prop_type_from_docstring(docstr)
            return inferred
        else:
            return None

    inferred = infer_prop_type(getattr(obj, "__doc__", None))
    if not inferred:
        fget = getattr(obj, "fget", None)
        inferred = infer_prop_type(getattr(fget, "__doc__", None))
    if not inferred:
        inferred = "Any"

    if module is not None and imports is not None and known_modules is not None:
        inferred = strip_or_import(inferred, module, known_modules, imports)

    if is_static_property(obj):
        trailing_comment = "  # read-only" if readonly else ""
        static_properties.append(f"{name}: ClassVar[{inferred}] = ...{trailing_comment}")
    else:  # regular property
        if readonly:
            ro_properties.append("@property")
            ro_properties.append(f"def {name}(self) -> {inferred}: ...")
        else:
            rw_properties.append(f"{name}: {inferred}")


def generate_c_type_stub(
    module: ModuleType,
    class_name: str,
    obj: type,
    output: list[str],
    known_modules: list[str],
    imports: list[str],
    sig_generators: Iterable[SignatureGenerator],
) -> None:
    """Generate stub for a single class using runtime introspection.

    The result lines will be appended to 'output'. If necessary, any
    required names will be added to 'imports'.
    """
    raw_lookup = getattr(obj, "__dict__")  # noqa: B009
    items = sorted(get_members(obj), key=lambda x: method_name_sort_key(x[0]))
    names = set(x[0] for x in items)
    methods: list[str] = []
    types: list[str] = []
    static_properties: list[str] = []
    rw_properties: list[str] = []
    ro_properties: list[str] = []
    attrs: list[tuple[str, Any]] = []
    for attr, value in items:
        # use unevaluated descriptors when dealing with property inspection
        raw_value = raw_lookup.get(attr, value)
        if is_c_method(value) or is_c_classmethod(value):
            if attr == "__new__":
                # TODO: We should support __new__.
                if "__init__" in names:
                    # Avoid duplicate functions if both are present.
                    # But is there any case where .__new__() has a
                    # better signature than __init__() ?
                    continue
                attr = "__init__"
            if is_c_classmethod(value):
                self_var = "cls"
            else:
                self_var = "self"
            generate_c_function_stub(
                module,
                attr,
                value,
                output=methods,
                known_modules=known_modules,
                imports=imports,
                self_var=self_var,
                cls=obj,
                class_name=class_name,
                sig_generators=sig_generators,
            )
        elif is_c_property(raw_value):
            generate_c_property_stub(
                attr,
                raw_value,
                static_properties,
                rw_properties,
                ro_properties,
                is_c_property_readonly(raw_value),
                module=module,
                known_modules=known_modules,
                imports=imports,
            )
        elif is_c_type(value):
            generate_c_type_stub(
                module,
                attr,
                value,
                types,
                imports=imports,
                known_modules=known_modules,
                sig_generators=sig_generators,
            )
        else:
            attrs.append((attr, value))

    for attr, value in attrs:
        static_properties.append(
            "{}: ClassVar[{}] = ...".format(
                attr,
                strip_or_import(get_type_fullname(type(value)), module, known_modules, imports),
            )
        )
    all_bases = type.mro(obj)
    if all_bases[-1] is object:
        # TODO: Is this always object?
        del all_bases[-1]
    # remove pybind11_object. All classes generated by pybind11 have pybind11_object in their MRO,
    # which only overrides a few functions in object type
    if all_bases and all_bases[-1].__name__ == "pybind11_object":
        del all_bases[-1]
    # remove the class itself
    all_bases = all_bases[1:]
    # Remove base classes of other bases as redundant.
    bases: list[type] = []
    for base in all_bases:
        if not any(issubclass(b, base) for b in bases):
            bases.append(base)
    if bases:
        bases_str = "(%s)" % ", ".join(
            strip_or_import(get_type_fullname(base), module, known_modules, imports)
            for base in bases
        )
    else:
        bases_str = ""
    if types or static_properties or rw_properties or methods or ro_properties:
        output.append(f"class {class_name}{bases_str}:")
        for line in types:
            if (
                output
                and output[-1]
                and not output[-1].startswith("class")
                and line.startswith("class")
            ):
                output.append("")
            output.append("    " + line)
        for line in static_properties:
            output.append(f"    {line}")
        for line in rw_properties:
            output.append(f"    {line}")
        for line in methods:
            output.append(f"    {line}")
        for line in ro_properties:
            output.append(f"    {line}")
    else:
        output.append(f"class {class_name}{bases_str}: ...")


def get_type_fullname(typ: type) -> str:
    return f"{typ.__module__}.{getattr(typ, '__qualname__', typ.__name__)}"


def method_name_sort_key(name: str) -> tuple[int, str]:
    """Sort methods in classes in a typical order.

    I.e.: constructor, normal methods, special methods.
    """
    if name in ("__new__", "__init__"):
        return 0, name
    if name.startswith("__") and name.endswith("__"):
        return 2, name
    return 1, name


def is_pybind_skipped_attribute(attr: str) -> bool:
    return attr.startswith("__pybind11_module_local_")


def is_skipped_attribute(attr: str) -> bool:
    return attr in (
        "__class__",
        "__getattribute__",
        "__str__",
        "__repr__",
        "__doc__",
        "__dict__",
        "__module__",
        "__weakref__",
    ) or is_pybind_skipped_attribute(  # For pickling
        attr
    )


def infer_method_args(name: str, self_var: str | None = None) -> list[ArgSig]:
    args: list[ArgSig] | None = None
    if name.startswith("__") and name.endswith("__"):
        name = name[2:-2]
        if name in (
            "hash",
            "iter",
            "next",
            "sizeof",
            "copy",
            "deepcopy",
            "reduce",
            "getinitargs",
            "int",
            "float",
            "trunc",
            "complex",
            "bool",
            "abs",
            "bytes",
            "dir",
            "len",
            "reversed",
            "round",
            "index",
            "enter",
        ):
            args = []
        elif name == "getitem":
            args = [ArgSig(name="index")]
        elif name == "setitem":
            args = [ArgSig(name="index"), ArgSig(name="object")]
        elif name in ("delattr", "getattr"):
            args = [ArgSig(name="name")]
        elif name == "setattr":
            args = [ArgSig(name="name"), ArgSig(name="value")]
        elif name == "getstate":
            args = []
        elif name == "setstate":
            args = [ArgSig(name="state")]
        elif name in (
            "eq",
            "ne",
            "lt",
            "le",
            "gt",
            "ge",
            "add",
            "radd",
            "sub",
            "rsub",
            "mul",
            "rmul",
            "mod",
            "rmod",
            "floordiv",
            "rfloordiv",
            "truediv",
            "rtruediv",
            "divmod",
            "rdivmod",
            "pow",
            "rpow",
            "xor",
            "rxor",
            "or",
            "ror",
            "and",
            "rand",
            "lshift",
            "rlshift",
            "rshift",
            "rrshift",
            "contains",
            "delitem",
            "iadd",
            "iand",
            "ifloordiv",
            "ilshift",
            "imod",
            "imul",
            "ior",
            "ipow",
            "irshift",
            "isub",
            "itruediv",
            "ixor",
        ):
            args = [ArgSig(name="other")]
        elif name in ("neg", "pos", "invert"):
            args = []
        elif name == "get":
            args = [ArgSig(name="instance"), ArgSig(name="owner")]
        elif name == "set":
            args = [ArgSig(name="instance"), ArgSig(name="value")]
        elif name == "reduce_ex":
            args = [ArgSig(name="protocol")]
        elif name == "exit":
            args = [ArgSig(name="type"), ArgSig(name="value"), ArgSig(name="traceback")]
    if args is None:
        args = [ArgSig(name="*args"), ArgSig(name="**kwargs")]
    return [ArgSig(name=self_var or "self")] + args


def infer_method_ret_type(name: str) -> str:
    if name.startswith("__") and name.endswith("__"):
        name = name[2:-2]
        if name in ("float", "bool", "bytes", "int"):
            return name
        # Note: __eq__ and co may return arbitrary types, but bool is good enough for stubgen.
        elif name in ("eq", "ne", "lt", "le", "gt", "ge", "contains"):
            return "bool"
        elif name in ("len", "hash", "sizeof", "trunc", "floor", "ceil"):
            return "int"
        elif name in ("init", "setitem"):
            return "None"
    return "Any"